1 //
2 // Copyright (c) 2003, 2022, Oracle and/or its affiliates. All rights reserved.
3 // Copyright (c) 2014, 2021, Red Hat, Inc. All rights reserved.
4 // DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
5 //
6 // This code is free software; you can redistribute it and/or modify it
7 // under the terms of the GNU General Public License version 2 only, as
8 // published by the Free Software Foundation.
9 //
10 // This code is distributed in the hope that it will be useful, but WITHOUT
11 // ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
12 // FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
13 // version 2 for more details (a copy is included in the LICENSE file that
14 // accompanied this code).
15 //
16 // You should have received a copy of the GNU General Public License version
17 // 2 along with this work; if not, write to the Free Software Foundation,
18 // Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
19 //
20 // Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
21 // or visit www.oracle.com if you need additional information or have any
22 // questions.
23 //
24 //
25
26 // AArch64 Architecture Description File
27
28 //----------REGISTER DEFINITION BLOCK------------------------------------------
29 // This information is used by the matcher and the register allocator to
30 // describe individual registers and classes of registers within the target
31 // architecture.
32
33 register %{
34 //----------Architecture Description Register Definitions----------------------
35 // General Registers
36 // "reg_def" name ( register save type, C convention save type,
37 // ideal register type, encoding );
38 // Register Save Types:
39 //
40 // NS = No-Save: The register allocator assumes that these registers
41 // can be used without saving upon entry to the method, &
42 // that they do not need to be saved at call sites.
43 //
44 // SOC = Save-On-Call: The register allocator assumes that these registers
45 // can be used without saving upon entry to the method,
46 // but that they must be saved at call sites.
47 //
48 // SOE = Save-On-Entry: The register allocator assumes that these registers
49 // must be saved before using them upon entry to the
50 // method, but they do not need to be saved at call
51 // sites.
52 //
53 // AS = Always-Save: The register allocator assumes that these registers
54 // must be saved before using them upon entry to the
55 // method, & that they must be saved at call sites.
56 //
57 // Ideal Register Type is used to determine how to save & restore a
58 // register. Op_RegI will get spilled with LoadI/StoreI, Op_RegP will get
59 // spilled with LoadP/StoreP. If the register supports both, use Op_RegI.
60 //
61 // The encoding number is the actual bit-pattern placed into the opcodes.
62
63 // We must define the 64 bit int registers in two 32 bit halves, the
64 // real lower register and a virtual upper half register. upper halves
65 // are used by the register allocator but are not actually supplied as
66 // operands to memory ops.
67 //
68 // follow the C1 compiler in making registers
69 //
70 // r0-r7,r10-r26 volatile (caller save)
71 // r27-r32 system (no save, no allocate)
72 // r8-r9 non-allocatable (so we can use them as scratch regs)
73 //
74 // as regards Java usage. we don't use any callee save registers
75 // because this makes it difficult to de-optimise a frame (see comment
76 // in x86 implementation of Deoptimization::unwind_callee_save_values)
77 //
78
79 // General Registers
80
81 reg_def R0 ( SOC, SOC, Op_RegI, 0, r0->as_VMReg() );
82 reg_def R0_H ( SOC, SOC, Op_RegI, 0, r0->as_VMReg()->next() );
83 reg_def R1 ( SOC, SOC, Op_RegI, 1, r1->as_VMReg() );
84 reg_def R1_H ( SOC, SOC, Op_RegI, 1, r1->as_VMReg()->next() );
85 reg_def R2 ( SOC, SOC, Op_RegI, 2, r2->as_VMReg() );
86 reg_def R2_H ( SOC, SOC, Op_RegI, 2, r2->as_VMReg()->next() );
87 reg_def R3 ( SOC, SOC, Op_RegI, 3, r3->as_VMReg() );
88 reg_def R3_H ( SOC, SOC, Op_RegI, 3, r3->as_VMReg()->next() );
89 reg_def R4 ( SOC, SOC, Op_RegI, 4, r4->as_VMReg() );
90 reg_def R4_H ( SOC, SOC, Op_RegI, 4, r4->as_VMReg()->next() );
91 reg_def R5 ( SOC, SOC, Op_RegI, 5, r5->as_VMReg() );
92 reg_def R5_H ( SOC, SOC, Op_RegI, 5, r5->as_VMReg()->next() );
93 reg_def R6 ( SOC, SOC, Op_RegI, 6, r6->as_VMReg() );
94 reg_def R6_H ( SOC, SOC, Op_RegI, 6, r6->as_VMReg()->next() );
95 reg_def R7 ( SOC, SOC, Op_RegI, 7, r7->as_VMReg() );
96 reg_def R7_H ( SOC, SOC, Op_RegI, 7, r7->as_VMReg()->next() );
97 reg_def R8 ( NS, SOC, Op_RegI, 8, r8->as_VMReg() ); // rscratch1, non-allocatable
98 reg_def R8_H ( NS, SOC, Op_RegI, 8, r8->as_VMReg()->next() );
99 reg_def R9 ( NS, SOC, Op_RegI, 9, r9->as_VMReg() ); // rscratch2, non-allocatable
100 reg_def R9_H ( NS, SOC, Op_RegI, 9, r9->as_VMReg()->next() );
101 reg_def R10 ( SOC, SOC, Op_RegI, 10, r10->as_VMReg() );
102 reg_def R10_H ( SOC, SOC, Op_RegI, 10, r10->as_VMReg()->next());
103 reg_def R11 ( SOC, SOC, Op_RegI, 11, r11->as_VMReg() );
104 reg_def R11_H ( SOC, SOC, Op_RegI, 11, r11->as_VMReg()->next());
105 reg_def R12 ( SOC, SOC, Op_RegI, 12, r12->as_VMReg() );
106 reg_def R12_H ( SOC, SOC, Op_RegI, 12, r12->as_VMReg()->next());
107 reg_def R13 ( SOC, SOC, Op_RegI, 13, r13->as_VMReg() );
108 reg_def R13_H ( SOC, SOC, Op_RegI, 13, r13->as_VMReg()->next());
109 reg_def R14 ( SOC, SOC, Op_RegI, 14, r14->as_VMReg() );
110 reg_def R14_H ( SOC, SOC, Op_RegI, 14, r14->as_VMReg()->next());
111 reg_def R15 ( SOC, SOC, Op_RegI, 15, r15->as_VMReg() );
112 reg_def R15_H ( SOC, SOC, Op_RegI, 15, r15->as_VMReg()->next());
113 reg_def R16 ( SOC, SOC, Op_RegI, 16, r16->as_VMReg() );
114 reg_def R16_H ( SOC, SOC, Op_RegI, 16, r16->as_VMReg()->next());
115 reg_def R17 ( SOC, SOC, Op_RegI, 17, r17->as_VMReg() );
116 reg_def R17_H ( SOC, SOC, Op_RegI, 17, r17->as_VMReg()->next());
117 reg_def R18 ( SOC, SOC, Op_RegI, 18, r18_tls->as_VMReg() );
118 reg_def R18_H ( SOC, SOC, Op_RegI, 18, r18_tls->as_VMReg()->next());
119 reg_def R19 ( SOC, SOE, Op_RegI, 19, r19->as_VMReg() );
120 reg_def R19_H ( SOC, SOE, Op_RegI, 19, r19->as_VMReg()->next());
121 reg_def R20 ( SOC, SOE, Op_RegI, 20, r20->as_VMReg() ); // caller esp
122 reg_def R20_H ( SOC, SOE, Op_RegI, 20, r20->as_VMReg()->next());
123 reg_def R21 ( SOC, SOE, Op_RegI, 21, r21->as_VMReg() );
124 reg_def R21_H ( SOC, SOE, Op_RegI, 21, r21->as_VMReg()->next());
125 reg_def R22 ( SOC, SOE, Op_RegI, 22, r22->as_VMReg() );
126 reg_def R22_H ( SOC, SOE, Op_RegI, 22, r22->as_VMReg()->next());
127 reg_def R23 ( SOC, SOE, Op_RegI, 23, r23->as_VMReg() );
128 reg_def R23_H ( SOC, SOE, Op_RegI, 23, r23->as_VMReg()->next());
129 reg_def R24 ( SOC, SOE, Op_RegI, 24, r24->as_VMReg() );
130 reg_def R24_H ( SOC, SOE, Op_RegI, 24, r24->as_VMReg()->next());
131 reg_def R25 ( SOC, SOE, Op_RegI, 25, r25->as_VMReg() );
132 reg_def R25_H ( SOC, SOE, Op_RegI, 25, r25->as_VMReg()->next());
133 reg_def R26 ( SOC, SOE, Op_RegI, 26, r26->as_VMReg() );
134 reg_def R26_H ( SOC, SOE, Op_RegI, 26, r26->as_VMReg()->next());
135 reg_def R27 ( SOC, SOE, Op_RegI, 27, r27->as_VMReg() ); // heapbase
136 reg_def R27_H ( SOC, SOE, Op_RegI, 27, r27->as_VMReg()->next());
137 reg_def R28 ( NS, SOE, Op_RegI, 28, r28->as_VMReg() ); // thread
138 reg_def R28_H ( NS, SOE, Op_RegI, 28, r28->as_VMReg()->next());
139 reg_def R29 ( NS, NS, Op_RegI, 29, r29->as_VMReg() ); // fp
140 reg_def R29_H ( NS, NS, Op_RegI, 29, r29->as_VMReg()->next());
141 reg_def R30 ( NS, NS, Op_RegI, 30, r30->as_VMReg() ); // lr
142 reg_def R30_H ( NS, NS, Op_RegI, 30, r30->as_VMReg()->next());
143 reg_def R31 ( NS, NS, Op_RegI, 31, r31_sp->as_VMReg() ); // sp
144 reg_def R31_H ( NS, NS, Op_RegI, 31, r31_sp->as_VMReg()->next());
145
146 // ----------------------------
147 // Float/Double/Vector Registers
148 // ----------------------------
149
150 // Double Registers
151
152 // The rules of ADL require that double registers be defined in pairs.
153 // Each pair must be two 32-bit values, but not necessarily a pair of
154 // single float registers. In each pair, ADLC-assigned register numbers
155 // must be adjacent, with the lower number even. Finally, when the
156 // CPU stores such a register pair to memory, the word associated with
157 // the lower ADLC-assigned number must be stored to the lower address.
158
159 // AArch64 has 32 floating-point registers. Each can store a vector of
160 // single or double precision floating-point values up to 8 * 32
161 // floats, 4 * 64 bit floats or 2 * 128 bit floats. We currently only
162 // use the first float or double element of the vector.
163
164 // for Java use float registers v0-v15 are always save on call whereas
165 // the platform ABI treats v8-v15 as callee save). float registers
166 // v16-v31 are SOC as per the platform spec
167
168 // For SVE vector registers, we simply extend vector register size to 8
169 // 'logical' slots. This is nominally 256 bits but it actually covers
170 // all possible 'physical' SVE vector register lengths from 128 ~ 2048
171 // bits. The 'physical' SVE vector register length is detected during
172 // startup, so the register allocator is able to identify the correct
173 // number of bytes needed for an SVE spill/unspill.
174 // Note that a vector register with 4 slots denotes a 128-bit NEON
175 // register allowing it to be distinguished from the corresponding SVE
176 // vector register when the SVE vector length is 128 bits.
177
178 reg_def V0 ( SOC, SOC, Op_RegF, 0, v0->as_VMReg() );
179 reg_def V0_H ( SOC, SOC, Op_RegF, 0, v0->as_VMReg()->next() );
180 reg_def V0_J ( SOC, SOC, Op_RegF, 0, v0->as_VMReg()->next(2) );
181 reg_def V0_K ( SOC, SOC, Op_RegF, 0, v0->as_VMReg()->next(3) );
182
183 reg_def V1 ( SOC, SOC, Op_RegF, 1, v1->as_VMReg() );
184 reg_def V1_H ( SOC, SOC, Op_RegF, 1, v1->as_VMReg()->next() );
185 reg_def V1_J ( SOC, SOC, Op_RegF, 1, v1->as_VMReg()->next(2) );
186 reg_def V1_K ( SOC, SOC, Op_RegF, 1, v1->as_VMReg()->next(3) );
187
188 reg_def V2 ( SOC, SOC, Op_RegF, 2, v2->as_VMReg() );
189 reg_def V2_H ( SOC, SOC, Op_RegF, 2, v2->as_VMReg()->next() );
190 reg_def V2_J ( SOC, SOC, Op_RegF, 2, v2->as_VMReg()->next(2) );
191 reg_def V2_K ( SOC, SOC, Op_RegF, 2, v2->as_VMReg()->next(3) );
192
193 reg_def V3 ( SOC, SOC, Op_RegF, 3, v3->as_VMReg() );
194 reg_def V3_H ( SOC, SOC, Op_RegF, 3, v3->as_VMReg()->next() );
195 reg_def V3_J ( SOC, SOC, Op_RegF, 3, v3->as_VMReg()->next(2) );
196 reg_def V3_K ( SOC, SOC, Op_RegF, 3, v3->as_VMReg()->next(3) );
197
198 reg_def V4 ( SOC, SOC, Op_RegF, 4, v4->as_VMReg() );
199 reg_def V4_H ( SOC, SOC, Op_RegF, 4, v4->as_VMReg()->next() );
200 reg_def V4_J ( SOC, SOC, Op_RegF, 4, v4->as_VMReg()->next(2) );
201 reg_def V4_K ( SOC, SOC, Op_RegF, 4, v4->as_VMReg()->next(3) );
202
203 reg_def V5 ( SOC, SOC, Op_RegF, 5, v5->as_VMReg() );
204 reg_def V5_H ( SOC, SOC, Op_RegF, 5, v5->as_VMReg()->next() );
205 reg_def V5_J ( SOC, SOC, Op_RegF, 5, v5->as_VMReg()->next(2) );
206 reg_def V5_K ( SOC, SOC, Op_RegF, 5, v5->as_VMReg()->next(3) );
207
208 reg_def V6 ( SOC, SOC, Op_RegF, 6, v6->as_VMReg() );
209 reg_def V6_H ( SOC, SOC, Op_RegF, 6, v6->as_VMReg()->next() );
210 reg_def V6_J ( SOC, SOC, Op_RegF, 6, v6->as_VMReg()->next(2) );
211 reg_def V6_K ( SOC, SOC, Op_RegF, 6, v6->as_VMReg()->next(3) );
212
213 reg_def V7 ( SOC, SOC, Op_RegF, 7, v7->as_VMReg() );
214 reg_def V7_H ( SOC, SOC, Op_RegF, 7, v7->as_VMReg()->next() );
215 reg_def V7_J ( SOC, SOC, Op_RegF, 7, v7->as_VMReg()->next(2) );
216 reg_def V7_K ( SOC, SOC, Op_RegF, 7, v7->as_VMReg()->next(3) );
217
218 reg_def V8 ( SOC, SOE, Op_RegF, 8, v8->as_VMReg() );
219 reg_def V8_H ( SOC, SOE, Op_RegF, 8, v8->as_VMReg()->next() );
220 reg_def V8_J ( SOC, SOC, Op_RegF, 8, v8->as_VMReg()->next(2) );
221 reg_def V8_K ( SOC, SOC, Op_RegF, 8, v8->as_VMReg()->next(3) );
222
223 reg_def V9 ( SOC, SOE, Op_RegF, 9, v9->as_VMReg() );
224 reg_def V9_H ( SOC, SOE, Op_RegF, 9, v9->as_VMReg()->next() );
225 reg_def V9_J ( SOC, SOC, Op_RegF, 9, v9->as_VMReg()->next(2) );
226 reg_def V9_K ( SOC, SOC, Op_RegF, 9, v9->as_VMReg()->next(3) );
227
228 reg_def V10 ( SOC, SOE, Op_RegF, 10, v10->as_VMReg() );
229 reg_def V10_H ( SOC, SOE, Op_RegF, 10, v10->as_VMReg()->next() );
230 reg_def V10_J ( SOC, SOC, Op_RegF, 10, v10->as_VMReg()->next(2) );
231 reg_def V10_K ( SOC, SOC, Op_RegF, 10, v10->as_VMReg()->next(3) );
232
233 reg_def V11 ( SOC, SOE, Op_RegF, 11, v11->as_VMReg() );
234 reg_def V11_H ( SOC, SOE, Op_RegF, 11, v11->as_VMReg()->next() );
235 reg_def V11_J ( SOC, SOC, Op_RegF, 11, v11->as_VMReg()->next(2) );
236 reg_def V11_K ( SOC, SOC, Op_RegF, 11, v11->as_VMReg()->next(3) );
237
238 reg_def V12 ( SOC, SOE, Op_RegF, 12, v12->as_VMReg() );
239 reg_def V12_H ( SOC, SOE, Op_RegF, 12, v12->as_VMReg()->next() );
240 reg_def V12_J ( SOC, SOC, Op_RegF, 12, v12->as_VMReg()->next(2) );
241 reg_def V12_K ( SOC, SOC, Op_RegF, 12, v12->as_VMReg()->next(3) );
242
243 reg_def V13 ( SOC, SOE, Op_RegF, 13, v13->as_VMReg() );
244 reg_def V13_H ( SOC, SOE, Op_RegF, 13, v13->as_VMReg()->next() );
245 reg_def V13_J ( SOC, SOC, Op_RegF, 13, v13->as_VMReg()->next(2) );
246 reg_def V13_K ( SOC, SOC, Op_RegF, 13, v13->as_VMReg()->next(3) );
247
248 reg_def V14 ( SOC, SOE, Op_RegF, 14, v14->as_VMReg() );
249 reg_def V14_H ( SOC, SOE, Op_RegF, 14, v14->as_VMReg()->next() );
250 reg_def V14_J ( SOC, SOC, Op_RegF, 14, v14->as_VMReg()->next(2) );
251 reg_def V14_K ( SOC, SOC, Op_RegF, 14, v14->as_VMReg()->next(3) );
252
253 reg_def V15 ( SOC, SOE, Op_RegF, 15, v15->as_VMReg() );
254 reg_def V15_H ( SOC, SOE, Op_RegF, 15, v15->as_VMReg()->next() );
255 reg_def V15_J ( SOC, SOC, Op_RegF, 15, v15->as_VMReg()->next(2) );
256 reg_def V15_K ( SOC, SOC, Op_RegF, 15, v15->as_VMReg()->next(3) );
257
258 reg_def V16 ( SOC, SOC, Op_RegF, 16, v16->as_VMReg() );
259 reg_def V16_H ( SOC, SOC, Op_RegF, 16, v16->as_VMReg()->next() );
260 reg_def V16_J ( SOC, SOC, Op_RegF, 16, v16->as_VMReg()->next(2) );
261 reg_def V16_K ( SOC, SOC, Op_RegF, 16, v16->as_VMReg()->next(3) );
262
263 reg_def V17 ( SOC, SOC, Op_RegF, 17, v17->as_VMReg() );
264 reg_def V17_H ( SOC, SOC, Op_RegF, 17, v17->as_VMReg()->next() );
265 reg_def V17_J ( SOC, SOC, Op_RegF, 17, v17->as_VMReg()->next(2) );
266 reg_def V17_K ( SOC, SOC, Op_RegF, 17, v17->as_VMReg()->next(3) );
267
268 reg_def V18 ( SOC, SOC, Op_RegF, 18, v18->as_VMReg() );
269 reg_def V18_H ( SOC, SOC, Op_RegF, 18, v18->as_VMReg()->next() );
270 reg_def V18_J ( SOC, SOC, Op_RegF, 18, v18->as_VMReg()->next(2) );
271 reg_def V18_K ( SOC, SOC, Op_RegF, 18, v18->as_VMReg()->next(3) );
272
273 reg_def V19 ( SOC, SOC, Op_RegF, 19, v19->as_VMReg() );
274 reg_def V19_H ( SOC, SOC, Op_RegF, 19, v19->as_VMReg()->next() );
275 reg_def V19_J ( SOC, SOC, Op_RegF, 19, v19->as_VMReg()->next(2) );
276 reg_def V19_K ( SOC, SOC, Op_RegF, 19, v19->as_VMReg()->next(3) );
277
278 reg_def V20 ( SOC, SOC, Op_RegF, 20, v20->as_VMReg() );
279 reg_def V20_H ( SOC, SOC, Op_RegF, 20, v20->as_VMReg()->next() );
280 reg_def V20_J ( SOC, SOC, Op_RegF, 20, v20->as_VMReg()->next(2) );
281 reg_def V20_K ( SOC, SOC, Op_RegF, 20, v20->as_VMReg()->next(3) );
282
283 reg_def V21 ( SOC, SOC, Op_RegF, 21, v21->as_VMReg() );
284 reg_def V21_H ( SOC, SOC, Op_RegF, 21, v21->as_VMReg()->next() );
285 reg_def V21_J ( SOC, SOC, Op_RegF, 21, v21->as_VMReg()->next(2) );
286 reg_def V21_K ( SOC, SOC, Op_RegF, 21, v21->as_VMReg()->next(3) );
287
288 reg_def V22 ( SOC, SOC, Op_RegF, 22, v22->as_VMReg() );
289 reg_def V22_H ( SOC, SOC, Op_RegF, 22, v22->as_VMReg()->next() );
290 reg_def V22_J ( SOC, SOC, Op_RegF, 22, v22->as_VMReg()->next(2) );
291 reg_def V22_K ( SOC, SOC, Op_RegF, 22, v22->as_VMReg()->next(3) );
292
293 reg_def V23 ( SOC, SOC, Op_RegF, 23, v23->as_VMReg() );
294 reg_def V23_H ( SOC, SOC, Op_RegF, 23, v23->as_VMReg()->next() );
295 reg_def V23_J ( SOC, SOC, Op_RegF, 23, v23->as_VMReg()->next(2) );
296 reg_def V23_K ( SOC, SOC, Op_RegF, 23, v23->as_VMReg()->next(3) );
297
298 reg_def V24 ( SOC, SOC, Op_RegF, 24, v24->as_VMReg() );
299 reg_def V24_H ( SOC, SOC, Op_RegF, 24, v24->as_VMReg()->next() );
300 reg_def V24_J ( SOC, SOC, Op_RegF, 24, v24->as_VMReg()->next(2) );
301 reg_def V24_K ( SOC, SOC, Op_RegF, 24, v24->as_VMReg()->next(3) );
302
303 reg_def V25 ( SOC, SOC, Op_RegF, 25, v25->as_VMReg() );
304 reg_def V25_H ( SOC, SOC, Op_RegF, 25, v25->as_VMReg()->next() );
305 reg_def V25_J ( SOC, SOC, Op_RegF, 25, v25->as_VMReg()->next(2) );
306 reg_def V25_K ( SOC, SOC, Op_RegF, 25, v25->as_VMReg()->next(3) );
307
308 reg_def V26 ( SOC, SOC, Op_RegF, 26, v26->as_VMReg() );
309 reg_def V26_H ( SOC, SOC, Op_RegF, 26, v26->as_VMReg()->next() );
310 reg_def V26_J ( SOC, SOC, Op_RegF, 26, v26->as_VMReg()->next(2) );
311 reg_def V26_K ( SOC, SOC, Op_RegF, 26, v26->as_VMReg()->next(3) );
312
313 reg_def V27 ( SOC, SOC, Op_RegF, 27, v27->as_VMReg() );
314 reg_def V27_H ( SOC, SOC, Op_RegF, 27, v27->as_VMReg()->next() );
315 reg_def V27_J ( SOC, SOC, Op_RegF, 27, v27->as_VMReg()->next(2) );
316 reg_def V27_K ( SOC, SOC, Op_RegF, 27, v27->as_VMReg()->next(3) );
317
318 reg_def V28 ( SOC, SOC, Op_RegF, 28, v28->as_VMReg() );
319 reg_def V28_H ( SOC, SOC, Op_RegF, 28, v28->as_VMReg()->next() );
320 reg_def V28_J ( SOC, SOC, Op_RegF, 28, v28->as_VMReg()->next(2) );
321 reg_def V28_K ( SOC, SOC, Op_RegF, 28, v28->as_VMReg()->next(3) );
322
323 reg_def V29 ( SOC, SOC, Op_RegF, 29, v29->as_VMReg() );
324 reg_def V29_H ( SOC, SOC, Op_RegF, 29, v29->as_VMReg()->next() );
325 reg_def V29_J ( SOC, SOC, Op_RegF, 29, v29->as_VMReg()->next(2) );
326 reg_def V29_K ( SOC, SOC, Op_RegF, 29, v29->as_VMReg()->next(3) );
327
328 reg_def V30 ( SOC, SOC, Op_RegF, 30, v30->as_VMReg() );
329 reg_def V30_H ( SOC, SOC, Op_RegF, 30, v30->as_VMReg()->next() );
330 reg_def V30_J ( SOC, SOC, Op_RegF, 30, v30->as_VMReg()->next(2) );
331 reg_def V30_K ( SOC, SOC, Op_RegF, 30, v30->as_VMReg()->next(3) );
332
333 reg_def V31 ( SOC, SOC, Op_RegF, 31, v31->as_VMReg() );
334 reg_def V31_H ( SOC, SOC, Op_RegF, 31, v31->as_VMReg()->next() );
335 reg_def V31_J ( SOC, SOC, Op_RegF, 31, v31->as_VMReg()->next(2) );
336 reg_def V31_K ( SOC, SOC, Op_RegF, 31, v31->as_VMReg()->next(3) );
337
338 // ----------------------------
339 // SVE Predicate Registers
340 // ----------------------------
341 reg_def P0 (SOC, SOC, Op_RegVectMask, 0, p0->as_VMReg());
342 reg_def P1 (SOC, SOC, Op_RegVectMask, 1, p1->as_VMReg());
343 reg_def P2 (SOC, SOC, Op_RegVectMask, 2, p2->as_VMReg());
344 reg_def P3 (SOC, SOC, Op_RegVectMask, 3, p3->as_VMReg());
345 reg_def P4 (SOC, SOC, Op_RegVectMask, 4, p4->as_VMReg());
346 reg_def P5 (SOC, SOC, Op_RegVectMask, 5, p5->as_VMReg());
347 reg_def P6 (SOC, SOC, Op_RegVectMask, 6, p6->as_VMReg());
348 reg_def P7 (SOC, SOC, Op_RegVectMask, 7, p7->as_VMReg());
349 reg_def P8 (SOC, SOC, Op_RegVectMask, 8, p8->as_VMReg());
350 reg_def P9 (SOC, SOC, Op_RegVectMask, 9, p9->as_VMReg());
351 reg_def P10 (SOC, SOC, Op_RegVectMask, 10, p10->as_VMReg());
352 reg_def P11 (SOC, SOC, Op_RegVectMask, 11, p11->as_VMReg());
353 reg_def P12 (SOC, SOC, Op_RegVectMask, 12, p12->as_VMReg());
354 reg_def P13 (SOC, SOC, Op_RegVectMask, 13, p13->as_VMReg());
355 reg_def P14 (SOC, SOC, Op_RegVectMask, 14, p14->as_VMReg());
356 reg_def P15 (SOC, SOC, Op_RegVectMask, 15, p15->as_VMReg());
357
358 // ----------------------------
359 // Special Registers
360 // ----------------------------
361
362 // the AArch64 CSPR status flag register is not directly accessible as
363 // instruction operand. the FPSR status flag register is a system
364 // register which can be written/read using MSR/MRS but again does not
365 // appear as an operand (a code identifying the FSPR occurs as an
366 // immediate value in the instruction).
367
368 reg_def RFLAGS(SOC, SOC, 0, 32, VMRegImpl::Bad());
369
370 // Specify priority of register selection within phases of register
371 // allocation. Highest priority is first. A useful heuristic is to
372 // give registers a low priority when they are required by machine
373 // instructions, like EAX and EDX on I486, and choose no-save registers
374 // before save-on-call, & save-on-call before save-on-entry. Registers
375 // which participate in fixed calling sequences should come last.
376 // Registers which are used as pairs must fall on an even boundary.
377
378 alloc_class chunk0(
379 // volatiles
380 R10, R10_H,
381 R11, R11_H,
382 R12, R12_H,
383 R13, R13_H,
384 R14, R14_H,
385 R15, R15_H,
386 R16, R16_H,
387 R17, R17_H,
388 R18, R18_H,
389
390 // arg registers
391 R0, R0_H,
392 R1, R1_H,
393 R2, R2_H,
394 R3, R3_H,
395 R4, R4_H,
396 R5, R5_H,
397 R6, R6_H,
398 R7, R7_H,
399
400 // non-volatiles
401 R19, R19_H,
402 R20, R20_H,
403 R21, R21_H,
404 R22, R22_H,
405 R23, R23_H,
406 R24, R24_H,
407 R25, R25_H,
408 R26, R26_H,
409
410 // non-allocatable registers
411
412 R27, R27_H, // heapbase
413 R28, R28_H, // thread
414 R29, R29_H, // fp
415 R30, R30_H, // lr
416 R31, R31_H, // sp
417 R8, R8_H, // rscratch1
418 R9, R9_H, // rscratch2
419 );
420
421 alloc_class chunk1(
422
423 // no save
424 V16, V16_H, V16_J, V16_K,
425 V17, V17_H, V17_J, V17_K,
426 V18, V18_H, V18_J, V18_K,
427 V19, V19_H, V19_J, V19_K,
428 V20, V20_H, V20_J, V20_K,
429 V21, V21_H, V21_J, V21_K,
430 V22, V22_H, V22_J, V22_K,
431 V23, V23_H, V23_J, V23_K,
432 V24, V24_H, V24_J, V24_K,
433 V25, V25_H, V25_J, V25_K,
434 V26, V26_H, V26_J, V26_K,
435 V27, V27_H, V27_J, V27_K,
436 V28, V28_H, V28_J, V28_K,
437 V29, V29_H, V29_J, V29_K,
438 V30, V30_H, V30_J, V30_K,
439 V31, V31_H, V31_J, V31_K,
440
441 // arg registers
442 V0, V0_H, V0_J, V0_K,
443 V1, V1_H, V1_J, V1_K,
444 V2, V2_H, V2_J, V2_K,
445 V3, V3_H, V3_J, V3_K,
446 V4, V4_H, V4_J, V4_K,
447 V5, V5_H, V5_J, V5_K,
448 V6, V6_H, V6_J, V6_K,
449 V7, V7_H, V7_J, V7_K,
450
451 // non-volatiles
452 V8, V8_H, V8_J, V8_K,
453 V9, V9_H, V9_J, V9_K,
454 V10, V10_H, V10_J, V10_K,
455 V11, V11_H, V11_J, V11_K,
456 V12, V12_H, V12_J, V12_K,
457 V13, V13_H, V13_J, V13_K,
458 V14, V14_H, V14_J, V14_K,
459 V15, V15_H, V15_J, V15_K,
460 );
461
462 alloc_class chunk2 (
463 P0,
464 P1,
465 P2,
466 P3,
467 P4,
468 P5,
469 P6,
470 P7,
471
472 P8,
473 P9,
474 P10,
475 P11,
476 P12,
477 P13,
478 P14,
479 P15,
480 );
481
482 alloc_class chunk3(RFLAGS);
483
484 //----------Architecture Description Register Classes--------------------------
485 // Several register classes are automatically defined based upon information in
486 // this architecture description.
487 // 1) reg_class inline_cache_reg ( /* as def'd in frame section */ )
488 // 2) reg_class stack_slots( /* one chunk of stack-based "registers" */ )
489 //
490
491 // Class for all 32 bit general purpose registers
492 reg_class all_reg32(
493 R0,
494 R1,
495 R2,
496 R3,
497 R4,
498 R5,
499 R6,
500 R7,
501 R10,
502 R11,
503 R12,
504 R13,
505 R14,
506 R15,
507 R16,
508 R17,
509 R18,
510 R19,
511 R20,
512 R21,
513 R22,
514 R23,
515 R24,
516 R25,
517 R26,
518 R27,
519 R28,
520 R29,
521 R30,
522 R31
523 );
524
525
526 // Class for all 32 bit integer registers (excluding SP which
527 // will never be used as an integer register)
528 reg_class any_reg32 %{
529 return _ANY_REG32_mask;
530 %}
531
532 // Singleton class for R0 int register
533 reg_class int_r0_reg(R0);
534
535 // Singleton class for R2 int register
536 reg_class int_r2_reg(R2);
537
538 // Singleton class for R3 int register
539 reg_class int_r3_reg(R3);
540
541 // Singleton class for R4 int register
542 reg_class int_r4_reg(R4);
543
544 // Singleton class for R31 int register
545 reg_class int_r31_reg(R31);
546
547 // Class for all 64 bit general purpose registers
548 reg_class all_reg(
549 R0, R0_H,
550 R1, R1_H,
551 R2, R2_H,
552 R3, R3_H,
553 R4, R4_H,
554 R5, R5_H,
555 R6, R6_H,
556 R7, R7_H,
557 R10, R10_H,
558 R11, R11_H,
559 R12, R12_H,
560 R13, R13_H,
561 R14, R14_H,
562 R15, R15_H,
563 R16, R16_H,
564 R17, R17_H,
565 R18, R18_H,
566 R19, R19_H,
567 R20, R20_H,
568 R21, R21_H,
569 R22, R22_H,
570 R23, R23_H,
571 R24, R24_H,
572 R25, R25_H,
573 R26, R26_H,
574 R27, R27_H,
575 R28, R28_H,
576 R29, R29_H,
577 R30, R30_H,
578 R31, R31_H
579 );
580
581 // Class for all long integer registers (including SP)
582 reg_class any_reg %{
583 return _ANY_REG_mask;
584 %}
585
586 // Class for non-allocatable 32 bit registers
587 reg_class non_allocatable_reg32(
588 #ifdef R18_RESERVED
589 // See comment in register_aarch64.hpp
590 R18, // tls on Windows
591 #endif
592 R28, // thread
593 R30, // lr
594 R31 // sp
595 );
596
597 // Class for non-allocatable 64 bit registers
598 reg_class non_allocatable_reg(
599 #ifdef R18_RESERVED
600 // See comment in register_aarch64.hpp
601 R18, R18_H, // tls on Windows, platform register on macOS
602 #endif
603 R28, R28_H, // thread
604 R30, R30_H, // lr
605 R31, R31_H // sp
606 );
607
608 // Class for all non-special integer registers
609 reg_class no_special_reg32 %{
610 return _NO_SPECIAL_REG32_mask;
611 %}
612
613 // Class for all non-special long integer registers
614 reg_class no_special_reg %{
615 return _NO_SPECIAL_REG_mask;
616 %}
617
618 // Class for 64 bit register r0
619 reg_class r0_reg(
620 R0, R0_H
621 );
622
623 // Class for 64 bit register r1
624 reg_class r1_reg(
625 R1, R1_H
626 );
627
628 // Class for 64 bit register r2
629 reg_class r2_reg(
630 R2, R2_H
631 );
632
633 // Class for 64 bit register r3
634 reg_class r3_reg(
635 R3, R3_H
636 );
637
638 // Class for 64 bit register r4
639 reg_class r4_reg(
640 R4, R4_H
641 );
642
643 // Class for 64 bit register r5
644 reg_class r5_reg(
645 R5, R5_H
646 );
647
648 // Class for 64 bit register r10
649 reg_class r10_reg(
650 R10, R10_H
651 );
652
653 // Class for 64 bit register r11
654 reg_class r11_reg(
655 R11, R11_H
656 );
657
658 // Class for method register
659 reg_class method_reg(
660 R12, R12_H
661 );
662
663 // Class for heapbase register
664 reg_class heapbase_reg(
665 R27, R27_H
666 );
667
668 // Class for thread register
669 reg_class thread_reg(
670 R28, R28_H
671 );
672
673 // Class for frame pointer register
674 reg_class fp_reg(
675 R29, R29_H
676 );
677
678 // Class for link register
679 reg_class lr_reg(
680 R30, R30_H
681 );
682
683 // Class for long sp register
684 reg_class sp_reg(
685 R31, R31_H
686 );
687
688 // Class for all pointer registers
689 reg_class ptr_reg %{
690 return _PTR_REG_mask;
691 %}
692
693 // Class for all non_special pointer registers
694 reg_class no_special_ptr_reg %{
695 return _NO_SPECIAL_PTR_REG_mask;
696 %}
697
698 // Class for all float registers
699 reg_class float_reg(
700 V0,
701 V1,
702 V2,
703 V3,
704 V4,
705 V5,
706 V6,
707 V7,
708 V8,
709 V9,
710 V10,
711 V11,
712 V12,
713 V13,
714 V14,
715 V15,
716 V16,
717 V17,
718 V18,
719 V19,
720 V20,
721 V21,
722 V22,
723 V23,
724 V24,
725 V25,
726 V26,
727 V27,
728 V28,
729 V29,
730 V30,
731 V31
732 );
733
734 // Double precision float registers have virtual `high halves' that
735 // are needed by the allocator.
736 // Class for all double registers
737 reg_class double_reg(
738 V0, V0_H,
739 V1, V1_H,
740 V2, V2_H,
741 V3, V3_H,
742 V4, V4_H,
743 V5, V5_H,
744 V6, V6_H,
745 V7, V7_H,
746 V8, V8_H,
747 V9, V9_H,
748 V10, V10_H,
749 V11, V11_H,
750 V12, V12_H,
751 V13, V13_H,
752 V14, V14_H,
753 V15, V15_H,
754 V16, V16_H,
755 V17, V17_H,
756 V18, V18_H,
757 V19, V19_H,
758 V20, V20_H,
759 V21, V21_H,
760 V22, V22_H,
761 V23, V23_H,
762 V24, V24_H,
763 V25, V25_H,
764 V26, V26_H,
765 V27, V27_H,
766 V28, V28_H,
767 V29, V29_H,
768 V30, V30_H,
769 V31, V31_H
770 );
771
772 // Class for all SVE vector registers.
773 reg_class vectora_reg (
774 V0, V0_H, V0_J, V0_K,
775 V1, V1_H, V1_J, V1_K,
776 V2, V2_H, V2_J, V2_K,
777 V3, V3_H, V3_J, V3_K,
778 V4, V4_H, V4_J, V4_K,
779 V5, V5_H, V5_J, V5_K,
780 V6, V6_H, V6_J, V6_K,
781 V7, V7_H, V7_J, V7_K,
782 V8, V8_H, V8_J, V8_K,
783 V9, V9_H, V9_J, V9_K,
784 V10, V10_H, V10_J, V10_K,
785 V11, V11_H, V11_J, V11_K,
786 V12, V12_H, V12_J, V12_K,
787 V13, V13_H, V13_J, V13_K,
788 V14, V14_H, V14_J, V14_K,
789 V15, V15_H, V15_J, V15_K,
790 V16, V16_H, V16_J, V16_K,
791 V17, V17_H, V17_J, V17_K,
792 V18, V18_H, V18_J, V18_K,
793 V19, V19_H, V19_J, V19_K,
794 V20, V20_H, V20_J, V20_K,
795 V21, V21_H, V21_J, V21_K,
796 V22, V22_H, V22_J, V22_K,
797 V23, V23_H, V23_J, V23_K,
798 V24, V24_H, V24_J, V24_K,
799 V25, V25_H, V25_J, V25_K,
800 V26, V26_H, V26_J, V26_K,
801 V27, V27_H, V27_J, V27_K,
802 V28, V28_H, V28_J, V28_K,
803 V29, V29_H, V29_J, V29_K,
804 V30, V30_H, V30_J, V30_K,
805 V31, V31_H, V31_J, V31_K,
806 );
807
808 // Class for all 64bit vector registers
809 reg_class vectord_reg(
810 V0, V0_H,
811 V1, V1_H,
812 V2, V2_H,
813 V3, V3_H,
814 V4, V4_H,
815 V5, V5_H,
816 V6, V6_H,
817 V7, V7_H,
818 V8, V8_H,
819 V9, V9_H,
820 V10, V10_H,
821 V11, V11_H,
822 V12, V12_H,
823 V13, V13_H,
824 V14, V14_H,
825 V15, V15_H,
826 V16, V16_H,
827 V17, V17_H,
828 V18, V18_H,
829 V19, V19_H,
830 V20, V20_H,
831 V21, V21_H,
832 V22, V22_H,
833 V23, V23_H,
834 V24, V24_H,
835 V25, V25_H,
836 V26, V26_H,
837 V27, V27_H,
838 V28, V28_H,
839 V29, V29_H,
840 V30, V30_H,
841 V31, V31_H
842 );
843
844 // Class for all 128bit vector registers
845 reg_class vectorx_reg(
846 V0, V0_H, V0_J, V0_K,
847 V1, V1_H, V1_J, V1_K,
848 V2, V2_H, V2_J, V2_K,
849 V3, V3_H, V3_J, V3_K,
850 V4, V4_H, V4_J, V4_K,
851 V5, V5_H, V5_J, V5_K,
852 V6, V6_H, V6_J, V6_K,
853 V7, V7_H, V7_J, V7_K,
854 V8, V8_H, V8_J, V8_K,
855 V9, V9_H, V9_J, V9_K,
856 V10, V10_H, V10_J, V10_K,
857 V11, V11_H, V11_J, V11_K,
858 V12, V12_H, V12_J, V12_K,
859 V13, V13_H, V13_J, V13_K,
860 V14, V14_H, V14_J, V14_K,
861 V15, V15_H, V15_J, V15_K,
862 V16, V16_H, V16_J, V16_K,
863 V17, V17_H, V17_J, V17_K,
864 V18, V18_H, V18_J, V18_K,
865 V19, V19_H, V19_J, V19_K,
866 V20, V20_H, V20_J, V20_K,
867 V21, V21_H, V21_J, V21_K,
868 V22, V22_H, V22_J, V22_K,
869 V23, V23_H, V23_J, V23_K,
870 V24, V24_H, V24_J, V24_K,
871 V25, V25_H, V25_J, V25_K,
872 V26, V26_H, V26_J, V26_K,
873 V27, V27_H, V27_J, V27_K,
874 V28, V28_H, V28_J, V28_K,
875 V29, V29_H, V29_J, V29_K,
876 V30, V30_H, V30_J, V30_K,
877 V31, V31_H, V31_J, V31_K
878 );
879
880 // Class for 128 bit register v0
881 reg_class v0_reg(
882 V0, V0_H
883 );
884
885 // Class for 128 bit register v1
886 reg_class v1_reg(
887 V1, V1_H
888 );
889
890 // Class for 128 bit register v2
891 reg_class v2_reg(
892 V2, V2_H
893 );
894
895 // Class for 128 bit register v3
896 reg_class v3_reg(
897 V3, V3_H
898 );
899
900 // Class for 128 bit register v4
901 reg_class v4_reg(
902 V4, V4_H
903 );
904
905 // Class for 128 bit register v5
906 reg_class v5_reg(
907 V5, V5_H
908 );
909
910 // Class for 128 bit register v6
911 reg_class v6_reg(
912 V6, V6_H
913 );
914
915 // Class for 128 bit register v7
916 reg_class v7_reg(
917 V7, V7_H
918 );
919
920 // Class for 128 bit register v8
921 reg_class v8_reg(
922 V8, V8_H
923 );
924
925 // Class for 128 bit register v9
926 reg_class v9_reg(
927 V9, V9_H
928 );
929
930 // Class for 128 bit register v10
931 reg_class v10_reg(
932 V10, V10_H
933 );
934
935 // Class for 128 bit register v11
936 reg_class v11_reg(
937 V11, V11_H
938 );
939
940 // Class for 128 bit register v12
941 reg_class v12_reg(
942 V12, V12_H
943 );
944
945 // Class for 128 bit register v13
946 reg_class v13_reg(
947 V13, V13_H
948 );
949
950 // Class for 128 bit register v14
951 reg_class v14_reg(
952 V14, V14_H
953 );
954
955 // Class for 128 bit register v15
956 reg_class v15_reg(
957 V15, V15_H
958 );
959
960 // Class for 128 bit register v16
961 reg_class v16_reg(
962 V16, V16_H
963 );
964
965 // Class for 128 bit register v17
966 reg_class v17_reg(
967 V17, V17_H
968 );
969
970 // Class for 128 bit register v18
971 reg_class v18_reg(
972 V18, V18_H
973 );
974
975 // Class for 128 bit register v19
976 reg_class v19_reg(
977 V19, V19_H
978 );
979
980 // Class for 128 bit register v20
981 reg_class v20_reg(
982 V20, V20_H
983 );
984
985 // Class for 128 bit register v21
986 reg_class v21_reg(
987 V21, V21_H
988 );
989
990 // Class for 128 bit register v22
991 reg_class v22_reg(
992 V22, V22_H
993 );
994
995 // Class for 128 bit register v23
996 reg_class v23_reg(
997 V23, V23_H
998 );
999
1000 // Class for 128 bit register v24
1001 reg_class v24_reg(
1002 V24, V24_H
1003 );
1004
1005 // Class for 128 bit register v25
1006 reg_class v25_reg(
1007 V25, V25_H
1008 );
1009
1010 // Class for 128 bit register v26
1011 reg_class v26_reg(
1012 V26, V26_H
1013 );
1014
1015 // Class for 128 bit register v27
1016 reg_class v27_reg(
1017 V27, V27_H
1018 );
1019
1020 // Class for 128 bit register v28
1021 reg_class v28_reg(
1022 V28, V28_H
1023 );
1024
1025 // Class for 128 bit register v29
1026 reg_class v29_reg(
1027 V29, V29_H
1028 );
1029
1030 // Class for 128 bit register v30
1031 reg_class v30_reg(
1032 V30, V30_H
1033 );
1034
1035 // Class for 128 bit register v31
1036 reg_class v31_reg(
1037 V31, V31_H
1038 );
1039
1040 // Class for all SVE predicate registers.
1041 reg_class pr_reg (
1042 P0,
1043 P1,
1044 P2,
1045 P3,
1046 P4,
1047 P5,
1048 P6,
1049 // P7, non-allocatable, preserved with all elements preset to TRUE.
1050 P8,
1051 P9,
1052 P10,
1053 P11,
1054 P12,
1055 P13,
1056 P14,
1057 P15
1058 );
1059
1060 // Class for SVE governing predicate registers, which are used
1061 // to determine the active elements of a predicated instruction.
1062 reg_class gov_pr (
1063 P0,
1064 P1,
1065 P2,
1066 P3,
1067 P4,
1068 P5,
1069 P6,
1070 // P7, non-allocatable, preserved with all elements preset to TRUE.
1071 );
1072
1073 reg_class p0_reg(P0);
1074 reg_class p1_reg(P1);
1075
1076 // Singleton class for condition codes
1077 reg_class int_flags(RFLAGS);
1078
1079 %}
1080
1081 //----------DEFINITION BLOCK---------------------------------------------------
1082 // Define name --> value mappings to inform the ADLC of an integer valued name
1083 // Current support includes integer values in the range [0, 0x7FFFFFFF]
1084 // Format:
1085 // int_def <name> ( <int_value>, <expression>);
1086 // Generated Code in ad_<arch>.hpp
1087 // #define <name> (<expression>)
1088 // // value == <int_value>
1089 // Generated code in ad_<arch>.cpp adlc_verification()
1090 // assert( <name> == <int_value>, "Expect (<expression>) to equal <int_value>");
1091 //
1092
1093 // we follow the ppc-aix port in using a simple cost model which ranks
1094 // register operations as cheap, memory ops as more expensive and
1095 // branches as most expensive. the first two have a low as well as a
1096 // normal cost. huge cost appears to be a way of saying don't do
1097 // something
1098
1099 definitions %{
1100 // The default cost (of a register move instruction).
1101 int_def INSN_COST ( 100, 100);
1102 int_def BRANCH_COST ( 200, 2 * INSN_COST);
1103 int_def CALL_COST ( 200, 2 * INSN_COST);
1104 int_def VOLATILE_REF_COST ( 1000, 10 * INSN_COST);
1105 %}
1106
1107
1108 //----------SOURCE BLOCK-------------------------------------------------------
1109 // This is a block of C++ code which provides values, functions, and
1110 // definitions necessary in the rest of the architecture description
1111
1112 source_hpp %{
1113
1114 #include "asm/macroAssembler.hpp"
1115 #include "gc/shared/barrierSetAssembler.hpp"
1116 #include "gc/shared/cardTable.hpp"
1117 #include "gc/shared/cardTableBarrierSet.hpp"
1118 #include "gc/shared/collectedHeap.hpp"
1119 #include "opto/addnode.hpp"
1120 #include "opto/convertnode.hpp"
1121 #include "runtime/objectMonitor.hpp"
1122
1123 extern RegMask _ANY_REG32_mask;
1124 extern RegMask _ANY_REG_mask;
1125 extern RegMask _PTR_REG_mask;
1126 extern RegMask _NO_SPECIAL_REG32_mask;
1127 extern RegMask _NO_SPECIAL_REG_mask;
1128 extern RegMask _NO_SPECIAL_PTR_REG_mask;
1129
1130 class CallStubImpl {
1131
1132 //--------------------------------------------------------------
1133 //---< Used for optimization in Compile::shorten_branches >---
1134 //--------------------------------------------------------------
1135
1136 public:
1137 // Size of call trampoline stub.
1138 static uint size_call_trampoline() {
1139 return 0; // no call trampolines on this platform
1140 }
1141
1142 // number of relocations needed by a call trampoline stub
1143 static uint reloc_call_trampoline() {
1144 return 0; // no call trampolines on this platform
1145 }
1146 };
1147
1148 class HandlerImpl {
1149
1150 public:
1151
1152 static int emit_exception_handler(CodeBuffer &cbuf);
1153 static int emit_deopt_handler(CodeBuffer& cbuf);
1154
1155 static uint size_exception_handler() {
1156 return MacroAssembler::far_codestub_branch_size();
1157 }
1158
1159 static uint size_deopt_handler() {
1160 // count one adr and one far branch instruction
1161 return NativeInstruction::instruction_size + MacroAssembler::far_codestub_branch_size();
1162 }
1163 };
1164
1165 class Node::PD {
1166 public:
1167 enum NodeFlags {
1168 _last_flag = Node::_last_flag
1169 };
1170 };
1171
1172 bool is_CAS(int opcode, bool maybe_volatile);
1173
1174 // predicates controlling emit of ldr<x>/ldar<x> and associated dmb
1175
1176 bool unnecessary_acquire(const Node *barrier);
1177 bool needs_acquiring_load(const Node *load);
1178
1179 // predicates controlling emit of str<x>/stlr<x> and associated dmbs
1180
1181 bool unnecessary_release(const Node *barrier);
1182 bool unnecessary_volatile(const Node *barrier);
1183 bool needs_releasing_store(const Node *store);
1184
1185 // predicate controlling translation of CompareAndSwapX
1186 bool needs_acquiring_load_exclusive(const Node *load);
1187
1188 // predicate controlling addressing modes
1189 bool size_fits_all_mem_uses(AddPNode* addp, int shift);
1190 %}
1191
1192 source %{
1193
1194 // Derived RegMask with conditionally allocatable registers
1195
1196 void PhaseOutput::pd_perform_mach_node_analysis() {
1197 }
1198
1199 int MachNode::pd_alignment_required() const {
1200 return 1;
1201 }
1202
1203 int MachNode::compute_padding(int current_offset) const {
1204 return 0;
1205 }
1206
1207 RegMask _ANY_REG32_mask;
1208 RegMask _ANY_REG_mask;
1209 RegMask _PTR_REG_mask;
1210 RegMask _NO_SPECIAL_REG32_mask;
1211 RegMask _NO_SPECIAL_REG_mask;
1212 RegMask _NO_SPECIAL_PTR_REG_mask;
1213
1214 void reg_mask_init() {
1215 // We derive below RegMask(s) from the ones which are auto-generated from
1216 // adlc register classes to make AArch64 rheapbase (r27) and rfp (r29)
1217 // registers conditionally reserved.
1218
1219 _ANY_REG32_mask = _ALL_REG32_mask;
1220 _ANY_REG32_mask.Remove(OptoReg::as_OptoReg(r31_sp->as_VMReg()));
1221
1222 _ANY_REG_mask = _ALL_REG_mask;
1223
1224 _PTR_REG_mask = _ALL_REG_mask;
1225
1226 _NO_SPECIAL_REG32_mask = _ALL_REG32_mask;
1227 _NO_SPECIAL_REG32_mask.SUBTRACT(_NON_ALLOCATABLE_REG32_mask);
1228
1229 _NO_SPECIAL_REG_mask = _ALL_REG_mask;
1230 _NO_SPECIAL_REG_mask.SUBTRACT(_NON_ALLOCATABLE_REG_mask);
1231
1232 _NO_SPECIAL_PTR_REG_mask = _ALL_REG_mask;
1233 _NO_SPECIAL_PTR_REG_mask.SUBTRACT(_NON_ALLOCATABLE_REG_mask);
1234
1235 // r27 is not allocatable when compressed oops is on and heapbase is not
1236 // zero, compressed klass pointers doesn't use r27 after JDK-8234794
1237 if (UseCompressedOops && (CompressedOops::ptrs_base() != NULL)) {
1238 _NO_SPECIAL_REG32_mask.Remove(OptoReg::as_OptoReg(r27->as_VMReg()));
1239 _NO_SPECIAL_REG_mask.SUBTRACT(_HEAPBASE_REG_mask);
1240 _NO_SPECIAL_PTR_REG_mask.SUBTRACT(_HEAPBASE_REG_mask);
1241 }
1242
1243 // r29 is not allocatable when PreserveFramePointer is on
1244 if (PreserveFramePointer) {
1245 _NO_SPECIAL_REG32_mask.Remove(OptoReg::as_OptoReg(r29->as_VMReg()));
1246 _NO_SPECIAL_REG_mask.SUBTRACT(_FP_REG_mask);
1247 _NO_SPECIAL_PTR_REG_mask.SUBTRACT(_FP_REG_mask);
1248 }
1249 }
1250
1251 // Optimizaton of volatile gets and puts
1252 // -------------------------------------
1253 //
1254 // AArch64 has ldar<x> and stlr<x> instructions which we can safely
1255 // use to implement volatile reads and writes. For a volatile read
1256 // we simply need
1257 //
1258 // ldar<x>
1259 //
1260 // and for a volatile write we need
1261 //
1262 // stlr<x>
1263 //
1264 // Alternatively, we can implement them by pairing a normal
1265 // load/store with a memory barrier. For a volatile read we need
1266 //
1267 // ldr<x>
1268 // dmb ishld
1269 //
1270 // for a volatile write
1271 //
1272 // dmb ish
1273 // str<x>
1274 // dmb ish
1275 //
1276 // We can also use ldaxr and stlxr to implement compare and swap CAS
1277 // sequences. These are normally translated to an instruction
1278 // sequence like the following
1279 //
1280 // dmb ish
1281 // retry:
1282 // ldxr<x> rval raddr
1283 // cmp rval rold
1284 // b.ne done
1285 // stlxr<x> rval, rnew, rold
1286 // cbnz rval retry
1287 // done:
1288 // cset r0, eq
1289 // dmb ishld
1290 //
1291 // Note that the exclusive store is already using an stlxr
1292 // instruction. That is required to ensure visibility to other
1293 // threads of the exclusive write (assuming it succeeds) before that
1294 // of any subsequent writes.
1295 //
1296 // The following instruction sequence is an improvement on the above
1297 //
1298 // retry:
1299 // ldaxr<x> rval raddr
1300 // cmp rval rold
1301 // b.ne done
1302 // stlxr<x> rval, rnew, rold
1303 // cbnz rval retry
1304 // done:
1305 // cset r0, eq
1306 //
1307 // We don't need the leading dmb ish since the stlxr guarantees
1308 // visibility of prior writes in the case that the swap is
1309 // successful. Crucially we don't have to worry about the case where
1310 // the swap is not successful since no valid program should be
1311 // relying on visibility of prior changes by the attempting thread
1312 // in the case where the CAS fails.
1313 //
1314 // Similarly, we don't need the trailing dmb ishld if we substitute
1315 // an ldaxr instruction since that will provide all the guarantees we
1316 // require regarding observation of changes made by other threads
1317 // before any change to the CAS address observed by the load.
1318 //
1319 // In order to generate the desired instruction sequence we need to
1320 // be able to identify specific 'signature' ideal graph node
1321 // sequences which i) occur as a translation of a volatile reads or
1322 // writes or CAS operations and ii) do not occur through any other
1323 // translation or graph transformation. We can then provide
1324 // alternative aldc matching rules which translate these node
1325 // sequences to the desired machine code sequences. Selection of the
1326 // alternative rules can be implemented by predicates which identify
1327 // the relevant node sequences.
1328 //
1329 // The ideal graph generator translates a volatile read to the node
1330 // sequence
1331 //
1332 // LoadX[mo_acquire]
1333 // MemBarAcquire
1334 //
1335 // As a special case when using the compressed oops optimization we
1336 // may also see this variant
1337 //
1338 // LoadN[mo_acquire]
1339 // DecodeN
1340 // MemBarAcquire
1341 //
1342 // A volatile write is translated to the node sequence
1343 //
1344 // MemBarRelease
1345 // StoreX[mo_release] {CardMark}-optional
1346 // MemBarVolatile
1347 //
1348 // n.b. the above node patterns are generated with a strict
1349 // 'signature' configuration of input and output dependencies (see
1350 // the predicates below for exact details). The card mark may be as
1351 // simple as a few extra nodes or, in a few GC configurations, may
1352 // include more complex control flow between the leading and
1353 // trailing memory barriers. However, whatever the card mark
1354 // configuration these signatures are unique to translated volatile
1355 // reads/stores -- they will not appear as a result of any other
1356 // bytecode translation or inlining nor as a consequence of
1357 // optimizing transforms.
1358 //
1359 // We also want to catch inlined unsafe volatile gets and puts and
1360 // be able to implement them using either ldar<x>/stlr<x> or some
1361 // combination of ldr<x>/stlr<x> and dmb instructions.
1362 //
1363 // Inlined unsafe volatiles puts manifest as a minor variant of the
1364 // normal volatile put node sequence containing an extra cpuorder
1365 // membar
1366 //
1367 // MemBarRelease
1368 // MemBarCPUOrder
1369 // StoreX[mo_release] {CardMark}-optional
1370 // MemBarCPUOrder
1371 // MemBarVolatile
1372 //
1373 // n.b. as an aside, a cpuorder membar is not itself subject to
1374 // matching and translation by adlc rules. However, the rule
1375 // predicates need to detect its presence in order to correctly
1376 // select the desired adlc rules.
1377 //
1378 // Inlined unsafe volatile gets manifest as a slightly different
1379 // node sequence to a normal volatile get because of the
1380 // introduction of some CPUOrder memory barriers to bracket the
1381 // Load. However, but the same basic skeleton of a LoadX feeding a
1382 // MemBarAcquire, possibly through an optional DecodeN, is still
1383 // present
1384 //
1385 // MemBarCPUOrder
1386 // || \\
1387 // MemBarCPUOrder LoadX[mo_acquire]
1388 // || |
1389 // || {DecodeN} optional
1390 // || /
1391 // MemBarAcquire
1392 //
1393 // In this case the acquire membar does not directly depend on the
1394 // load. However, we can be sure that the load is generated from an
1395 // inlined unsafe volatile get if we see it dependent on this unique
1396 // sequence of membar nodes. Similarly, given an acquire membar we
1397 // can know that it was added because of an inlined unsafe volatile
1398 // get if it is fed and feeds a cpuorder membar and if its feed
1399 // membar also feeds an acquiring load.
1400 //
1401 // Finally an inlined (Unsafe) CAS operation is translated to the
1402 // following ideal graph
1403 //
1404 // MemBarRelease
1405 // MemBarCPUOrder
1406 // CompareAndSwapX {CardMark}-optional
1407 // MemBarCPUOrder
1408 // MemBarAcquire
1409 //
1410 // So, where we can identify these volatile read and write
1411 // signatures we can choose to plant either of the above two code
1412 // sequences. For a volatile read we can simply plant a normal
1413 // ldr<x> and translate the MemBarAcquire to a dmb. However, we can
1414 // also choose to inhibit translation of the MemBarAcquire and
1415 // inhibit planting of the ldr<x>, instead planting an ldar<x>.
1416 //
1417 // When we recognise a volatile store signature we can choose to
1418 // plant at a dmb ish as a translation for the MemBarRelease, a
1419 // normal str<x> and then a dmb ish for the MemBarVolatile.
1420 // Alternatively, we can inhibit translation of the MemBarRelease
1421 // and MemBarVolatile and instead plant a simple stlr<x>
1422 // instruction.
1423 //
1424 // when we recognise a CAS signature we can choose to plant a dmb
1425 // ish as a translation for the MemBarRelease, the conventional
1426 // macro-instruction sequence for the CompareAndSwap node (which
1427 // uses ldxr<x>) and then a dmb ishld for the MemBarAcquire.
1428 // Alternatively, we can elide generation of the dmb instructions
1429 // and plant the alternative CompareAndSwap macro-instruction
1430 // sequence (which uses ldaxr<x>).
1431 //
1432 // Of course, the above only applies when we see these signature
1433 // configurations. We still want to plant dmb instructions in any
1434 // other cases where we may see a MemBarAcquire, MemBarRelease or
1435 // MemBarVolatile. For example, at the end of a constructor which
1436 // writes final/volatile fields we will see a MemBarRelease
1437 // instruction and this needs a 'dmb ish' lest we risk the
1438 // constructed object being visible without making the
1439 // final/volatile field writes visible.
1440 //
1441 // n.b. the translation rules below which rely on detection of the
1442 // volatile signatures and insert ldar<x> or stlr<x> are failsafe.
1443 // If we see anything other than the signature configurations we
1444 // always just translate the loads and stores to ldr<x> and str<x>
1445 // and translate acquire, release and volatile membars to the
1446 // relevant dmb instructions.
1447 //
1448
1449 // is_CAS(int opcode, bool maybe_volatile)
1450 //
1451 // return true if opcode is one of the possible CompareAndSwapX
1452 // values otherwise false.
1453
1454 bool is_CAS(int opcode, bool maybe_volatile)
1455 {
1456 switch(opcode) {
1457 // We handle these
1458 case Op_CompareAndSwapI:
1459 case Op_CompareAndSwapL:
1460 case Op_CompareAndSwapP:
1461 case Op_CompareAndSwapN:
1462 case Op_ShenandoahCompareAndSwapP:
1463 case Op_ShenandoahCompareAndSwapN:
1464 case Op_CompareAndSwapB:
1465 case Op_CompareAndSwapS:
1466 case Op_GetAndSetI:
1467 case Op_GetAndSetL:
1468 case Op_GetAndSetP:
1469 case Op_GetAndSetN:
1470 case Op_GetAndAddI:
1471 case Op_GetAndAddL:
1472 return true;
1473 case Op_CompareAndExchangeI:
1474 case Op_CompareAndExchangeN:
1475 case Op_CompareAndExchangeB:
1476 case Op_CompareAndExchangeS:
1477 case Op_CompareAndExchangeL:
1478 case Op_CompareAndExchangeP:
1479 case Op_WeakCompareAndSwapB:
1480 case Op_WeakCompareAndSwapS:
1481 case Op_WeakCompareAndSwapI:
1482 case Op_WeakCompareAndSwapL:
1483 case Op_WeakCompareAndSwapP:
1484 case Op_WeakCompareAndSwapN:
1485 case Op_ShenandoahWeakCompareAndSwapP:
1486 case Op_ShenandoahWeakCompareAndSwapN:
1487 case Op_ShenandoahCompareAndExchangeP:
1488 case Op_ShenandoahCompareAndExchangeN:
1489 return maybe_volatile;
1490 default:
1491 return false;
1492 }
1493 }
1494
1495 // helper to determine the maximum number of Phi nodes we may need to
1496 // traverse when searching from a card mark membar for the merge mem
1497 // feeding a trailing membar or vice versa
1498
1499 // predicates controlling emit of ldr<x>/ldar<x>
1500
1501 bool unnecessary_acquire(const Node *barrier)
1502 {
1503 assert(barrier->is_MemBar(), "expecting a membar");
1504
1505 MemBarNode* mb = barrier->as_MemBar();
1506
1507 if (mb->trailing_load()) {
1508 return true;
1509 }
1510
1511 if (mb->trailing_load_store()) {
1512 Node* load_store = mb->in(MemBarNode::Precedent);
1513 assert(load_store->is_LoadStore(), "unexpected graph shape");
1514 return is_CAS(load_store->Opcode(), true);
1515 }
1516
1517 return false;
1518 }
1519
1520 bool needs_acquiring_load(const Node *n)
1521 {
1522 assert(n->is_Load(), "expecting a load");
1523 LoadNode *ld = n->as_Load();
1524 return ld->is_acquire();
1525 }
1526
1527 bool unnecessary_release(const Node *n)
1528 {
1529 assert((n->is_MemBar() &&
1530 n->Opcode() == Op_MemBarRelease),
1531 "expecting a release membar");
1532
1533 MemBarNode *barrier = n->as_MemBar();
1534 if (!barrier->leading()) {
1535 return false;
1536 } else {
1537 Node* trailing = barrier->trailing_membar();
1538 MemBarNode* trailing_mb = trailing->as_MemBar();
1539 assert(trailing_mb->trailing(), "Not a trailing membar?");
1540 assert(trailing_mb->leading_membar() == n, "inconsistent leading/trailing membars");
1541
1542 Node* mem = trailing_mb->in(MemBarNode::Precedent);
1543 if (mem->is_Store()) {
1544 assert(mem->as_Store()->is_release(), "");
1545 assert(trailing_mb->Opcode() == Op_MemBarVolatile, "");
1546 return true;
1547 } else {
1548 assert(mem->is_LoadStore(), "");
1549 assert(trailing_mb->Opcode() == Op_MemBarAcquire, "");
1550 return is_CAS(mem->Opcode(), true);
1551 }
1552 }
1553 return false;
1554 }
1555
1556 bool unnecessary_volatile(const Node *n)
1557 {
1558 // assert n->is_MemBar();
1559 MemBarNode *mbvol = n->as_MemBar();
1560
1561 bool release = mbvol->trailing_store();
1562 assert(!release || (mbvol->in(MemBarNode::Precedent)->is_Store() && mbvol->in(MemBarNode::Precedent)->as_Store()->is_release()), "");
1563 #ifdef ASSERT
1564 if (release) {
1565 Node* leading = mbvol->leading_membar();
1566 assert(leading->Opcode() == Op_MemBarRelease, "");
1567 assert(leading->as_MemBar()->leading_store(), "");
1568 assert(leading->as_MemBar()->trailing_membar() == mbvol, "");
1569 }
1570 #endif
1571
1572 return release;
1573 }
1574
1575 // predicates controlling emit of str<x>/stlr<x>
1576
1577 bool needs_releasing_store(const Node *n)
1578 {
1579 // assert n->is_Store();
1580 StoreNode *st = n->as_Store();
1581 return st->trailing_membar() != NULL;
1582 }
1583
1584 // predicate controlling translation of CAS
1585 //
1586 // returns true if CAS needs to use an acquiring load otherwise false
1587
1588 bool needs_acquiring_load_exclusive(const Node *n)
1589 {
1590 assert(is_CAS(n->Opcode(), true), "expecting a compare and swap");
1591 LoadStoreNode* ldst = n->as_LoadStore();
1592 if (is_CAS(n->Opcode(), false)) {
1593 assert(ldst->trailing_membar() != NULL, "expected trailing membar");
1594 } else {
1595 return ldst->trailing_membar() != NULL;
1596 }
1597
1598 // so we can just return true here
1599 return true;
1600 }
1601
1602 #define __ _masm.
1603
1604 // advance declarations for helper functions to convert register
1605 // indices to register objects
1606
1607 // the ad file has to provide implementations of certain methods
1608 // expected by the generic code
1609 //
1610 // REQUIRED FUNCTIONALITY
1611
1612 //=============================================================================
1613
1614 // !!!!! Special hack to get all types of calls to specify the byte offset
1615 // from the start of the call to the point where the return address
1616 // will point.
1617
1618 int MachCallStaticJavaNode::ret_addr_offset()
1619 {
1620 // call should be a simple bl
1621 int off = 4;
1622 return off;
1623 }
1624
1625 int MachCallDynamicJavaNode::ret_addr_offset()
1626 {
1627 return 16; // movz, movk, movk, bl
1628 }
1629
1630 int MachCallRuntimeNode::ret_addr_offset() {
1631 // for generated stubs the call will be
1632 // bl(addr)
1633 // or with far branches
1634 // bl(trampoline_stub)
1635 // for real runtime callouts it will be six instructions
1636 // see aarch64_enc_java_to_runtime
1637 // adr(rscratch2, retaddr)
1638 // lea(rscratch1, RuntimeAddress(addr)
1639 // stp(zr, rscratch2, Address(__ pre(sp, -2 * wordSize)))
1640 // blr(rscratch1)
1641 CodeBlob *cb = CodeCache::find_blob(_entry_point);
1642 if (cb) {
1643 return 1 * NativeInstruction::instruction_size;
1644 } else {
1645 return 6 * NativeInstruction::instruction_size;
1646 }
1647 }
1648
1649 //=============================================================================
1650
1651 #ifndef PRODUCT
1652 void MachBreakpointNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
1653 st->print("BREAKPOINT");
1654 }
1655 #endif
1656
1657 void MachBreakpointNode::emit(CodeBuffer &cbuf, PhaseRegAlloc *ra_) const {
1658 C2_MacroAssembler _masm(&cbuf);
1659 __ brk(0);
1660 }
1661
1662 uint MachBreakpointNode::size(PhaseRegAlloc *ra_) const {
1663 return MachNode::size(ra_);
1664 }
1665
1666 //=============================================================================
1667
1668 #ifndef PRODUCT
1669 void MachNopNode::format(PhaseRegAlloc*, outputStream* st) const {
1670 st->print("nop \t# %d bytes pad for loops and calls", _count);
1671 }
1672 #endif
1673
1674 void MachNopNode::emit(CodeBuffer &cbuf, PhaseRegAlloc*) const {
1675 C2_MacroAssembler _masm(&cbuf);
1676 for (int i = 0; i < _count; i++) {
1677 __ nop();
1678 }
1679 }
1680
1681 uint MachNopNode::size(PhaseRegAlloc*) const {
1682 return _count * NativeInstruction::instruction_size;
1683 }
1684
1685 //=============================================================================
1686 const RegMask& MachConstantBaseNode::_out_RegMask = RegMask::Empty;
1687
1688 int ConstantTable::calculate_table_base_offset() const {
1689 return 0; // absolute addressing, no offset
1690 }
1691
1692 bool MachConstantBaseNode::requires_postalloc_expand() const { return false; }
1693 void MachConstantBaseNode::postalloc_expand(GrowableArray <Node *> *nodes, PhaseRegAlloc *ra_) {
1694 ShouldNotReachHere();
1695 }
1696
1697 void MachConstantBaseNode::emit(CodeBuffer& cbuf, PhaseRegAlloc* ra_) const {
1698 // Empty encoding
1699 }
1700
1701 uint MachConstantBaseNode::size(PhaseRegAlloc* ra_) const {
1702 return 0;
1703 }
1704
1705 #ifndef PRODUCT
1706 void MachConstantBaseNode::format(PhaseRegAlloc* ra_, outputStream* st) const {
1707 st->print("-- \t// MachConstantBaseNode (empty encoding)");
1708 }
1709 #endif
1710
1711 #ifndef PRODUCT
1712 void MachPrologNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
1713 Compile* C = ra_->C;
1714
1715 int framesize = C->output()->frame_slots() << LogBytesPerInt;
1716
1717 if (C->output()->need_stack_bang(framesize))
1718 st->print("# stack bang size=%d\n\t", framesize);
1719
1720 if (VM_Version::use_rop_protection()) {
1721 st->print("ldr zr, [lr]\n\t");
1722 st->print("pacia lr, rfp\n\t");
1723 }
1724 if (framesize < ((1 << 9) + 2 * wordSize)) {
1725 st->print("sub sp, sp, #%d\n\t", framesize);
1726 st->print("stp rfp, lr, [sp, #%d]", framesize - 2 * wordSize);
1727 if (PreserveFramePointer) st->print("\n\tadd rfp, sp, #%d", framesize - 2 * wordSize);
1728 } else {
1729 st->print("stp lr, rfp, [sp, #%d]!\n\t", -(2 * wordSize));
1730 if (PreserveFramePointer) st->print("mov rfp, sp\n\t");
1731 st->print("mov rscratch1, #%d\n\t", framesize - 2 * wordSize);
1732 st->print("sub sp, sp, rscratch1");
1733 }
1734 if (C->stub_function() == NULL && BarrierSet::barrier_set()->barrier_set_nmethod() != NULL) {
1735 st->print("\n\t");
1736 st->print("ldr rscratch1, [guard]\n\t");
1737 st->print("dmb ishld\n\t");
1738 st->print("ldr rscratch2, [rthread, #thread_disarmed_offset]\n\t");
1739 st->print("cmp rscratch1, rscratch2\n\t");
1740 st->print("b.eq skip");
1741 st->print("\n\t");
1742 st->print("blr #nmethod_entry_barrier_stub\n\t");
1743 st->print("b skip\n\t");
1744 st->print("guard: int\n\t");
1745 st->print("\n\t");
1746 st->print("skip:\n\t");
1747 }
1748 }
1749 #endif
1750
1751 void MachPrologNode::emit(CodeBuffer &cbuf, PhaseRegAlloc *ra_) const {
1752 Compile* C = ra_->C;
1753 C2_MacroAssembler _masm(&cbuf);
1754
1755 // n.b. frame size includes space for return pc and rfp
1756 const int framesize = C->output()->frame_size_in_bytes();
1757
1758 // insert a nop at the start of the prolog so we can patch in a
1759 // branch if we need to invalidate the method later
1760 __ nop();
1761
1762 if (C->clinit_barrier_on_entry()) {
1763 assert(!C->method()->holder()->is_not_initialized(), "initialization should have been started");
1764
1765 Label L_skip_barrier;
1766
1767 __ mov_metadata(rscratch2, C->method()->holder()->constant_encoding());
1768 __ clinit_barrier(rscratch2, rscratch1, &L_skip_barrier);
1769 __ far_jump(RuntimeAddress(SharedRuntime::get_handle_wrong_method_stub()));
1770 __ bind(L_skip_barrier);
1771 }
1772
1773 if (C->max_vector_size() > 0) {
1774 __ reinitialize_ptrue();
1775 }
1776
1777 int bangsize = C->output()->bang_size_in_bytes();
1778 if (C->output()->need_stack_bang(bangsize))
1779 __ generate_stack_overflow_check(bangsize);
1780
1781 __ build_frame(framesize);
1782
1783 if (C->stub_function() == NULL) {
1784 BarrierSetAssembler* bs = BarrierSet::barrier_set()->barrier_set_assembler();
1785 if (BarrierSet::barrier_set()->barrier_set_nmethod() != NULL) {
1786 // Dummy labels for just measuring the code size
1787 Label dummy_slow_path;
1788 Label dummy_continuation;
1789 Label dummy_guard;
1790 Label* slow_path = &dummy_slow_path;
1791 Label* continuation = &dummy_continuation;
1792 Label* guard = &dummy_guard;
1793 if (!Compile::current()->output()->in_scratch_emit_size()) {
1794 // Use real labels from actual stub when not emitting code for the purpose of measuring its size
1795 C2EntryBarrierStub* stub = Compile::current()->output()->entry_barrier_table()->add_entry_barrier();
1796 slow_path = &stub->slow_path();
1797 continuation = &stub->continuation();
1798 guard = &stub->guard();
1799 }
1800 // In the C2 code, we move the non-hot part of nmethod entry barriers out-of-line to a stub.
1801 bs->nmethod_entry_barrier(&_masm, slow_path, continuation, guard);
1802 }
1803 }
1804
1805 if (VerifyStackAtCalls) {
1806 Unimplemented();
1807 }
1808
1809 C->output()->set_frame_complete(cbuf.insts_size());
1810
1811 if (C->has_mach_constant_base_node()) {
1812 // NOTE: We set the table base offset here because users might be
1813 // emitted before MachConstantBaseNode.
1814 ConstantTable& constant_table = C->output()->constant_table();
1815 constant_table.set_table_base_offset(constant_table.calculate_table_base_offset());
1816 }
1817 }
1818
1819 uint MachPrologNode::size(PhaseRegAlloc* ra_) const
1820 {
1821 return MachNode::size(ra_); // too many variables; just compute it
1822 // the hard way
1823 }
1824
1825 int MachPrologNode::reloc() const
1826 {
1827 return 0;
1828 }
1829
1830 //=============================================================================
1831
1832 #ifndef PRODUCT
1833 void MachEpilogNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
1834 Compile* C = ra_->C;
1835 int framesize = C->output()->frame_slots() << LogBytesPerInt;
1836
1837 st->print("# pop frame %d\n\t",framesize);
1838
1839 if (framesize == 0) {
1840 st->print("ldp lr, rfp, [sp],#%d\n\t", (2 * wordSize));
1841 } else if (framesize < ((1 << 9) + 2 * wordSize)) {
1842 st->print("ldp lr, rfp, [sp,#%d]\n\t", framesize - 2 * wordSize);
1843 st->print("add sp, sp, #%d\n\t", framesize);
1844 } else {
1845 st->print("mov rscratch1, #%d\n\t", framesize - 2 * wordSize);
1846 st->print("add sp, sp, rscratch1\n\t");
1847 st->print("ldp lr, rfp, [sp],#%d\n\t", (2 * wordSize));
1848 }
1849 if (VM_Version::use_rop_protection()) {
1850 st->print("autia lr, rfp\n\t");
1851 st->print("ldr zr, [lr]\n\t");
1852 }
1853
1854 if (do_polling() && C->is_method_compilation()) {
1855 st->print("# test polling word\n\t");
1856 st->print("ldr rscratch1, [rthread],#%d\n\t", in_bytes(JavaThread::polling_word_offset()));
1857 st->print("cmp sp, rscratch1\n\t");
1858 st->print("bhi #slow_path");
1859 }
1860 }
1861 #endif
1862
1863 void MachEpilogNode::emit(CodeBuffer &cbuf, PhaseRegAlloc *ra_) const {
1864 Compile* C = ra_->C;
1865 C2_MacroAssembler _masm(&cbuf);
1866 int framesize = C->output()->frame_slots() << LogBytesPerInt;
1867
1868 __ remove_frame(framesize);
1869
1870 if (StackReservedPages > 0 && C->has_reserved_stack_access()) {
1871 __ reserved_stack_check();
1872 }
1873
1874 if (do_polling() && C->is_method_compilation()) {
1875 Label dummy_label;
1876 Label* code_stub = &dummy_label;
1877 if (!C->output()->in_scratch_emit_size()) {
1878 code_stub = &C->output()->safepoint_poll_table()->add_safepoint(__ offset());
1879 }
1880 __ relocate(relocInfo::poll_return_type);
1881 __ safepoint_poll(*code_stub, true /* at_return */, false /* acquire */, true /* in_nmethod */);
1882 }
1883 }
1884
1885 uint MachEpilogNode::size(PhaseRegAlloc *ra_) const {
1886 // Variable size. Determine dynamically.
1887 return MachNode::size(ra_);
1888 }
1889
1890 int MachEpilogNode::reloc() const {
1891 // Return number of relocatable values contained in this instruction.
1892 return 1; // 1 for polling page.
1893 }
1894
1895 const Pipeline * MachEpilogNode::pipeline() const {
1896 return MachNode::pipeline_class();
1897 }
1898
1899 //=============================================================================
1900
1901 // Figure out which register class each belongs in: rc_int, rc_float or
1902 // rc_stack.
1903 enum RC { rc_bad, rc_int, rc_float, rc_predicate, rc_stack };
1904
1905 static enum RC rc_class(OptoReg::Name reg) {
1906
1907 if (reg == OptoReg::Bad) {
1908 return rc_bad;
1909 }
1910
1911 // we have 32 int registers * 2 halves
1912 int slots_of_int_registers = Register::number_of_registers * Register::max_slots_per_register;
1913
1914 if (reg < slots_of_int_registers) {
1915 return rc_int;
1916 }
1917
1918 // we have 32 float register * 8 halves
1919 int slots_of_float_registers = FloatRegister::number_of_registers * FloatRegister::max_slots_per_register;
1920 if (reg < slots_of_int_registers + slots_of_float_registers) {
1921 return rc_float;
1922 }
1923
1924 int slots_of_predicate_registers = PRegister::number_of_registers * PRegister::max_slots_per_register;
1925 if (reg < slots_of_int_registers + slots_of_float_registers + slots_of_predicate_registers) {
1926 return rc_predicate;
1927 }
1928
1929 // Between predicate regs & stack is the flags.
1930 assert(OptoReg::is_stack(reg), "blow up if spilling flags");
1931
1932 return rc_stack;
1933 }
1934
1935 uint MachSpillCopyNode::implementation(CodeBuffer *cbuf, PhaseRegAlloc *ra_, bool do_size, outputStream *st) const {
1936 Compile* C = ra_->C;
1937
1938 // Get registers to move.
1939 OptoReg::Name src_hi = ra_->get_reg_second(in(1));
1940 OptoReg::Name src_lo = ra_->get_reg_first(in(1));
1941 OptoReg::Name dst_hi = ra_->get_reg_second(this);
1942 OptoReg::Name dst_lo = ra_->get_reg_first(this);
1943
1944 enum RC src_hi_rc = rc_class(src_hi);
1945 enum RC src_lo_rc = rc_class(src_lo);
1946 enum RC dst_hi_rc = rc_class(dst_hi);
1947 enum RC dst_lo_rc = rc_class(dst_lo);
1948
1949 assert(src_lo != OptoReg::Bad && dst_lo != OptoReg::Bad, "must move at least 1 register");
1950
1951 if (src_hi != OptoReg::Bad && !bottom_type()->isa_vectmask()) {
1952 assert((src_lo&1)==0 && src_lo+1==src_hi &&
1953 (dst_lo&1)==0 && dst_lo+1==dst_hi,
1954 "expected aligned-adjacent pairs");
1955 }
1956
1957 if (src_lo == dst_lo && src_hi == dst_hi) {
1958 return 0; // Self copy, no move.
1959 }
1960
1961 bool is64 = (src_lo & 1) == 0 && src_lo + 1 == src_hi &&
1962 (dst_lo & 1) == 0 && dst_lo + 1 == dst_hi;
1963 int src_offset = ra_->reg2offset(src_lo);
1964 int dst_offset = ra_->reg2offset(dst_lo);
1965
1966 if (bottom_type()->isa_vect() && !bottom_type()->isa_vectmask()) {
1967 uint ireg = ideal_reg();
1968 if (ireg == Op_VecA && cbuf) {
1969 C2_MacroAssembler _masm(cbuf);
1970 int sve_vector_reg_size_in_bytes = Matcher::scalable_vector_reg_size(T_BYTE);
1971 if (src_lo_rc == rc_stack && dst_lo_rc == rc_stack) {
1972 // stack->stack
1973 __ spill_copy_sve_vector_stack_to_stack(src_offset, dst_offset,
1974 sve_vector_reg_size_in_bytes);
1975 } else if (src_lo_rc == rc_float && dst_lo_rc == rc_stack) {
1976 __ spill_sve_vector(as_FloatRegister(Matcher::_regEncode[src_lo]), ra_->reg2offset(dst_lo),
1977 sve_vector_reg_size_in_bytes);
1978 } else if (src_lo_rc == rc_stack && dst_lo_rc == rc_float) {
1979 __ unspill_sve_vector(as_FloatRegister(Matcher::_regEncode[dst_lo]), ra_->reg2offset(src_lo),
1980 sve_vector_reg_size_in_bytes);
1981 } else if (src_lo_rc == rc_float && dst_lo_rc == rc_float) {
1982 __ sve_orr(as_FloatRegister(Matcher::_regEncode[dst_lo]),
1983 as_FloatRegister(Matcher::_regEncode[src_lo]),
1984 as_FloatRegister(Matcher::_regEncode[src_lo]));
1985 } else {
1986 ShouldNotReachHere();
1987 }
1988 } else if (cbuf) {
1989 assert(ireg == Op_VecD || ireg == Op_VecX, "must be 64 bit or 128 bit vector");
1990 C2_MacroAssembler _masm(cbuf);
1991 assert((src_lo_rc != rc_int && dst_lo_rc != rc_int), "sanity");
1992 if (src_lo_rc == rc_stack && dst_lo_rc == rc_stack) {
1993 // stack->stack
1994 assert((src_offset & 7) == 0 && (dst_offset & 7) == 0, "unaligned stack offset");
1995 if (ireg == Op_VecD) {
1996 __ unspill(rscratch1, true, src_offset);
1997 __ spill(rscratch1, true, dst_offset);
1998 } else {
1999 __ spill_copy128(src_offset, dst_offset);
2000 }
2001 } else if (src_lo_rc == rc_float && dst_lo_rc == rc_float) {
2002 __ mov(as_FloatRegister(Matcher::_regEncode[dst_lo]),
2003 ireg == Op_VecD ? __ T8B : __ T16B,
2004 as_FloatRegister(Matcher::_regEncode[src_lo]));
2005 } else if (src_lo_rc == rc_float && dst_lo_rc == rc_stack) {
2006 __ spill(as_FloatRegister(Matcher::_regEncode[src_lo]),
2007 ireg == Op_VecD ? __ D : __ Q,
2008 ra_->reg2offset(dst_lo));
2009 } else if (src_lo_rc == rc_stack && dst_lo_rc == rc_float) {
2010 __ unspill(as_FloatRegister(Matcher::_regEncode[dst_lo]),
2011 ireg == Op_VecD ? __ D : __ Q,
2012 ra_->reg2offset(src_lo));
2013 } else {
2014 ShouldNotReachHere();
2015 }
2016 }
2017 } else if (cbuf) {
2018 C2_MacroAssembler _masm(cbuf);
2019 switch (src_lo_rc) {
2020 case rc_int:
2021 if (dst_lo_rc == rc_int) { // gpr --> gpr copy
2022 if (is64) {
2023 __ mov(as_Register(Matcher::_regEncode[dst_lo]),
2024 as_Register(Matcher::_regEncode[src_lo]));
2025 } else {
2026 C2_MacroAssembler _masm(cbuf);
2027 __ movw(as_Register(Matcher::_regEncode[dst_lo]),
2028 as_Register(Matcher::_regEncode[src_lo]));
2029 }
2030 } else if (dst_lo_rc == rc_float) { // gpr --> fpr copy
2031 if (is64) {
2032 __ fmovd(as_FloatRegister(Matcher::_regEncode[dst_lo]),
2033 as_Register(Matcher::_regEncode[src_lo]));
2034 } else {
2035 __ fmovs(as_FloatRegister(Matcher::_regEncode[dst_lo]),
2036 as_Register(Matcher::_regEncode[src_lo]));
2037 }
2038 } else { // gpr --> stack spill
2039 assert(dst_lo_rc == rc_stack, "spill to bad register class");
2040 __ spill(as_Register(Matcher::_regEncode[src_lo]), is64, dst_offset);
2041 }
2042 break;
2043 case rc_float:
2044 if (dst_lo_rc == rc_int) { // fpr --> gpr copy
2045 if (is64) {
2046 __ fmovd(as_Register(Matcher::_regEncode[dst_lo]),
2047 as_FloatRegister(Matcher::_regEncode[src_lo]));
2048 } else {
2049 __ fmovs(as_Register(Matcher::_regEncode[dst_lo]),
2050 as_FloatRegister(Matcher::_regEncode[src_lo]));
2051 }
2052 } else if (dst_lo_rc == rc_float) { // fpr --> fpr copy
2053 if (is64) {
2054 __ fmovd(as_FloatRegister(Matcher::_regEncode[dst_lo]),
2055 as_FloatRegister(Matcher::_regEncode[src_lo]));
2056 } else {
2057 __ fmovs(as_FloatRegister(Matcher::_regEncode[dst_lo]),
2058 as_FloatRegister(Matcher::_regEncode[src_lo]));
2059 }
2060 } else { // fpr --> stack spill
2061 assert(dst_lo_rc == rc_stack, "spill to bad register class");
2062 __ spill(as_FloatRegister(Matcher::_regEncode[src_lo]),
2063 is64 ? __ D : __ S, dst_offset);
2064 }
2065 break;
2066 case rc_stack:
2067 if (dst_lo_rc == rc_int) { // stack --> gpr load
2068 __ unspill(as_Register(Matcher::_regEncode[dst_lo]), is64, src_offset);
2069 } else if (dst_lo_rc == rc_float) { // stack --> fpr load
2070 __ unspill(as_FloatRegister(Matcher::_regEncode[dst_lo]),
2071 is64 ? __ D : __ S, src_offset);
2072 } else if (dst_lo_rc == rc_predicate) {
2073 __ unspill_sve_predicate(as_PRegister(Matcher::_regEncode[dst_lo]), ra_->reg2offset(src_lo),
2074 Matcher::scalable_vector_reg_size(T_BYTE) >> 3);
2075 } else { // stack --> stack copy
2076 assert(dst_lo_rc == rc_stack, "spill to bad register class");
2077 if (ideal_reg() == Op_RegVectMask) {
2078 __ spill_copy_sve_predicate_stack_to_stack(src_offset, dst_offset,
2079 Matcher::scalable_vector_reg_size(T_BYTE) >> 3);
2080 } else {
2081 __ unspill(rscratch1, is64, src_offset);
2082 __ spill(rscratch1, is64, dst_offset);
2083 }
2084 }
2085 break;
2086 case rc_predicate:
2087 if (dst_lo_rc == rc_predicate) {
2088 __ sve_mov(as_PRegister(Matcher::_regEncode[dst_lo]), as_PRegister(Matcher::_regEncode[src_lo]));
2089 } else if (dst_lo_rc == rc_stack) {
2090 __ spill_sve_predicate(as_PRegister(Matcher::_regEncode[src_lo]), ra_->reg2offset(dst_lo),
2091 Matcher::scalable_vector_reg_size(T_BYTE) >> 3);
2092 } else {
2093 assert(false, "bad src and dst rc_class combination.");
2094 ShouldNotReachHere();
2095 }
2096 break;
2097 default:
2098 assert(false, "bad rc_class for spill");
2099 ShouldNotReachHere();
2100 }
2101 }
2102
2103 if (st) {
2104 st->print("spill ");
2105 if (src_lo_rc == rc_stack) {
2106 st->print("[sp, #%d] -> ", ra_->reg2offset(src_lo));
2107 } else {
2108 st->print("%s -> ", Matcher::regName[src_lo]);
2109 }
2110 if (dst_lo_rc == rc_stack) {
2111 st->print("[sp, #%d]", ra_->reg2offset(dst_lo));
2112 } else {
2113 st->print("%s", Matcher::regName[dst_lo]);
2114 }
2115 if (bottom_type()->isa_vect() && !bottom_type()->isa_vectmask()) {
2116 int vsize = 0;
2117 switch (ideal_reg()) {
2118 case Op_VecD:
2119 vsize = 64;
2120 break;
2121 case Op_VecX:
2122 vsize = 128;
2123 break;
2124 case Op_VecA:
2125 vsize = Matcher::scalable_vector_reg_size(T_BYTE) * 8;
2126 break;
2127 default:
2128 assert(false, "bad register type for spill");
2129 ShouldNotReachHere();
2130 }
2131 st->print("\t# vector spill size = %d", vsize);
2132 } else if (ideal_reg() == Op_RegVectMask) {
2133 assert(Matcher::supports_scalable_vector(), "bad register type for spill");
2134 int vsize = Matcher::scalable_predicate_reg_slots() * 32;
2135 st->print("\t# predicate spill size = %d", vsize);
2136 } else {
2137 st->print("\t# spill size = %d", is64 ? 64 : 32);
2138 }
2139 }
2140
2141 return 0;
2142
2143 }
2144
2145 #ifndef PRODUCT
2146 void MachSpillCopyNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
2147 if (!ra_)
2148 st->print("N%d = SpillCopy(N%d)", _idx, in(1)->_idx);
2149 else
2150 implementation(NULL, ra_, false, st);
2151 }
2152 #endif
2153
2154 void MachSpillCopyNode::emit(CodeBuffer &cbuf, PhaseRegAlloc *ra_) const {
2155 implementation(&cbuf, ra_, false, NULL);
2156 }
2157
2158 uint MachSpillCopyNode::size(PhaseRegAlloc *ra_) const {
2159 return MachNode::size(ra_);
2160 }
2161
2162 //=============================================================================
2163
2164 #ifndef PRODUCT
2165 void BoxLockNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
2166 int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
2167 int reg = ra_->get_reg_first(this);
2168 st->print("add %s, rsp, #%d]\t# box lock",
2169 Matcher::regName[reg], offset);
2170 }
2171 #endif
2172
2173 void BoxLockNode::emit(CodeBuffer &cbuf, PhaseRegAlloc *ra_) const {
2174 C2_MacroAssembler _masm(&cbuf);
2175
2176 int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
2177 int reg = ra_->get_encode(this);
2178
2179 // This add will handle any 24-bit signed offset. 24 bits allows an
2180 // 8 megabyte stack frame.
2181 __ add(as_Register(reg), sp, offset);
2182 }
2183
2184 uint BoxLockNode::size(PhaseRegAlloc *ra_) const {
2185 // BoxLockNode is not a MachNode, so we can't just call MachNode::size(ra_).
2186 int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
2187
2188 if (Assembler::operand_valid_for_add_sub_immediate(offset)) {
2189 return NativeInstruction::instruction_size;
2190 } else {
2191 return 2 * NativeInstruction::instruction_size;
2192 }
2193 }
2194
2195 //=============================================================================
2196
2197 #ifndef PRODUCT
2198 void MachUEPNode::format(PhaseRegAlloc* ra_, outputStream* st) const
2199 {
2200 st->print_cr("# MachUEPNode");
2201 if (UseCompressedClassPointers) {
2202 st->print_cr("\tldrw rscratch1, j_rarg0 + oopDesc::klass_offset_in_bytes()]\t# compressed klass");
2203 if (CompressedKlassPointers::shift() != 0) {
2204 st->print_cr("\tdecode_klass_not_null rscratch1, rscratch1");
2205 }
2206 } else {
2207 st->print_cr("\tldr rscratch1, j_rarg0 + oopDesc::klass_offset_in_bytes()]\t# compressed klass");
2208 }
2209 st->print_cr("\tcmp r0, rscratch1\t # Inline cache check");
2210 st->print_cr("\tbne, SharedRuntime::_ic_miss_stub");
2211 }
2212 #endif
2213
2214 void MachUEPNode::emit(CodeBuffer& cbuf, PhaseRegAlloc* ra_) const
2215 {
2216 // This is the unverified entry point.
2217 C2_MacroAssembler _masm(&cbuf);
2218
2219 __ cmp_klass(j_rarg0, rscratch2, rscratch1);
2220 Label skip;
2221 // TODO
2222 // can we avoid this skip and still use a reloc?
2223 __ br(Assembler::EQ, skip);
2224 __ far_jump(RuntimeAddress(SharedRuntime::get_ic_miss_stub()));
2225 __ bind(skip);
2226 }
2227
2228 uint MachUEPNode::size(PhaseRegAlloc* ra_) const
2229 {
2230 return MachNode::size(ra_);
2231 }
2232
2233 // REQUIRED EMIT CODE
2234
2235 //=============================================================================
2236
2237 // Emit exception handler code.
2238 int HandlerImpl::emit_exception_handler(CodeBuffer& cbuf)
2239 {
2240 // mov rscratch1 #exception_blob_entry_point
2241 // br rscratch1
2242 // Note that the code buffer's insts_mark is always relative to insts.
2243 // That's why we must use the macroassembler to generate a handler.
2244 C2_MacroAssembler _masm(&cbuf);
2245 address base = __ start_a_stub(size_exception_handler());
2246 if (base == NULL) {
2247 ciEnv::current()->record_failure("CodeCache is full");
2248 return 0; // CodeBuffer::expand failed
2249 }
2250 int offset = __ offset();
2251 __ far_jump(RuntimeAddress(OptoRuntime::exception_blob()->entry_point()));
2252 assert(__ offset() - offset <= (int) size_exception_handler(), "overflow");
2253 __ end_a_stub();
2254 return offset;
2255 }
2256
2257 // Emit deopt handler code.
2258 int HandlerImpl::emit_deopt_handler(CodeBuffer& cbuf)
2259 {
2260 // Note that the code buffer's insts_mark is always relative to insts.
2261 // That's why we must use the macroassembler to generate a handler.
2262 C2_MacroAssembler _masm(&cbuf);
2263 address base = __ start_a_stub(size_deopt_handler());
2264 if (base == NULL) {
2265 ciEnv::current()->record_failure("CodeCache is full");
2266 return 0; // CodeBuffer::expand failed
2267 }
2268 int offset = __ offset();
2269
2270 __ adr(lr, __ pc());
2271 __ far_jump(RuntimeAddress(SharedRuntime::deopt_blob()->unpack()));
2272
2273 assert(__ offset() - offset == (int) size_deopt_handler(), "overflow");
2274 __ end_a_stub();
2275 return offset;
2276 }
2277
2278 // REQUIRED MATCHER CODE
2279
2280 //=============================================================================
2281
2282 const bool Matcher::match_rule_supported(int opcode) {
2283 if (!has_match_rule(opcode))
2284 return false;
2285
2286 bool ret_value = true;
2287 switch (opcode) {
2288 case Op_OnSpinWait:
2289 return VM_Version::supports_on_spin_wait();
2290 case Op_CacheWB:
2291 case Op_CacheWBPreSync:
2292 case Op_CacheWBPostSync:
2293 if (!VM_Version::supports_data_cache_line_flush()) {
2294 ret_value = false;
2295 }
2296 break;
2297 }
2298
2299 return ret_value; // Per default match rules are supported.
2300 }
2301
2302 const RegMask* Matcher::predicate_reg_mask(void) {
2303 return &_PR_REG_mask;
2304 }
2305
2306 const TypeVectMask* Matcher::predicate_reg_type(const Type* elemTy, int length) {
2307 return new TypeVectMask(elemTy, length);
2308 }
2309
2310 // Vector calling convention not yet implemented.
2311 const bool Matcher::supports_vector_calling_convention(void) {
2312 return false;
2313 }
2314
2315 OptoRegPair Matcher::vector_return_value(uint ideal_reg) {
2316 Unimplemented();
2317 return OptoRegPair(0, 0);
2318 }
2319
2320 // Is this branch offset short enough that a short branch can be used?
2321 //
2322 // NOTE: If the platform does not provide any short branch variants, then
2323 // this method should return false for offset 0.
2324 bool Matcher::is_short_branch_offset(int rule, int br_size, int offset) {
2325 // The passed offset is relative to address of the branch.
2326
2327 return (-32768 <= offset && offset < 32768);
2328 }
2329
2330 // Vector width in bytes.
2331 const int Matcher::vector_width_in_bytes(BasicType bt) {
2332 // The MaxVectorSize should have been set by detecting SVE max vector register size.
2333 int size = MIN2((UseSVE > 0) ? 256 : 16, (int)MaxVectorSize);
2334 // Minimum 2 values in vector
2335 if (size < 2*type2aelembytes(bt)) size = 0;
2336 // But never < 4
2337 if (size < 4) size = 0;
2338 return size;
2339 }
2340
2341 // Limits on vector size (number of elements) loaded into vector.
2342 const int Matcher::max_vector_size(const BasicType bt) {
2343 return vector_width_in_bytes(bt)/type2aelembytes(bt);
2344 }
2345
2346 const int Matcher::min_vector_size(const BasicType bt) {
2347 int max_size = max_vector_size(bt);
2348 // Limit the min vector size to 8 bytes.
2349 int size = 8 / type2aelembytes(bt);
2350 if (bt == T_BYTE) {
2351 // To support vector api shuffle/rearrange.
2352 size = 4;
2353 } else if (bt == T_BOOLEAN) {
2354 // To support vector api load/store mask.
2355 size = 2;
2356 }
2357 if (size < 2) size = 2;
2358 return MIN2(size, max_size);
2359 }
2360
2361 // Actual max scalable vector register length.
2362 const int Matcher::scalable_vector_reg_size(const BasicType bt) {
2363 return Matcher::max_vector_size(bt);
2364 }
2365
2366 // Vector ideal reg.
2367 const uint Matcher::vector_ideal_reg(int len) {
2368 if (UseSVE > 0 && 16 < len && len <= 256) {
2369 return Op_VecA;
2370 }
2371 switch(len) {
2372 // For 16-bit/32-bit mask vector, reuse VecD.
2373 case 2:
2374 case 4:
2375 case 8: return Op_VecD;
2376 case 16: return Op_VecX;
2377 }
2378 ShouldNotReachHere();
2379 return 0;
2380 }
2381
2382 MachOper* Matcher::pd_specialize_generic_vector_operand(MachOper* generic_opnd, uint ideal_reg, bool is_temp) {
2383 assert(Matcher::is_generic_vector(generic_opnd), "not generic");
2384 switch (ideal_reg) {
2385 case Op_VecA: return new vecAOper();
2386 case Op_VecD: return new vecDOper();
2387 case Op_VecX: return new vecXOper();
2388 }
2389 ShouldNotReachHere();
2390 return NULL;
2391 }
2392
2393 bool Matcher::is_reg2reg_move(MachNode* m) {
2394 return false;
2395 }
2396
2397 bool Matcher::is_generic_vector(MachOper* opnd) {
2398 return opnd->opcode() == VREG;
2399 }
2400
2401 // Return whether or not this register is ever used as an argument.
2402 // This function is used on startup to build the trampoline stubs in
2403 // generateOptoStub. Registers not mentioned will be killed by the VM
2404 // call in the trampoline, and arguments in those registers not be
2405 // available to the callee.
2406 bool Matcher::can_be_java_arg(int reg)
2407 {
2408 return
2409 reg == R0_num || reg == R0_H_num ||
2410 reg == R1_num || reg == R1_H_num ||
2411 reg == R2_num || reg == R2_H_num ||
2412 reg == R3_num || reg == R3_H_num ||
2413 reg == R4_num || reg == R4_H_num ||
2414 reg == R5_num || reg == R5_H_num ||
2415 reg == R6_num || reg == R6_H_num ||
2416 reg == R7_num || reg == R7_H_num ||
2417 reg == V0_num || reg == V0_H_num ||
2418 reg == V1_num || reg == V1_H_num ||
2419 reg == V2_num || reg == V2_H_num ||
2420 reg == V3_num || reg == V3_H_num ||
2421 reg == V4_num || reg == V4_H_num ||
2422 reg == V5_num || reg == V5_H_num ||
2423 reg == V6_num || reg == V6_H_num ||
2424 reg == V7_num || reg == V7_H_num;
2425 }
2426
2427 bool Matcher::is_spillable_arg(int reg)
2428 {
2429 return can_be_java_arg(reg);
2430 }
2431
2432 uint Matcher::int_pressure_limit()
2433 {
2434 // JDK-8183543: When taking the number of available registers as int
2435 // register pressure threshold, the jtreg test:
2436 // test/hotspot/jtreg/compiler/regalloc/TestC2IntPressure.java
2437 // failed due to C2 compilation failure with
2438 // "COMPILE SKIPPED: failed spill-split-recycle sanity check".
2439 //
2440 // A derived pointer is live at CallNode and then is flagged by RA
2441 // as a spilled LRG. Spilling heuristics(Spill-USE) explicitly skip
2442 // derived pointers and lastly fail to spill after reaching maximum
2443 // number of iterations. Lowering the default pressure threshold to
2444 // (_NO_SPECIAL_REG32_mask.Size() minus 1) forces CallNode to become
2445 // a high register pressure area of the code so that split_DEF can
2446 // generate DefinitionSpillCopy for the derived pointer.
2447 uint default_int_pressure_threshold = _NO_SPECIAL_REG32_mask.Size() - 1;
2448 if (!PreserveFramePointer) {
2449 // When PreserveFramePointer is off, frame pointer is allocatable,
2450 // but different from other SOC registers, it is excluded from
2451 // fatproj's mask because its save type is No-Save. Decrease 1 to
2452 // ensure high pressure at fatproj when PreserveFramePointer is off.
2453 // See check_pressure_at_fatproj().
2454 default_int_pressure_threshold--;
2455 }
2456 return (INTPRESSURE == -1) ? default_int_pressure_threshold : INTPRESSURE;
2457 }
2458
2459 uint Matcher::float_pressure_limit()
2460 {
2461 // _FLOAT_REG_mask is generated by adlc from the float_reg register class.
2462 return (FLOATPRESSURE == -1) ? _FLOAT_REG_mask.Size() : FLOATPRESSURE;
2463 }
2464
2465 bool Matcher::use_asm_for_ldiv_by_con(jlong divisor) {
2466 return false;
2467 }
2468
2469 RegMask Matcher::divI_proj_mask() {
2470 ShouldNotReachHere();
2471 return RegMask();
2472 }
2473
2474 // Register for MODI projection of divmodI.
2475 RegMask Matcher::modI_proj_mask() {
2476 ShouldNotReachHere();
2477 return RegMask();
2478 }
2479
2480 // Register for DIVL projection of divmodL.
2481 RegMask Matcher::divL_proj_mask() {
2482 ShouldNotReachHere();
2483 return RegMask();
2484 }
2485
2486 // Register for MODL projection of divmodL.
2487 RegMask Matcher::modL_proj_mask() {
2488 ShouldNotReachHere();
2489 return RegMask();
2490 }
2491
2492 const RegMask Matcher::method_handle_invoke_SP_save_mask() {
2493 return FP_REG_mask();
2494 }
2495
2496 bool size_fits_all_mem_uses(AddPNode* addp, int shift) {
2497 for (DUIterator_Fast imax, i = addp->fast_outs(imax); i < imax; i++) {
2498 Node* u = addp->fast_out(i);
2499 if (u->is_LoadStore()) {
2500 // On AArch64, LoadStoreNodes (i.e. compare and swap
2501 // instructions) only take register indirect as an operand, so
2502 // any attempt to use an AddPNode as an input to a LoadStoreNode
2503 // must fail.
2504 return false;
2505 }
2506 if (u->is_Mem()) {
2507 int opsize = u->as_Mem()->memory_size();
2508 assert(opsize > 0, "unexpected memory operand size");
2509 if (u->as_Mem()->memory_size() != (1<<shift)) {
2510 return false;
2511 }
2512 }
2513 }
2514 return true;
2515 }
2516
2517 // Binary src (Replicate con)
2518 bool is_valid_sve_arith_imm_pattern(Node* n, Node* m) {
2519 if (n == NULL || m == NULL) {
2520 return false;
2521 }
2522
2523 if (UseSVE == 0 || !VectorNode::is_invariant_vector(m)) {
2524 return false;
2525 }
2526
2527 Node* imm_node = m->in(1);
2528 if (!imm_node->is_Con()) {
2529 return false;
2530 }
2531
2532 const Type* t = imm_node->bottom_type();
2533 if (!(t->isa_int() || t->isa_long())) {
2534 return false;
2535 }
2536
2537 switch (n->Opcode()) {
2538 case Op_AndV:
2539 case Op_OrV:
2540 case Op_XorV: {
2541 Assembler::SIMD_RegVariant T = Assembler::elemType_to_regVariant(Matcher::vector_element_basic_type(n));
2542 uint64_t value = t->isa_long() ? (uint64_t)imm_node->get_long() : (uint64_t)imm_node->get_int();
2543 return Assembler::operand_valid_for_sve_logical_immediate(Assembler::regVariant_to_elemBits(T), value);
2544 }
2545 case Op_AddVB:
2546 return (imm_node->get_int() <= 255 && imm_node->get_int() >= -255);
2547 case Op_AddVS:
2548 case Op_AddVI:
2549 return Assembler::operand_valid_for_sve_add_sub_immediate((int64_t)imm_node->get_int());
2550 case Op_AddVL:
2551 return Assembler::operand_valid_for_sve_add_sub_immediate(imm_node->get_long());
2552 default:
2553 return false;
2554 }
2555 }
2556
2557 // (XorV src (Replicate m1))
2558 // (XorVMask src (MaskAll m1))
2559 bool is_vector_bitwise_not_pattern(Node* n, Node* m) {
2560 if (n != NULL && m != NULL) {
2561 return (n->Opcode() == Op_XorV || n->Opcode() == Op_XorVMask) &&
2562 VectorNode::is_all_ones_vector(m);
2563 }
2564 return false;
2565 }
2566
2567 // Should the matcher clone input 'm' of node 'n'?
2568 bool Matcher::pd_clone_node(Node* n, Node* m, Matcher::MStack& mstack) {
2569 if (is_vshift_con_pattern(n, m) ||
2570 is_vector_bitwise_not_pattern(n, m) ||
2571 is_valid_sve_arith_imm_pattern(n, m)) {
2572 mstack.push(m, Visit);
2573 return true;
2574 }
2575 return false;
2576 }
2577
2578 // Should the Matcher clone shifts on addressing modes, expecting them
2579 // to be subsumed into complex addressing expressions or compute them
2580 // into registers?
2581 bool Matcher::pd_clone_address_expressions(AddPNode* m, Matcher::MStack& mstack, VectorSet& address_visited) {
2582 if (clone_base_plus_offset_address(m, mstack, address_visited)) {
2583 return true;
2584 }
2585
2586 Node *off = m->in(AddPNode::Offset);
2587 if (off->Opcode() == Op_LShiftL && off->in(2)->is_Con() &&
2588 size_fits_all_mem_uses(m, off->in(2)->get_int()) &&
2589 // Are there other uses besides address expressions?
2590 !is_visited(off)) {
2591 address_visited.set(off->_idx); // Flag as address_visited
2592 mstack.push(off->in(2), Visit);
2593 Node *conv = off->in(1);
2594 if (conv->Opcode() == Op_ConvI2L &&
2595 // Are there other uses besides address expressions?
2596 !is_visited(conv)) {
2597 address_visited.set(conv->_idx); // Flag as address_visited
2598 mstack.push(conv->in(1), Pre_Visit);
2599 } else {
2600 mstack.push(conv, Pre_Visit);
2601 }
2602 address_visited.test_set(m->_idx); // Flag as address_visited
2603 mstack.push(m->in(AddPNode::Address), Pre_Visit);
2604 mstack.push(m->in(AddPNode::Base), Pre_Visit);
2605 return true;
2606 } else if (off->Opcode() == Op_ConvI2L &&
2607 // Are there other uses besides address expressions?
2608 !is_visited(off)) {
2609 address_visited.test_set(m->_idx); // Flag as address_visited
2610 address_visited.set(off->_idx); // Flag as address_visited
2611 mstack.push(off->in(1), Pre_Visit);
2612 mstack.push(m->in(AddPNode::Address), Pre_Visit);
2613 mstack.push(m->in(AddPNode::Base), Pre_Visit);
2614 return true;
2615 }
2616 return false;
2617 }
2618
2619 #define MOV_VOLATILE(REG, BASE, INDEX, SCALE, DISP, SCRATCH, INSN) \
2620 C2_MacroAssembler _masm(&cbuf); \
2621 { \
2622 guarantee(INDEX == -1, "mode not permitted for volatile"); \
2623 guarantee(DISP == 0, "mode not permitted for volatile"); \
2624 guarantee(SCALE == 0, "mode not permitted for volatile"); \
2625 __ INSN(REG, as_Register(BASE)); \
2626 }
2627
2628
2629 static Address mem2address(int opcode, Register base, int index, int size, int disp)
2630 {
2631 Address::extend scale;
2632
2633 // Hooboy, this is fugly. We need a way to communicate to the
2634 // encoder that the index needs to be sign extended, so we have to
2635 // enumerate all the cases.
2636 switch (opcode) {
2637 case INDINDEXSCALEDI2L:
2638 case INDINDEXSCALEDI2LN:
2639 case INDINDEXI2L:
2640 case INDINDEXI2LN:
2641 scale = Address::sxtw(size);
2642 break;
2643 default:
2644 scale = Address::lsl(size);
2645 }
2646
2647 if (index == -1) {
2648 return Address(base, disp);
2649 } else {
2650 assert(disp == 0, "unsupported address mode: disp = %d", disp);
2651 return Address(base, as_Register(index), scale);
2652 }
2653 }
2654
2655
2656 typedef void (MacroAssembler::* mem_insn)(Register Rt, const Address &adr);
2657 typedef void (MacroAssembler::* mem_insn2)(Register Rt, Register adr);
2658 typedef void (MacroAssembler::* mem_float_insn)(FloatRegister Rt, const Address &adr);
2659 typedef void (MacroAssembler::* mem_vector_insn)(FloatRegister Rt,
2660 MacroAssembler::SIMD_RegVariant T, const Address &adr);
2661
2662 // Used for all non-volatile memory accesses. The use of
2663 // $mem->opcode() to discover whether this pattern uses sign-extended
2664 // offsets is something of a kludge.
2665 static void loadStore(C2_MacroAssembler masm, mem_insn insn,
2666 Register reg, int opcode,
2667 Register base, int index, int scale, int disp,
2668 int size_in_memory)
2669 {
2670 Address addr = mem2address(opcode, base, index, scale, disp);
2671 if (addr.getMode() == Address::base_plus_offset) {
2672 /* If we get an out-of-range offset it is a bug in the compiler,
2673 so we assert here. */
2674 assert(Address::offset_ok_for_immed(addr.offset(), exact_log2(size_in_memory)),
2675 "c2 compiler bug");
2676 /* Fix up any out-of-range offsets. */
2677 assert_different_registers(rscratch1, base);
2678 assert_different_registers(rscratch1, reg);
2679 addr = masm.legitimize_address(addr, size_in_memory, rscratch1);
2680 }
2681 (masm.*insn)(reg, addr);
2682 }
2683
2684 static void loadStore(C2_MacroAssembler masm, mem_float_insn insn,
2685 FloatRegister reg, int opcode,
2686 Register base, int index, int size, int disp,
2687 int size_in_memory)
2688 {
2689 Address::extend scale;
2690
2691 switch (opcode) {
2692 case INDINDEXSCALEDI2L:
2693 case INDINDEXSCALEDI2LN:
2694 scale = Address::sxtw(size);
2695 break;
2696 default:
2697 scale = Address::lsl(size);
2698 }
2699
2700 if (index == -1) {
2701 /* If we get an out-of-range offset it is a bug in the compiler,
2702 so we assert here. */
2703 assert(Address::offset_ok_for_immed(disp, exact_log2(size_in_memory)), "c2 compiler bug");
2704 /* Fix up any out-of-range offsets. */
2705 assert_different_registers(rscratch1, base);
2706 Address addr = Address(base, disp);
2707 addr = masm.legitimize_address(addr, size_in_memory, rscratch1);
2708 (masm.*insn)(reg, addr);
2709 } else {
2710 assert(disp == 0, "unsupported address mode: disp = %d", disp);
2711 (masm.*insn)(reg, Address(base, as_Register(index), scale));
2712 }
2713 }
2714
2715 static void loadStore(C2_MacroAssembler masm, mem_vector_insn insn,
2716 FloatRegister reg, MacroAssembler::SIMD_RegVariant T,
2717 int opcode, Register base, int index, int size, int disp)
2718 {
2719 if (index == -1) {
2720 (masm.*insn)(reg, T, Address(base, disp));
2721 } else {
2722 assert(disp == 0, "unsupported address mode");
2723 (masm.*insn)(reg, T, Address(base, as_Register(index), Address::lsl(size)));
2724 }
2725 }
2726
2727 %}
2728
2729
2730
2731 //----------ENCODING BLOCK-----------------------------------------------------
2732 // This block specifies the encoding classes used by the compiler to
2733 // output byte streams. Encoding classes are parameterized macros
2734 // used by Machine Instruction Nodes in order to generate the bit
2735 // encoding of the instruction. Operands specify their base encoding
2736 // interface with the interface keyword. There are currently
2737 // supported four interfaces, REG_INTER, CONST_INTER, MEMORY_INTER, &
2738 // COND_INTER. REG_INTER causes an operand to generate a function
2739 // which returns its register number when queried. CONST_INTER causes
2740 // an operand to generate a function which returns the value of the
2741 // constant when queried. MEMORY_INTER causes an operand to generate
2742 // four functions which return the Base Register, the Index Register,
2743 // the Scale Value, and the Offset Value of the operand when queried.
2744 // COND_INTER causes an operand to generate six functions which return
2745 // the encoding code (ie - encoding bits for the instruction)
2746 // associated with each basic boolean condition for a conditional
2747 // instruction.
2748 //
2749 // Instructions specify two basic values for encoding. Again, a
2750 // function is available to check if the constant displacement is an
2751 // oop. They use the ins_encode keyword to specify their encoding
2752 // classes (which must be a sequence of enc_class names, and their
2753 // parameters, specified in the encoding block), and they use the
2754 // opcode keyword to specify, in order, their primary, secondary, and
2755 // tertiary opcode. Only the opcode sections which a particular
2756 // instruction needs for encoding need to be specified.
2757 encode %{
2758 // Build emit functions for each basic byte or larger field in the
2759 // intel encoding scheme (opcode, rm, sib, immediate), and call them
2760 // from C++ code in the enc_class source block. Emit functions will
2761 // live in the main source block for now. In future, we can
2762 // generalize this by adding a syntax that specifies the sizes of
2763 // fields in an order, so that the adlc can build the emit functions
2764 // automagically
2765
2766 // catch all for unimplemented encodings
2767 enc_class enc_unimplemented %{
2768 C2_MacroAssembler _masm(&cbuf);
2769 __ unimplemented("C2 catch all");
2770 %}
2771
2772 // BEGIN Non-volatile memory access
2773
2774 // This encoding class is generated automatically from ad_encode.m4.
2775 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2776 enc_class aarch64_enc_ldrsbw(iRegI dst, memory1 mem) %{
2777 Register dst_reg = as_Register($dst$$reg);
2778 loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrsbw, dst_reg, $mem->opcode(),
2779 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
2780 %}
2781
2782 // This encoding class is generated automatically from ad_encode.m4.
2783 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2784 enc_class aarch64_enc_ldrsb(iRegI dst, memory1 mem) %{
2785 Register dst_reg = as_Register($dst$$reg);
2786 loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrsb, dst_reg, $mem->opcode(),
2787 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
2788 %}
2789
2790 // This encoding class is generated automatically from ad_encode.m4.
2791 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2792 enc_class aarch64_enc_ldrb(iRegI dst, memory1 mem) %{
2793 Register dst_reg = as_Register($dst$$reg);
2794 loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrb, dst_reg, $mem->opcode(),
2795 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
2796 %}
2797
2798 // This encoding class is generated automatically from ad_encode.m4.
2799 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2800 enc_class aarch64_enc_ldrb(iRegL dst, memory1 mem) %{
2801 Register dst_reg = as_Register($dst$$reg);
2802 loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrb, dst_reg, $mem->opcode(),
2803 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
2804 %}
2805
2806 // This encoding class is generated automatically from ad_encode.m4.
2807 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2808 enc_class aarch64_enc_ldrshw(iRegI dst, memory2 mem) %{
2809 Register dst_reg = as_Register($dst$$reg);
2810 loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrshw, dst_reg, $mem->opcode(),
2811 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
2812 %}
2813
2814 // This encoding class is generated automatically from ad_encode.m4.
2815 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2816 enc_class aarch64_enc_ldrsh(iRegI dst, memory2 mem) %{
2817 Register dst_reg = as_Register($dst$$reg);
2818 loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrsh, dst_reg, $mem->opcode(),
2819 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
2820 %}
2821
2822 // This encoding class is generated automatically from ad_encode.m4.
2823 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2824 enc_class aarch64_enc_ldrh(iRegI dst, memory2 mem) %{
2825 Register dst_reg = as_Register($dst$$reg);
2826 loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrh, dst_reg, $mem->opcode(),
2827 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
2828 %}
2829
2830 // This encoding class is generated automatically from ad_encode.m4.
2831 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2832 enc_class aarch64_enc_ldrh(iRegL dst, memory2 mem) %{
2833 Register dst_reg = as_Register($dst$$reg);
2834 loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrh, dst_reg, $mem->opcode(),
2835 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
2836 %}
2837
2838 // This encoding class is generated automatically from ad_encode.m4.
2839 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2840 enc_class aarch64_enc_ldrw(iRegI dst, memory4 mem) %{
2841 Register dst_reg = as_Register($dst$$reg);
2842 loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrw, dst_reg, $mem->opcode(),
2843 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
2844 %}
2845
2846 // This encoding class is generated automatically from ad_encode.m4.
2847 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2848 enc_class aarch64_enc_ldrw(iRegL dst, memory4 mem) %{
2849 Register dst_reg = as_Register($dst$$reg);
2850 loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrw, dst_reg, $mem->opcode(),
2851 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
2852 %}
2853
2854 // This encoding class is generated automatically from ad_encode.m4.
2855 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2856 enc_class aarch64_enc_ldrsw(iRegL dst, memory4 mem) %{
2857 Register dst_reg = as_Register($dst$$reg);
2858 loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrsw, dst_reg, $mem->opcode(),
2859 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
2860 %}
2861
2862 // This encoding class is generated automatically from ad_encode.m4.
2863 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2864 enc_class aarch64_enc_ldr(iRegL dst, memory8 mem) %{
2865 Register dst_reg = as_Register($dst$$reg);
2866 loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldr, dst_reg, $mem->opcode(),
2867 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
2868 %}
2869
2870 // This encoding class is generated automatically from ad_encode.m4.
2871 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2872 enc_class aarch64_enc_ldrs(vRegF dst, memory4 mem) %{
2873 FloatRegister dst_reg = as_FloatRegister($dst$$reg);
2874 loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrs, dst_reg, $mem->opcode(),
2875 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
2876 %}
2877
2878 // This encoding class is generated automatically from ad_encode.m4.
2879 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2880 enc_class aarch64_enc_ldrd(vRegD dst, memory8 mem) %{
2881 FloatRegister dst_reg = as_FloatRegister($dst$$reg);
2882 loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrd, dst_reg, $mem->opcode(),
2883 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
2884 %}
2885
2886 // This encoding class is generated automatically from ad_encode.m4.
2887 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2888 enc_class aarch64_enc_strb(iRegI src, memory1 mem) %{
2889 Register src_reg = as_Register($src$$reg);
2890 loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::strb, src_reg, $mem->opcode(),
2891 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
2892 %}
2893
2894 // This encoding class is generated automatically from ad_encode.m4.
2895 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2896 enc_class aarch64_enc_strb0(memory1 mem) %{
2897 C2_MacroAssembler _masm(&cbuf);
2898 loadStore(_masm, &MacroAssembler::strb, zr, $mem->opcode(),
2899 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
2900 %}
2901
2902 // This encoding class is generated automatically from ad_encode.m4.
2903 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2904 enc_class aarch64_enc_strh(iRegI src, memory2 mem) %{
2905 Register src_reg = as_Register($src$$reg);
2906 loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::strh, src_reg, $mem->opcode(),
2907 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
2908 %}
2909
2910 // This encoding class is generated automatically from ad_encode.m4.
2911 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2912 enc_class aarch64_enc_strh0(memory2 mem) %{
2913 C2_MacroAssembler _masm(&cbuf);
2914 loadStore(_masm, &MacroAssembler::strh, zr, $mem->opcode(),
2915 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
2916 %}
2917
2918 // This encoding class is generated automatically from ad_encode.m4.
2919 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2920 enc_class aarch64_enc_strw(iRegI src, memory4 mem) %{
2921 Register src_reg = as_Register($src$$reg);
2922 loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::strw, src_reg, $mem->opcode(),
2923 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
2924 %}
2925
2926 // This encoding class is generated automatically from ad_encode.m4.
2927 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2928 enc_class aarch64_enc_strw0(memory4 mem) %{
2929 C2_MacroAssembler _masm(&cbuf);
2930 loadStore(_masm, &MacroAssembler::strw, zr, $mem->opcode(),
2931 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
2932 %}
2933
2934 // This encoding class is generated automatically from ad_encode.m4.
2935 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2936 enc_class aarch64_enc_str(iRegL src, memory8 mem) %{
2937 Register src_reg = as_Register($src$$reg);
2938 // we sometimes get asked to store the stack pointer into the
2939 // current thread -- we cannot do that directly on AArch64
2940 if (src_reg == r31_sp) {
2941 C2_MacroAssembler _masm(&cbuf);
2942 assert(as_Register($mem$$base) == rthread, "unexpected store for sp");
2943 __ mov(rscratch2, sp);
2944 src_reg = rscratch2;
2945 }
2946 loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::str, src_reg, $mem->opcode(),
2947 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
2948 %}
2949
2950 // This encoding class is generated automatically from ad_encode.m4.
2951 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2952 enc_class aarch64_enc_str0(memory8 mem) %{
2953 C2_MacroAssembler _masm(&cbuf);
2954 loadStore(_masm, &MacroAssembler::str, zr, $mem->opcode(),
2955 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
2956 %}
2957
2958 // This encoding class is generated automatically from ad_encode.m4.
2959 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2960 enc_class aarch64_enc_strs(vRegF src, memory4 mem) %{
2961 FloatRegister src_reg = as_FloatRegister($src$$reg);
2962 loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::strs, src_reg, $mem->opcode(),
2963 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
2964 %}
2965
2966 // This encoding class is generated automatically from ad_encode.m4.
2967 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2968 enc_class aarch64_enc_strd(vRegD src, memory8 mem) %{
2969 FloatRegister src_reg = as_FloatRegister($src$$reg);
2970 loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::strd, src_reg, $mem->opcode(),
2971 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
2972 %}
2973
2974 // This encoding class is generated automatically from ad_encode.m4.
2975 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2976 enc_class aarch64_enc_strb0_ordered(memory4 mem) %{
2977 C2_MacroAssembler _masm(&cbuf);
2978 __ membar(Assembler::StoreStore);
2979 loadStore(_masm, &MacroAssembler::strb, zr, $mem->opcode(),
2980 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
2981 %}
2982
2983 // END Non-volatile memory access
2984
2985 // Vector loads and stores
2986 enc_class aarch64_enc_ldrvH(vReg dst, memory mem) %{
2987 FloatRegister dst_reg = as_FloatRegister($dst$$reg);
2988 loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldr, dst_reg, MacroAssembler::H,
2989 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
2990 %}
2991
2992 enc_class aarch64_enc_ldrvS(vReg dst, memory mem) %{
2993 FloatRegister dst_reg = as_FloatRegister($dst$$reg);
2994 loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldr, dst_reg, MacroAssembler::S,
2995 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
2996 %}
2997
2998 enc_class aarch64_enc_ldrvD(vReg dst, memory mem) %{
2999 FloatRegister dst_reg = as_FloatRegister($dst$$reg);
3000 loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldr, dst_reg, MacroAssembler::D,
3001 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3002 %}
3003
3004 enc_class aarch64_enc_ldrvQ(vReg dst, memory mem) %{
3005 FloatRegister dst_reg = as_FloatRegister($dst$$reg);
3006 loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldr, dst_reg, MacroAssembler::Q,
3007 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3008 %}
3009
3010 enc_class aarch64_enc_strvH(vReg src, memory mem) %{
3011 FloatRegister src_reg = as_FloatRegister($src$$reg);
3012 loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::str, src_reg, MacroAssembler::H,
3013 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3014 %}
3015
3016 enc_class aarch64_enc_strvS(vReg src, memory mem) %{
3017 FloatRegister src_reg = as_FloatRegister($src$$reg);
3018 loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::str, src_reg, MacroAssembler::S,
3019 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3020 %}
3021
3022 enc_class aarch64_enc_strvD(vReg src, memory mem) %{
3023 FloatRegister src_reg = as_FloatRegister($src$$reg);
3024 loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::str, src_reg, MacroAssembler::D,
3025 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3026 %}
3027
3028 enc_class aarch64_enc_strvQ(vReg src, memory mem) %{
3029 FloatRegister src_reg = as_FloatRegister($src$$reg);
3030 loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::str, src_reg, MacroAssembler::Q,
3031 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3032 %}
3033
3034 // volatile loads and stores
3035
3036 enc_class aarch64_enc_stlrb(iRegI src, memory mem) %{
3037 MOV_VOLATILE(as_Register($src$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3038 rscratch1, stlrb);
3039 %}
3040
3041 enc_class aarch64_enc_stlrb0(memory mem) %{
3042 MOV_VOLATILE(zr, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3043 rscratch1, stlrb);
3044 %}
3045
3046 enc_class aarch64_enc_stlrh(iRegI src, memory mem) %{
3047 MOV_VOLATILE(as_Register($src$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3048 rscratch1, stlrh);
3049 %}
3050
3051 enc_class aarch64_enc_stlrh0(memory mem) %{
3052 MOV_VOLATILE(zr, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3053 rscratch1, stlrh);
3054 %}
3055
3056 enc_class aarch64_enc_stlrw(iRegI src, memory mem) %{
3057 MOV_VOLATILE(as_Register($src$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3058 rscratch1, stlrw);
3059 %}
3060
3061 enc_class aarch64_enc_stlrw0(memory mem) %{
3062 MOV_VOLATILE(zr, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3063 rscratch1, stlrw);
3064 %}
3065
3066 enc_class aarch64_enc_ldarsbw(iRegI dst, memory mem) %{
3067 Register dst_reg = as_Register($dst$$reg);
3068 MOV_VOLATILE(dst_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3069 rscratch1, ldarb);
3070 __ sxtbw(dst_reg, dst_reg);
3071 %}
3072
3073 enc_class aarch64_enc_ldarsb(iRegL dst, memory mem) %{
3074 Register dst_reg = as_Register($dst$$reg);
3075 MOV_VOLATILE(dst_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3076 rscratch1, ldarb);
3077 __ sxtb(dst_reg, dst_reg);
3078 %}
3079
3080 enc_class aarch64_enc_ldarbw(iRegI dst, memory mem) %{
3081 MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3082 rscratch1, ldarb);
3083 %}
3084
3085 enc_class aarch64_enc_ldarb(iRegL dst, memory mem) %{
3086 MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3087 rscratch1, ldarb);
3088 %}
3089
3090 enc_class aarch64_enc_ldarshw(iRegI dst, memory mem) %{
3091 Register dst_reg = as_Register($dst$$reg);
3092 MOV_VOLATILE(dst_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3093 rscratch1, ldarh);
3094 __ sxthw(dst_reg, dst_reg);
3095 %}
3096
3097 enc_class aarch64_enc_ldarsh(iRegL dst, memory mem) %{
3098 Register dst_reg = as_Register($dst$$reg);
3099 MOV_VOLATILE(dst_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3100 rscratch1, ldarh);
3101 __ sxth(dst_reg, dst_reg);
3102 %}
3103
3104 enc_class aarch64_enc_ldarhw(iRegI dst, memory mem) %{
3105 MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3106 rscratch1, ldarh);
3107 %}
3108
3109 enc_class aarch64_enc_ldarh(iRegL dst, memory mem) %{
3110 MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3111 rscratch1, ldarh);
3112 %}
3113
3114 enc_class aarch64_enc_ldarw(iRegI dst, memory mem) %{
3115 MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3116 rscratch1, ldarw);
3117 %}
3118
3119 enc_class aarch64_enc_ldarw(iRegL dst, memory mem) %{
3120 MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3121 rscratch1, ldarw);
3122 %}
3123
3124 enc_class aarch64_enc_ldar(iRegL dst, memory mem) %{
3125 MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3126 rscratch1, ldar);
3127 %}
3128
3129 enc_class aarch64_enc_fldars(vRegF dst, memory mem) %{
3130 MOV_VOLATILE(rscratch1, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3131 rscratch1, ldarw);
3132 __ fmovs(as_FloatRegister($dst$$reg), rscratch1);
3133 %}
3134
3135 enc_class aarch64_enc_fldard(vRegD dst, memory mem) %{
3136 MOV_VOLATILE(rscratch1, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3137 rscratch1, ldar);
3138 __ fmovd(as_FloatRegister($dst$$reg), rscratch1);
3139 %}
3140
3141 enc_class aarch64_enc_stlr(iRegL src, memory mem) %{
3142 Register src_reg = as_Register($src$$reg);
3143 // we sometimes get asked to store the stack pointer into the
3144 // current thread -- we cannot do that directly on AArch64
3145 if (src_reg == r31_sp) {
3146 C2_MacroAssembler _masm(&cbuf);
3147 assert(as_Register($mem$$base) == rthread, "unexpected store for sp");
3148 __ mov(rscratch2, sp);
3149 src_reg = rscratch2;
3150 }
3151 MOV_VOLATILE(src_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3152 rscratch1, stlr);
3153 %}
3154
3155 enc_class aarch64_enc_stlr0(memory mem) %{
3156 MOV_VOLATILE(zr, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3157 rscratch1, stlr);
3158 %}
3159
3160 enc_class aarch64_enc_fstlrs(vRegF src, memory mem) %{
3161 {
3162 C2_MacroAssembler _masm(&cbuf);
3163 FloatRegister src_reg = as_FloatRegister($src$$reg);
3164 __ fmovs(rscratch2, src_reg);
3165 }
3166 MOV_VOLATILE(rscratch2, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3167 rscratch1, stlrw);
3168 %}
3169
3170 enc_class aarch64_enc_fstlrd(vRegD src, memory mem) %{
3171 {
3172 C2_MacroAssembler _masm(&cbuf);
3173 FloatRegister src_reg = as_FloatRegister($src$$reg);
3174 __ fmovd(rscratch2, src_reg);
3175 }
3176 MOV_VOLATILE(rscratch2, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3177 rscratch1, stlr);
3178 %}
3179
3180 // synchronized read/update encodings
3181
3182 enc_class aarch64_enc_ldaxr(iRegL dst, memory8 mem) %{
3183 C2_MacroAssembler _masm(&cbuf);
3184 Register dst_reg = as_Register($dst$$reg);
3185 Register base = as_Register($mem$$base);
3186 int index = $mem$$index;
3187 int scale = $mem$$scale;
3188 int disp = $mem$$disp;
3189 if (index == -1) {
3190 if (disp != 0) {
3191 __ lea(rscratch1, Address(base, disp));
3192 __ ldaxr(dst_reg, rscratch1);
3193 } else {
3194 // TODO
3195 // should we ever get anything other than this case?
3196 __ ldaxr(dst_reg, base);
3197 }
3198 } else {
3199 Register index_reg = as_Register(index);
3200 if (disp == 0) {
3201 __ lea(rscratch1, Address(base, index_reg, Address::lsl(scale)));
3202 __ ldaxr(dst_reg, rscratch1);
3203 } else {
3204 __ lea(rscratch1, Address(base, disp));
3205 __ lea(rscratch1, Address(rscratch1, index_reg, Address::lsl(scale)));
3206 __ ldaxr(dst_reg, rscratch1);
3207 }
3208 }
3209 %}
3210
3211 enc_class aarch64_enc_stlxr(iRegLNoSp src, memory8 mem) %{
3212 C2_MacroAssembler _masm(&cbuf);
3213 Register src_reg = as_Register($src$$reg);
3214 Register base = as_Register($mem$$base);
3215 int index = $mem$$index;
3216 int scale = $mem$$scale;
3217 int disp = $mem$$disp;
3218 if (index == -1) {
3219 if (disp != 0) {
3220 __ lea(rscratch2, Address(base, disp));
3221 __ stlxr(rscratch1, src_reg, rscratch2);
3222 } else {
3223 // TODO
3224 // should we ever get anything other than this case?
3225 __ stlxr(rscratch1, src_reg, base);
3226 }
3227 } else {
3228 Register index_reg = as_Register(index);
3229 if (disp == 0) {
3230 __ lea(rscratch2, Address(base, index_reg, Address::lsl(scale)));
3231 __ stlxr(rscratch1, src_reg, rscratch2);
3232 } else {
3233 __ lea(rscratch2, Address(base, disp));
3234 __ lea(rscratch2, Address(rscratch2, index_reg, Address::lsl(scale)));
3235 __ stlxr(rscratch1, src_reg, rscratch2);
3236 }
3237 }
3238 __ cmpw(rscratch1, zr);
3239 %}
3240
3241 enc_class aarch64_enc_cmpxchg(memory mem, iRegLNoSp oldval, iRegLNoSp newval) %{
3242 C2_MacroAssembler _masm(&cbuf);
3243 guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
3244 __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
3245 Assembler::xword, /*acquire*/ false, /*release*/ true,
3246 /*weak*/ false, noreg);
3247 %}
3248
3249 enc_class aarch64_enc_cmpxchgw(memory mem, iRegINoSp oldval, iRegINoSp newval) %{
3250 C2_MacroAssembler _masm(&cbuf);
3251 guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
3252 __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
3253 Assembler::word, /*acquire*/ false, /*release*/ true,
3254 /*weak*/ false, noreg);
3255 %}
3256
3257 enc_class aarch64_enc_cmpxchgs(memory mem, iRegINoSp oldval, iRegINoSp newval) %{
3258 C2_MacroAssembler _masm(&cbuf);
3259 guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
3260 __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
3261 Assembler::halfword, /*acquire*/ false, /*release*/ true,
3262 /*weak*/ false, noreg);
3263 %}
3264
3265 enc_class aarch64_enc_cmpxchgb(memory mem, iRegINoSp oldval, iRegINoSp newval) %{
3266 C2_MacroAssembler _masm(&cbuf);
3267 guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
3268 __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
3269 Assembler::byte, /*acquire*/ false, /*release*/ true,
3270 /*weak*/ false, noreg);
3271 %}
3272
3273
3274 // The only difference between aarch64_enc_cmpxchg and
3275 // aarch64_enc_cmpxchg_acq is that we use load-acquire in the
3276 // CompareAndSwap sequence to serve as a barrier on acquiring a
3277 // lock.
3278 enc_class aarch64_enc_cmpxchg_acq(memory mem, iRegLNoSp oldval, iRegLNoSp newval) %{
3279 C2_MacroAssembler _masm(&cbuf);
3280 guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
3281 __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
3282 Assembler::xword, /*acquire*/ true, /*release*/ true,
3283 /*weak*/ false, noreg);
3284 %}
3285
3286 enc_class aarch64_enc_cmpxchgw_acq(memory mem, iRegINoSp oldval, iRegINoSp newval) %{
3287 C2_MacroAssembler _masm(&cbuf);
3288 guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
3289 __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
3290 Assembler::word, /*acquire*/ true, /*release*/ true,
3291 /*weak*/ false, noreg);
3292 %}
3293
3294 enc_class aarch64_enc_cmpxchgs_acq(memory mem, iRegINoSp oldval, iRegINoSp newval) %{
3295 C2_MacroAssembler _masm(&cbuf);
3296 guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
3297 __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
3298 Assembler::halfword, /*acquire*/ true, /*release*/ true,
3299 /*weak*/ false, noreg);
3300 %}
3301
3302 enc_class aarch64_enc_cmpxchgb_acq(memory mem, iRegINoSp oldval, iRegINoSp newval) %{
3303 C2_MacroAssembler _masm(&cbuf);
3304 guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
3305 __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
3306 Assembler::byte, /*acquire*/ true, /*release*/ true,
3307 /*weak*/ false, noreg);
3308 %}
3309
3310 // auxiliary used for CompareAndSwapX to set result register
3311 enc_class aarch64_enc_cset_eq(iRegINoSp res) %{
3312 C2_MacroAssembler _masm(&cbuf);
3313 Register res_reg = as_Register($res$$reg);
3314 __ cset(res_reg, Assembler::EQ);
3315 %}
3316
3317 // prefetch encodings
3318
3319 enc_class aarch64_enc_prefetchw(memory mem) %{
3320 C2_MacroAssembler _masm(&cbuf);
3321 Register base = as_Register($mem$$base);
3322 int index = $mem$$index;
3323 int scale = $mem$$scale;
3324 int disp = $mem$$disp;
3325 if (index == -1) {
3326 __ prfm(Address(base, disp), PSTL1KEEP);
3327 } else {
3328 Register index_reg = as_Register(index);
3329 if (disp == 0) {
3330 __ prfm(Address(base, index_reg, Address::lsl(scale)), PSTL1KEEP);
3331 } else {
3332 __ lea(rscratch1, Address(base, disp));
3333 __ prfm(Address(rscratch1, index_reg, Address::lsl(scale)), PSTL1KEEP);
3334 }
3335 }
3336 %}
3337
3338 /// mov envcodings
3339
3340 enc_class aarch64_enc_movw_imm(iRegI dst, immI src) %{
3341 C2_MacroAssembler _masm(&cbuf);
3342 uint32_t con = (uint32_t)$src$$constant;
3343 Register dst_reg = as_Register($dst$$reg);
3344 if (con == 0) {
3345 __ movw(dst_reg, zr);
3346 } else {
3347 __ movw(dst_reg, con);
3348 }
3349 %}
3350
3351 enc_class aarch64_enc_mov_imm(iRegL dst, immL src) %{
3352 C2_MacroAssembler _masm(&cbuf);
3353 Register dst_reg = as_Register($dst$$reg);
3354 uint64_t con = (uint64_t)$src$$constant;
3355 if (con == 0) {
3356 __ mov(dst_reg, zr);
3357 } else {
3358 __ mov(dst_reg, con);
3359 }
3360 %}
3361
3362 enc_class aarch64_enc_mov_p(iRegP dst, immP src) %{
3363 C2_MacroAssembler _masm(&cbuf);
3364 Register dst_reg = as_Register($dst$$reg);
3365 address con = (address)$src$$constant;
3366 if (con == NULL || con == (address)1) {
3367 ShouldNotReachHere();
3368 } else {
3369 relocInfo::relocType rtype = $src->constant_reloc();
3370 if (rtype == relocInfo::oop_type) {
3371 __ movoop(dst_reg, (jobject)con);
3372 } else if (rtype == relocInfo::metadata_type) {
3373 __ mov_metadata(dst_reg, (Metadata*)con);
3374 } else {
3375 assert(rtype == relocInfo::none, "unexpected reloc type");
3376 if (con < (address)(uintptr_t)os::vm_page_size()) {
3377 __ mov(dst_reg, con);
3378 } else {
3379 uint64_t offset;
3380 __ adrp(dst_reg, con, offset);
3381 __ add(dst_reg, dst_reg, offset);
3382 }
3383 }
3384 }
3385 %}
3386
3387 enc_class aarch64_enc_mov_p0(iRegP dst, immP0 src) %{
3388 C2_MacroAssembler _masm(&cbuf);
3389 Register dst_reg = as_Register($dst$$reg);
3390 __ mov(dst_reg, zr);
3391 %}
3392
3393 enc_class aarch64_enc_mov_p1(iRegP dst, immP_1 src) %{
3394 C2_MacroAssembler _masm(&cbuf);
3395 Register dst_reg = as_Register($dst$$reg);
3396 __ mov(dst_reg, (uint64_t)1);
3397 %}
3398
3399 enc_class aarch64_enc_mov_byte_map_base(iRegP dst, immByteMapBase src) %{
3400 C2_MacroAssembler _masm(&cbuf);
3401 __ load_byte_map_base($dst$$Register);
3402 %}
3403
3404 enc_class aarch64_enc_mov_n(iRegN dst, immN src) %{
3405 C2_MacroAssembler _masm(&cbuf);
3406 Register dst_reg = as_Register($dst$$reg);
3407 address con = (address)$src$$constant;
3408 if (con == NULL) {
3409 ShouldNotReachHere();
3410 } else {
3411 relocInfo::relocType rtype = $src->constant_reloc();
3412 assert(rtype == relocInfo::oop_type, "unexpected reloc type");
3413 __ set_narrow_oop(dst_reg, (jobject)con);
3414 }
3415 %}
3416
3417 enc_class aarch64_enc_mov_n0(iRegN dst, immN0 src) %{
3418 C2_MacroAssembler _masm(&cbuf);
3419 Register dst_reg = as_Register($dst$$reg);
3420 __ mov(dst_reg, zr);
3421 %}
3422
3423 enc_class aarch64_enc_mov_nk(iRegN dst, immNKlass src) %{
3424 C2_MacroAssembler _masm(&cbuf);
3425 Register dst_reg = as_Register($dst$$reg);
3426 address con = (address)$src$$constant;
3427 if (con == NULL) {
3428 ShouldNotReachHere();
3429 } else {
3430 relocInfo::relocType rtype = $src->constant_reloc();
3431 assert(rtype == relocInfo::metadata_type, "unexpected reloc type");
3432 __ set_narrow_klass(dst_reg, (Klass *)con);
3433 }
3434 %}
3435
3436 // arithmetic encodings
3437
3438 enc_class aarch64_enc_addsubw_imm(iRegI dst, iRegI src1, immIAddSub src2) %{
3439 C2_MacroAssembler _masm(&cbuf);
3440 Register dst_reg = as_Register($dst$$reg);
3441 Register src_reg = as_Register($src1$$reg);
3442 int32_t con = (int32_t)$src2$$constant;
3443 // add has primary == 0, subtract has primary == 1
3444 if ($primary) { con = -con; }
3445 if (con < 0) {
3446 __ subw(dst_reg, src_reg, -con);
3447 } else {
3448 __ addw(dst_reg, src_reg, con);
3449 }
3450 %}
3451
3452 enc_class aarch64_enc_addsub_imm(iRegL dst, iRegL src1, immLAddSub src2) %{
3453 C2_MacroAssembler _masm(&cbuf);
3454 Register dst_reg = as_Register($dst$$reg);
3455 Register src_reg = as_Register($src1$$reg);
3456 int32_t con = (int32_t)$src2$$constant;
3457 // add has primary == 0, subtract has primary == 1
3458 if ($primary) { con = -con; }
3459 if (con < 0) {
3460 __ sub(dst_reg, src_reg, -con);
3461 } else {
3462 __ add(dst_reg, src_reg, con);
3463 }
3464 %}
3465
3466 enc_class aarch64_enc_divw(iRegI dst, iRegI src1, iRegI src2) %{
3467 C2_MacroAssembler _masm(&cbuf);
3468 Register dst_reg = as_Register($dst$$reg);
3469 Register src1_reg = as_Register($src1$$reg);
3470 Register src2_reg = as_Register($src2$$reg);
3471 __ corrected_idivl(dst_reg, src1_reg, src2_reg, false, rscratch1);
3472 %}
3473
3474 enc_class aarch64_enc_div(iRegI dst, iRegI src1, iRegI src2) %{
3475 C2_MacroAssembler _masm(&cbuf);
3476 Register dst_reg = as_Register($dst$$reg);
3477 Register src1_reg = as_Register($src1$$reg);
3478 Register src2_reg = as_Register($src2$$reg);
3479 __ corrected_idivq(dst_reg, src1_reg, src2_reg, false, rscratch1);
3480 %}
3481
3482 enc_class aarch64_enc_modw(iRegI dst, iRegI src1, iRegI src2) %{
3483 C2_MacroAssembler _masm(&cbuf);
3484 Register dst_reg = as_Register($dst$$reg);
3485 Register src1_reg = as_Register($src1$$reg);
3486 Register src2_reg = as_Register($src2$$reg);
3487 __ corrected_idivl(dst_reg, src1_reg, src2_reg, true, rscratch1);
3488 %}
3489
3490 enc_class aarch64_enc_mod(iRegI dst, iRegI src1, iRegI src2) %{
3491 C2_MacroAssembler _masm(&cbuf);
3492 Register dst_reg = as_Register($dst$$reg);
3493 Register src1_reg = as_Register($src1$$reg);
3494 Register src2_reg = as_Register($src2$$reg);
3495 __ corrected_idivq(dst_reg, src1_reg, src2_reg, true, rscratch1);
3496 %}
3497
3498 // compare instruction encodings
3499
3500 enc_class aarch64_enc_cmpw(iRegI src1, iRegI src2) %{
3501 C2_MacroAssembler _masm(&cbuf);
3502 Register reg1 = as_Register($src1$$reg);
3503 Register reg2 = as_Register($src2$$reg);
3504 __ cmpw(reg1, reg2);
3505 %}
3506
3507 enc_class aarch64_enc_cmpw_imm_addsub(iRegI src1, immIAddSub src2) %{
3508 C2_MacroAssembler _masm(&cbuf);
3509 Register reg = as_Register($src1$$reg);
3510 int32_t val = $src2$$constant;
3511 if (val >= 0) {
3512 __ subsw(zr, reg, val);
3513 } else {
3514 __ addsw(zr, reg, -val);
3515 }
3516 %}
3517
3518 enc_class aarch64_enc_cmpw_imm(iRegI src1, immI src2) %{
3519 C2_MacroAssembler _masm(&cbuf);
3520 Register reg1 = as_Register($src1$$reg);
3521 uint32_t val = (uint32_t)$src2$$constant;
3522 __ movw(rscratch1, val);
3523 __ cmpw(reg1, rscratch1);
3524 %}
3525
3526 enc_class aarch64_enc_cmp(iRegL src1, iRegL src2) %{
3527 C2_MacroAssembler _masm(&cbuf);
3528 Register reg1 = as_Register($src1$$reg);
3529 Register reg2 = as_Register($src2$$reg);
3530 __ cmp(reg1, reg2);
3531 %}
3532
3533 enc_class aarch64_enc_cmp_imm_addsub(iRegL src1, immL12 src2) %{
3534 C2_MacroAssembler _masm(&cbuf);
3535 Register reg = as_Register($src1$$reg);
3536 int64_t val = $src2$$constant;
3537 if (val >= 0) {
3538 __ subs(zr, reg, val);
3539 } else if (val != -val) {
3540 __ adds(zr, reg, -val);
3541 } else {
3542 // aargh, Long.MIN_VALUE is a special case
3543 __ orr(rscratch1, zr, (uint64_t)val);
3544 __ subs(zr, reg, rscratch1);
3545 }
3546 %}
3547
3548 enc_class aarch64_enc_cmp_imm(iRegL src1, immL src2) %{
3549 C2_MacroAssembler _masm(&cbuf);
3550 Register reg1 = as_Register($src1$$reg);
3551 uint64_t val = (uint64_t)$src2$$constant;
3552 __ mov(rscratch1, val);
3553 __ cmp(reg1, rscratch1);
3554 %}
3555
3556 enc_class aarch64_enc_cmpp(iRegP src1, iRegP src2) %{
3557 C2_MacroAssembler _masm(&cbuf);
3558 Register reg1 = as_Register($src1$$reg);
3559 Register reg2 = as_Register($src2$$reg);
3560 __ cmp(reg1, reg2);
3561 %}
3562
3563 enc_class aarch64_enc_cmpn(iRegN src1, iRegN src2) %{
3564 C2_MacroAssembler _masm(&cbuf);
3565 Register reg1 = as_Register($src1$$reg);
3566 Register reg2 = as_Register($src2$$reg);
3567 __ cmpw(reg1, reg2);
3568 %}
3569
3570 enc_class aarch64_enc_testp(iRegP src) %{
3571 C2_MacroAssembler _masm(&cbuf);
3572 Register reg = as_Register($src$$reg);
3573 __ cmp(reg, zr);
3574 %}
3575
3576 enc_class aarch64_enc_testn(iRegN src) %{
3577 C2_MacroAssembler _masm(&cbuf);
3578 Register reg = as_Register($src$$reg);
3579 __ cmpw(reg, zr);
3580 %}
3581
3582 enc_class aarch64_enc_b(label lbl) %{
3583 C2_MacroAssembler _masm(&cbuf);
3584 Label *L = $lbl$$label;
3585 __ b(*L);
3586 %}
3587
3588 enc_class aarch64_enc_br_con(cmpOp cmp, label lbl) %{
3589 C2_MacroAssembler _masm(&cbuf);
3590 Label *L = $lbl$$label;
3591 __ br ((Assembler::Condition)$cmp$$cmpcode, *L);
3592 %}
3593
3594 enc_class aarch64_enc_br_conU(cmpOpU cmp, label lbl) %{
3595 C2_MacroAssembler _masm(&cbuf);
3596 Label *L = $lbl$$label;
3597 __ br ((Assembler::Condition)$cmp$$cmpcode, *L);
3598 %}
3599
3600 enc_class aarch64_enc_partial_subtype_check(iRegP sub, iRegP super, iRegP temp, iRegP result)
3601 %{
3602 Register sub_reg = as_Register($sub$$reg);
3603 Register super_reg = as_Register($super$$reg);
3604 Register temp_reg = as_Register($temp$$reg);
3605 Register result_reg = as_Register($result$$reg);
3606
3607 Label miss;
3608 C2_MacroAssembler _masm(&cbuf);
3609 __ check_klass_subtype_slow_path(sub_reg, super_reg, temp_reg, result_reg,
3610 NULL, &miss,
3611 /*set_cond_codes:*/ true);
3612 if ($primary) {
3613 __ mov(result_reg, zr);
3614 }
3615 __ bind(miss);
3616 %}
3617
3618 enc_class aarch64_enc_java_static_call(method meth) %{
3619 C2_MacroAssembler _masm(&cbuf);
3620
3621 address addr = (address)$meth$$method;
3622 address call;
3623 if (!_method) {
3624 // A call to a runtime wrapper, e.g. new, new_typeArray_Java, uncommon_trap.
3625 call = __ trampoline_call(Address(addr, relocInfo::runtime_call_type));
3626 if (call == NULL) {
3627 ciEnv::current()->record_failure("CodeCache is full");
3628 return;
3629 }
3630 } else {
3631 int method_index = resolved_method_index(cbuf);
3632 RelocationHolder rspec = _optimized_virtual ? opt_virtual_call_Relocation::spec(method_index)
3633 : static_call_Relocation::spec(method_index);
3634 call = __ trampoline_call(Address(addr, rspec));
3635 if (call == NULL) {
3636 ciEnv::current()->record_failure("CodeCache is full");
3637 return;
3638 }
3639 if (CodeBuffer::supports_shared_stubs() && _method->can_be_statically_bound()) {
3640 // Calls of the same statically bound method can share
3641 // a stub to the interpreter.
3642 cbuf.shared_stub_to_interp_for(_method, call - cbuf.insts_begin());
3643 } else {
3644 // Emit stub for static call
3645 address stub = CompiledStaticCall::emit_to_interp_stub(cbuf, call);
3646 if (stub == NULL) {
3647 ciEnv::current()->record_failure("CodeCache is full");
3648 return;
3649 }
3650 }
3651 }
3652
3653 __ post_call_nop();
3654
3655 // Only non uncommon_trap calls need to reinitialize ptrue.
3656 if (Compile::current()->max_vector_size() > 0 && uncommon_trap_request() == 0) {
3657 __ reinitialize_ptrue();
3658 }
3659 %}
3660
3661 enc_class aarch64_enc_java_dynamic_call(method meth) %{
3662 C2_MacroAssembler _masm(&cbuf);
3663 int method_index = resolved_method_index(cbuf);
3664 address call = __ ic_call((address)$meth$$method, method_index);
3665 if (call == NULL) {
3666 ciEnv::current()->record_failure("CodeCache is full");
3667 return;
3668 }
3669 _masm.clear_inst_mark();
3670 __ post_call_nop();
3671 if (Compile::current()->max_vector_size() > 0) {
3672 __ reinitialize_ptrue();
3673 }
3674 %}
3675
3676 enc_class aarch64_enc_call_epilog() %{
3677 C2_MacroAssembler _masm(&cbuf);
3678 if (VerifyStackAtCalls) {
3679 // Check that stack depth is unchanged: find majik cookie on stack
3680 __ call_Unimplemented();
3681 }
3682 %}
3683
3684 enc_class aarch64_enc_java_to_runtime(method meth) %{
3685 C2_MacroAssembler _masm(&cbuf);
3686
3687 // some calls to generated routines (arraycopy code) are scheduled
3688 // by C2 as runtime calls. if so we can call them using a br (they
3689 // will be in a reachable segment) otherwise we have to use a blr
3690 // which loads the absolute address into a register.
3691 address entry = (address)$meth$$method;
3692 CodeBlob *cb = CodeCache::find_blob(entry);
3693 if (cb) {
3694 address call = __ trampoline_call(Address(entry, relocInfo::runtime_call_type));
3695 if (call == NULL) {
3696 ciEnv::current()->record_failure("CodeCache is full");
3697 return;
3698 }
3699 __ post_call_nop();
3700 } else {
3701 Label retaddr;
3702 __ adr(rscratch2, retaddr);
3703 __ lea(rscratch1, RuntimeAddress(entry));
3704 // Leave a breadcrumb for JavaFrameAnchor::capture_last_Java_pc()
3705 __ stp(zr, rscratch2, Address(__ pre(sp, -2 * wordSize)));
3706 __ blr(rscratch1);
3707 __ bind(retaddr);
3708 __ post_call_nop();
3709 __ add(sp, sp, 2 * wordSize);
3710 }
3711 if (Compile::current()->max_vector_size() > 0) {
3712 __ reinitialize_ptrue();
3713 }
3714 %}
3715
3716 enc_class aarch64_enc_rethrow() %{
3717 C2_MacroAssembler _masm(&cbuf);
3718 __ far_jump(RuntimeAddress(OptoRuntime::rethrow_stub()));
3719 %}
3720
3721 enc_class aarch64_enc_ret() %{
3722 C2_MacroAssembler _masm(&cbuf);
3723 #ifdef ASSERT
3724 if (Compile::current()->max_vector_size() > 0) {
3725 __ verify_ptrue();
3726 }
3727 #endif
3728 __ ret(lr);
3729 %}
3730
3731 enc_class aarch64_enc_tail_call(iRegP jump_target) %{
3732 C2_MacroAssembler _masm(&cbuf);
3733 Register target_reg = as_Register($jump_target$$reg);
3734 __ br(target_reg);
3735 %}
3736
3737 enc_class aarch64_enc_tail_jmp(iRegP jump_target) %{
3738 C2_MacroAssembler _masm(&cbuf);
3739 Register target_reg = as_Register($jump_target$$reg);
3740 // exception oop should be in r0
3741 // ret addr has been popped into lr
3742 // callee expects it in r3
3743 __ mov(r3, lr);
3744 __ br(target_reg);
3745 %}
3746
3747 enc_class aarch64_enc_fast_lock(iRegP object, iRegP box, iRegP tmp, iRegP tmp2) %{
3748 C2_MacroAssembler _masm(&cbuf);
3749 Register oop = as_Register($object$$reg);
3750 Register box = as_Register($box$$reg);
3751 Register disp_hdr = as_Register($tmp$$reg);
3752 Register tmp = as_Register($tmp2$$reg);
3753 Label cont;
3754 Label object_has_monitor;
3755 Label no_count;
3756
3757 assert_different_registers(oop, box, tmp, disp_hdr);
3758
3759 // Load markWord from object into displaced_header.
3760 __ ldr(disp_hdr, Address(oop, oopDesc::mark_offset_in_bytes()));
3761
3762 if (DiagnoseSyncOnValueBasedClasses != 0) {
3763 __ load_klass(tmp, oop);
3764 __ ldrw(tmp, Address(tmp, Klass::access_flags_offset()));
3765 __ tstw(tmp, JVM_ACC_IS_VALUE_BASED_CLASS);
3766 __ br(Assembler::NE, cont);
3767 }
3768
3769 // Check for existing monitor
3770 __ tbnz(disp_hdr, exact_log2(markWord::monitor_value), object_has_monitor);
3771
3772 if (!UseHeavyMonitors) {
3773 Label slow;
3774 __ fast_lock(oop, disp_hdr, box, tmp, rscratch1, slow);
3775
3776 // Indicate success at cont.
3777 __ cmp(oop, oop);
3778 __ b(cont);
3779 __ bind(slow);
3780 }
3781 __ tst(oop, oop); // Set NE to indicate 'failure' -> take slow-path. We know that oop != 0.
3782 __ b(cont);
3783
3784 // Handle existing monitor.
3785 __ bind(object_has_monitor);
3786
3787 // The object's monitor m is unlocked iff m->owner == NULL,
3788 // otherwise m->owner may contain a thread or a stack address.
3789 //
3790 // Try to CAS m->owner from NULL to current thread.
3791 __ add(tmp, disp_hdr, (ObjectMonitor::owner_offset_in_bytes()-markWord::monitor_value));
3792 __ cmpxchg(tmp, zr, rthread, Assembler::xword, /*acquire*/ true,
3793 /*release*/ true, /*weak*/ false, rscratch1); // Sets flags for result
3794
3795 __ br(Assembler::EQ, cont); // CAS success means locking succeeded
3796
3797 __ cmp(rscratch1, rthread);
3798 __ br(Assembler::NE, cont); // Check for recursive locking
3799
3800 // Recursive lock case
3801 __ increment(Address(disp_hdr, ObjectMonitor::recursions_offset_in_bytes() - markWord::monitor_value), 1);
3802 // flag == EQ still from the cmp above, checking if this is a reentrant lock
3803
3804 __ bind(cont);
3805 // flag == EQ indicates success
3806 // flag == NE indicates failure
3807 __ br(Assembler::NE, no_count);
3808
3809 __ increment(Address(rthread, JavaThread::held_monitor_count_offset()));
3810
3811 __ bind(no_count);
3812 %}
3813
3814 enc_class aarch64_enc_fast_unlock(iRegP object, iRegP box, iRegP tmp, iRegP tmp2) %{
3815 C2_MacroAssembler _masm(&cbuf);
3816 Register oop = as_Register($object$$reg);
3817 Register box = as_Register($box$$reg);
3818 Register disp_hdr = as_Register($tmp$$reg);
3819 Register tmp = as_Register($tmp2$$reg);
3820 Label cont;
3821 Label object_has_monitor;
3822 Label no_count;
3823
3824 assert_different_registers(oop, box, tmp, disp_hdr);
3825
3826 // Handle existing monitor.
3827 __ ldr(tmp, Address(oop, oopDesc::mark_offset_in_bytes()));
3828
3829 assert(oopDesc::mark_offset_in_bytes() == 0, "offset of _mark is not 0");
3830
3831 if (!UseHeavyMonitors) {
3832 Label slow;
3833 __ tbnz(tmp, exact_log2(markWord::monitor_value), object_has_monitor);
3834
3835 __ fast_unlock(oop, tmp, box, disp_hdr, slow);
3836
3837 // Indicate success at cont.
3838 __ cmp(oop, oop);
3839 __ b(cont);
3840
3841 __ bind(slow);
3842 }
3843 __ tst(oop, oop); // Set NE to indicate 'failure' -> take slow-path. We know that oop != 0.
3844 __ b(cont);
3845
3846 // Handle existing monitor.
3847 __ bind(object_has_monitor);
3848 STATIC_ASSERT(markWord::monitor_value <= INT_MAX);
3849 __ add(tmp, tmp, -(int)markWord::monitor_value); // monitor
3850
3851 // If the owner is anonymous, we need to fix it -- in the slow-path.
3852 {
3853 Label L;
3854 __ ldr(disp_hdr, Address(tmp, ObjectMonitor::owner_offset_in_bytes()));
3855 __ cmp(disp_hdr, (unsigned char)(intptr_t) ANONYMOUS_OWNER);
3856 __ br(Assembler::NE, L);
3857 __ tst(oop, oop); // Indicate failure at cont -- dive into slow-path.
3858 __ b(cont);
3859 __ bind(L);
3860 }
3861
3862 __ ldr(disp_hdr, Address(tmp, ObjectMonitor::recursions_offset_in_bytes()));
3863
3864 Label notRecursive;
3865 __ cbz(disp_hdr, notRecursive);
3866
3867 // Recursive lock
3868 __ sub(disp_hdr, disp_hdr, 1u);
3869 __ str(disp_hdr, Address(tmp, ObjectMonitor::recursions_offset_in_bytes()));
3870 __ cmp(disp_hdr, disp_hdr); // Sets flags for result
3871 __ b(cont);
3872
3873 __ bind(notRecursive);
3874 __ ldr(rscratch1, Address(tmp, ObjectMonitor::EntryList_offset_in_bytes()));
3875 __ ldr(disp_hdr, Address(tmp, ObjectMonitor::cxq_offset_in_bytes()));
3876 __ orr(rscratch1, rscratch1, disp_hdr); // Will be 0 if both are 0.
3877 __ cmp(rscratch1, zr); // Sets flags for result
3878 __ cbnz(rscratch1, cont);
3879 // need a release store here
3880 __ lea(tmp, Address(tmp, ObjectMonitor::owner_offset_in_bytes()));
3881 __ stlr(zr, tmp); // set unowned
3882
3883 __ bind(cont);
3884 // flag == EQ indicates success
3885 // flag == NE indicates failure
3886 __ br(Assembler::NE, no_count);
3887
3888 __ decrement(Address(rthread, JavaThread::held_monitor_count_offset()));
3889
3890 __ bind(no_count);
3891 %}
3892
3893 %}
3894
3895 //----------FRAME--------------------------------------------------------------
3896 // Definition of frame structure and management information.
3897 //
3898 // S T A C K L A Y O U T Allocators stack-slot number
3899 // | (to get allocators register number
3900 // G Owned by | | v add OptoReg::stack0())
3901 // r CALLER | |
3902 // o | +--------+ pad to even-align allocators stack-slot
3903 // w V | pad0 | numbers; owned by CALLER
3904 // t -----------+--------+----> Matcher::_in_arg_limit, unaligned
3905 // h ^ | in | 5
3906 // | | args | 4 Holes in incoming args owned by SELF
3907 // | | | | 3
3908 // | | +--------+
3909 // V | | old out| Empty on Intel, window on Sparc
3910 // | old |preserve| Must be even aligned.
3911 // | SP-+--------+----> Matcher::_old_SP, even aligned
3912 // | | in | 3 area for Intel ret address
3913 // Owned by |preserve| Empty on Sparc.
3914 // SELF +--------+
3915 // | | pad2 | 2 pad to align old SP
3916 // | +--------+ 1
3917 // | | locks | 0
3918 // | +--------+----> OptoReg::stack0(), even aligned
3919 // | | pad1 | 11 pad to align new SP
3920 // | +--------+
3921 // | | | 10
3922 // | | spills | 9 spills
3923 // V | | 8 (pad0 slot for callee)
3924 // -----------+--------+----> Matcher::_out_arg_limit, unaligned
3925 // ^ | out | 7
3926 // | | args | 6 Holes in outgoing args owned by CALLEE
3927 // Owned by +--------+
3928 // CALLEE | new out| 6 Empty on Intel, window on Sparc
3929 // | new |preserve| Must be even-aligned.
3930 // | SP-+--------+----> Matcher::_new_SP, even aligned
3931 // | | |
3932 //
3933 // Note 1: Only region 8-11 is determined by the allocator. Region 0-5 is
3934 // known from SELF's arguments and the Java calling convention.
3935 // Region 6-7 is determined per call site.
3936 // Note 2: If the calling convention leaves holes in the incoming argument
3937 // area, those holes are owned by SELF. Holes in the outgoing area
3938 // are owned by the CALLEE. Holes should not be necessary in the
3939 // incoming area, as the Java calling convention is completely under
3940 // the control of the AD file. Doubles can be sorted and packed to
3941 // avoid holes. Holes in the outgoing arguments may be necessary for
3942 // varargs C calling conventions.
3943 // Note 3: Region 0-3 is even aligned, with pad2 as needed. Region 3-5 is
3944 // even aligned with pad0 as needed.
3945 // Region 6 is even aligned. Region 6-7 is NOT even aligned;
3946 // (the latter is true on Intel but is it false on AArch64?)
3947 // region 6-11 is even aligned; it may be padded out more so that
3948 // the region from SP to FP meets the minimum stack alignment.
3949 // Note 4: For I2C adapters, the incoming FP may not meet the minimum stack
3950 // alignment. Region 11, pad1, may be dynamically extended so that
3951 // SP meets the minimum alignment.
3952
3953 frame %{
3954 // These three registers define part of the calling convention
3955 // between compiled code and the interpreter.
3956
3957 // Inline Cache Register or Method for I2C.
3958 inline_cache_reg(R12);
3959
3960 // Number of stack slots consumed by locking an object
3961 sync_stack_slots(2);
3962
3963 // Compiled code's Frame Pointer
3964 frame_pointer(R31);
3965
3966 // Interpreter stores its frame pointer in a register which is
3967 // stored to the stack by I2CAdaptors.
3968 // I2CAdaptors convert from interpreted java to compiled java.
3969 interpreter_frame_pointer(R29);
3970
3971 // Stack alignment requirement
3972 stack_alignment(StackAlignmentInBytes); // Alignment size in bytes (128-bit -> 16 bytes)
3973
3974 // Number of outgoing stack slots killed above the out_preserve_stack_slots
3975 // for calls to C. Supports the var-args backing area for register parms.
3976 varargs_C_out_slots_killed(frame::arg_reg_save_area_bytes/BytesPerInt);
3977
3978 // The after-PROLOG location of the return address. Location of
3979 // return address specifies a type (REG or STACK) and a number
3980 // representing the register number (i.e. - use a register name) or
3981 // stack slot.
3982 // Ret Addr is on stack in slot 0 if no locks or verification or alignment.
3983 // Otherwise, it is above the locks and verification slot and alignment word
3984 // TODO this may well be correct but need to check why that - 2 is there
3985 // ppc port uses 0 but we definitely need to allow for fixed_slots
3986 // which folds in the space used for monitors
3987 return_addr(STACK - 2 +
3988 align_up((Compile::current()->in_preserve_stack_slots() +
3989 Compile::current()->fixed_slots()),
3990 stack_alignment_in_slots()));
3991
3992 // Location of compiled Java return values. Same as C for now.
3993 return_value
3994 %{
3995 // TODO do we allow ideal_reg == Op_RegN???
3996 assert(ideal_reg >= Op_RegI && ideal_reg <= Op_RegL,
3997 "only return normal values");
3998
3999 static const int lo[Op_RegL + 1] = { // enum name
4000 0, // Op_Node
4001 0, // Op_Set
4002 R0_num, // Op_RegN
4003 R0_num, // Op_RegI
4004 R0_num, // Op_RegP
4005 V0_num, // Op_RegF
4006 V0_num, // Op_RegD
4007 R0_num // Op_RegL
4008 };
4009
4010 static const int hi[Op_RegL + 1] = { // enum name
4011 0, // Op_Node
4012 0, // Op_Set
4013 OptoReg::Bad, // Op_RegN
4014 OptoReg::Bad, // Op_RegI
4015 R0_H_num, // Op_RegP
4016 OptoReg::Bad, // Op_RegF
4017 V0_H_num, // Op_RegD
4018 R0_H_num // Op_RegL
4019 };
4020
4021 return OptoRegPair(hi[ideal_reg], lo[ideal_reg]);
4022 %}
4023 %}
4024
4025 //----------ATTRIBUTES---------------------------------------------------------
4026 //----------Operand Attributes-------------------------------------------------
4027 op_attrib op_cost(1); // Required cost attribute
4028
4029 //----------Instruction Attributes---------------------------------------------
4030 ins_attrib ins_cost(INSN_COST); // Required cost attribute
4031 ins_attrib ins_size(32); // Required size attribute (in bits)
4032 ins_attrib ins_short_branch(0); // Required flag: is this instruction
4033 // a non-matching short branch variant
4034 // of some long branch?
4035 ins_attrib ins_alignment(4); // Required alignment attribute (must
4036 // be a power of 2) specifies the
4037 // alignment that some part of the
4038 // instruction (not necessarily the
4039 // start) requires. If > 1, a
4040 // compute_padding() function must be
4041 // provided for the instruction
4042
4043 //----------OPERANDS-----------------------------------------------------------
4044 // Operand definitions must precede instruction definitions for correct parsing
4045 // in the ADLC because operands constitute user defined types which are used in
4046 // instruction definitions.
4047
4048 //----------Simple Operands----------------------------------------------------
4049
4050 // Integer operands 32 bit
4051 // 32 bit immediate
4052 operand immI()
4053 %{
4054 match(ConI);
4055
4056 op_cost(0);
4057 format %{ %}
4058 interface(CONST_INTER);
4059 %}
4060
4061 // 32 bit zero
4062 operand immI0()
4063 %{
4064 predicate(n->get_int() == 0);
4065 match(ConI);
4066
4067 op_cost(0);
4068 format %{ %}
4069 interface(CONST_INTER);
4070 %}
4071
4072 // 32 bit unit increment
4073 operand immI_1()
4074 %{
4075 predicate(n->get_int() == 1);
4076 match(ConI);
4077
4078 op_cost(0);
4079 format %{ %}
4080 interface(CONST_INTER);
4081 %}
4082
4083 // 32 bit unit decrement
4084 operand immI_M1()
4085 %{
4086 predicate(n->get_int() == -1);
4087 match(ConI);
4088
4089 op_cost(0);
4090 format %{ %}
4091 interface(CONST_INTER);
4092 %}
4093
4094 // Shift values for add/sub extension shift
4095 operand immIExt()
4096 %{
4097 predicate(0 <= n->get_int() && (n->get_int() <= 4));
4098 match(ConI);
4099
4100 op_cost(0);
4101 format %{ %}
4102 interface(CONST_INTER);
4103 %}
4104
4105 operand immI_gt_1()
4106 %{
4107 predicate(n->get_int() > 1);
4108 match(ConI);
4109
4110 op_cost(0);
4111 format %{ %}
4112 interface(CONST_INTER);
4113 %}
4114
4115 operand immI_le_4()
4116 %{
4117 predicate(n->get_int() <= 4);
4118 match(ConI);
4119
4120 op_cost(0);
4121 format %{ %}
4122 interface(CONST_INTER);
4123 %}
4124
4125 operand immI_16()
4126 %{
4127 predicate(n->get_int() == 16);
4128 match(ConI);
4129
4130 op_cost(0);
4131 format %{ %}
4132 interface(CONST_INTER);
4133 %}
4134
4135 operand immI_24()
4136 %{
4137 predicate(n->get_int() == 24);
4138 match(ConI);
4139
4140 op_cost(0);
4141 format %{ %}
4142 interface(CONST_INTER);
4143 %}
4144
4145 operand immI_32()
4146 %{
4147 predicate(n->get_int() == 32);
4148 match(ConI);
4149
4150 op_cost(0);
4151 format %{ %}
4152 interface(CONST_INTER);
4153 %}
4154
4155 operand immI_48()
4156 %{
4157 predicate(n->get_int() == 48);
4158 match(ConI);
4159
4160 op_cost(0);
4161 format %{ %}
4162 interface(CONST_INTER);
4163 %}
4164
4165 operand immI_56()
4166 %{
4167 predicate(n->get_int() == 56);
4168 match(ConI);
4169
4170 op_cost(0);
4171 format %{ %}
4172 interface(CONST_INTER);
4173 %}
4174
4175 operand immI_63()
4176 %{
4177 predicate(n->get_int() == 63);
4178 match(ConI);
4179
4180 op_cost(0);
4181 format %{ %}
4182 interface(CONST_INTER);
4183 %}
4184
4185 operand immI_64()
4186 %{
4187 predicate(n->get_int() == 64);
4188 match(ConI);
4189
4190 op_cost(0);
4191 format %{ %}
4192 interface(CONST_INTER);
4193 %}
4194
4195 operand immI_255()
4196 %{
4197 predicate(n->get_int() == 255);
4198 match(ConI);
4199
4200 op_cost(0);
4201 format %{ %}
4202 interface(CONST_INTER);
4203 %}
4204
4205 operand immI_65535()
4206 %{
4207 predicate(n->get_int() == 65535);
4208 match(ConI);
4209
4210 op_cost(0);
4211 format %{ %}
4212 interface(CONST_INTER);
4213 %}
4214
4215 operand immI_positive()
4216 %{
4217 predicate(n->get_int() > 0);
4218 match(ConI);
4219
4220 op_cost(0);
4221 format %{ %}
4222 interface(CONST_INTER);
4223 %}
4224
4225 operand immL_255()
4226 %{
4227 predicate(n->get_long() == 255L);
4228 match(ConL);
4229
4230 op_cost(0);
4231 format %{ %}
4232 interface(CONST_INTER);
4233 %}
4234
4235 operand immL_65535()
4236 %{
4237 predicate(n->get_long() == 65535L);
4238 match(ConL);
4239
4240 op_cost(0);
4241 format %{ %}
4242 interface(CONST_INTER);
4243 %}
4244
4245 operand immL_4294967295()
4246 %{
4247 predicate(n->get_long() == 4294967295L);
4248 match(ConL);
4249
4250 op_cost(0);
4251 format %{ %}
4252 interface(CONST_INTER);
4253 %}
4254
4255 operand immL_bitmask()
4256 %{
4257 predicate((n->get_long() != 0)
4258 && ((n->get_long() & 0xc000000000000000l) == 0)
4259 && is_power_of_2(n->get_long() + 1));
4260 match(ConL);
4261
4262 op_cost(0);
4263 format %{ %}
4264 interface(CONST_INTER);
4265 %}
4266
4267 operand immI_bitmask()
4268 %{
4269 predicate((n->get_int() != 0)
4270 && ((n->get_int() & 0xc0000000) == 0)
4271 && is_power_of_2(n->get_int() + 1));
4272 match(ConI);
4273
4274 op_cost(0);
4275 format %{ %}
4276 interface(CONST_INTER);
4277 %}
4278
4279 operand immL_positive_bitmaskI()
4280 %{
4281 predicate((n->get_long() != 0)
4282 && ((julong)n->get_long() < 0x80000000ULL)
4283 && is_power_of_2(n->get_long() + 1));
4284 match(ConL);
4285
4286 op_cost(0);
4287 format %{ %}
4288 interface(CONST_INTER);
4289 %}
4290
4291 // Scale values for scaled offset addressing modes (up to long but not quad)
4292 operand immIScale()
4293 %{
4294 predicate(0 <= n->get_int() && (n->get_int() <= 3));
4295 match(ConI);
4296
4297 op_cost(0);
4298 format %{ %}
4299 interface(CONST_INTER);
4300 %}
4301
4302 // 26 bit signed offset -- for pc-relative branches
4303 operand immI26()
4304 %{
4305 predicate(((-(1 << 25)) <= n->get_int()) && (n->get_int() < (1 << 25)));
4306 match(ConI);
4307
4308 op_cost(0);
4309 format %{ %}
4310 interface(CONST_INTER);
4311 %}
4312
4313 // 19 bit signed offset -- for pc-relative loads
4314 operand immI19()
4315 %{
4316 predicate(((-(1 << 18)) <= n->get_int()) && (n->get_int() < (1 << 18)));
4317 match(ConI);
4318
4319 op_cost(0);
4320 format %{ %}
4321 interface(CONST_INTER);
4322 %}
4323
4324 // 12 bit unsigned offset -- for base plus immediate loads
4325 operand immIU12()
4326 %{
4327 predicate((0 <= n->get_int()) && (n->get_int() < (1 << 12)));
4328 match(ConI);
4329
4330 op_cost(0);
4331 format %{ %}
4332 interface(CONST_INTER);
4333 %}
4334
4335 operand immLU12()
4336 %{
4337 predicate((0 <= n->get_long()) && (n->get_long() < (1 << 12)));
4338 match(ConL);
4339
4340 op_cost(0);
4341 format %{ %}
4342 interface(CONST_INTER);
4343 %}
4344
4345 // Offset for scaled or unscaled immediate loads and stores
4346 operand immIOffset()
4347 %{
4348 predicate(Address::offset_ok_for_immed(n->get_int(), 0));
4349 match(ConI);
4350
4351 op_cost(0);
4352 format %{ %}
4353 interface(CONST_INTER);
4354 %}
4355
4356 operand immIOffset1()
4357 %{
4358 predicate(Address::offset_ok_for_immed(n->get_int(), 0));
4359 match(ConI);
4360
4361 op_cost(0);
4362 format %{ %}
4363 interface(CONST_INTER);
4364 %}
4365
4366 operand immIOffset2()
4367 %{
4368 predicate(Address::offset_ok_for_immed(n->get_int(), 1));
4369 match(ConI);
4370
4371 op_cost(0);
4372 format %{ %}
4373 interface(CONST_INTER);
4374 %}
4375
4376 operand immIOffset4()
4377 %{
4378 predicate(Address::offset_ok_for_immed(n->get_int(), 2));
4379 match(ConI);
4380
4381 op_cost(0);
4382 format %{ %}
4383 interface(CONST_INTER);
4384 %}
4385
4386 operand immIOffset8()
4387 %{
4388 predicate(Address::offset_ok_for_immed(n->get_int(), 3));
4389 match(ConI);
4390
4391 op_cost(0);
4392 format %{ %}
4393 interface(CONST_INTER);
4394 %}
4395
4396 operand immIOffset16()
4397 %{
4398 predicate(Address::offset_ok_for_immed(n->get_int(), 4));
4399 match(ConI);
4400
4401 op_cost(0);
4402 format %{ %}
4403 interface(CONST_INTER);
4404 %}
4405
4406 operand immLoffset()
4407 %{
4408 predicate(Address::offset_ok_for_immed(n->get_long(), 0));
4409 match(ConL);
4410
4411 op_cost(0);
4412 format %{ %}
4413 interface(CONST_INTER);
4414 %}
4415
4416 operand immLoffset1()
4417 %{
4418 predicate(Address::offset_ok_for_immed(n->get_long(), 0));
4419 match(ConL);
4420
4421 op_cost(0);
4422 format %{ %}
4423 interface(CONST_INTER);
4424 %}
4425
4426 operand immLoffset2()
4427 %{
4428 predicate(Address::offset_ok_for_immed(n->get_long(), 1));
4429 match(ConL);
4430
4431 op_cost(0);
4432 format %{ %}
4433 interface(CONST_INTER);
4434 %}
4435
4436 operand immLoffset4()
4437 %{
4438 predicate(Address::offset_ok_for_immed(n->get_long(), 2));
4439 match(ConL);
4440
4441 op_cost(0);
4442 format %{ %}
4443 interface(CONST_INTER);
4444 %}
4445
4446 operand immLoffset8()
4447 %{
4448 predicate(Address::offset_ok_for_immed(n->get_long(), 3));
4449 match(ConL);
4450
4451 op_cost(0);
4452 format %{ %}
4453 interface(CONST_INTER);
4454 %}
4455
4456 operand immLoffset16()
4457 %{
4458 predicate(Address::offset_ok_for_immed(n->get_long(), 4));
4459 match(ConL);
4460
4461 op_cost(0);
4462 format %{ %}
4463 interface(CONST_INTER);
4464 %}
4465
4466 // 8 bit signed value.
4467 operand immI8()
4468 %{
4469 predicate(n->get_int() <= 127 && n->get_int() >= -128);
4470 match(ConI);
4471
4472 op_cost(0);
4473 format %{ %}
4474 interface(CONST_INTER);
4475 %}
4476
4477 // 8 bit signed value (simm8), or #simm8 LSL 8.
4478 operand immI8_shift8()
4479 %{
4480 predicate((n->get_int() <= 127 && n->get_int() >= -128) ||
4481 (n->get_int() <= 32512 && n->get_int() >= -32768 && (n->get_int() & 0xff) == 0));
4482 match(ConI);
4483
4484 op_cost(0);
4485 format %{ %}
4486 interface(CONST_INTER);
4487 %}
4488
4489 // 8 bit signed value (simm8), or #simm8 LSL 8.
4490 operand immL8_shift8()
4491 %{
4492 predicate((n->get_long() <= 127 && n->get_long() >= -128) ||
4493 (n->get_long() <= 32512 && n->get_long() >= -32768 && (n->get_long() & 0xff) == 0));
4494 match(ConL);
4495
4496 op_cost(0);
4497 format %{ %}
4498 interface(CONST_INTER);
4499 %}
4500
4501 // 8 bit integer valid for vector add sub immediate
4502 operand immBAddSubV()
4503 %{
4504 predicate(n->get_int() <= 255 && n->get_int() >= -255);
4505 match(ConI);
4506
4507 op_cost(0);
4508 format %{ %}
4509 interface(CONST_INTER);
4510 %}
4511
4512 // 32 bit integer valid for add sub immediate
4513 operand immIAddSub()
4514 %{
4515 predicate(Assembler::operand_valid_for_add_sub_immediate((int64_t)n->get_int()));
4516 match(ConI);
4517 op_cost(0);
4518 format %{ %}
4519 interface(CONST_INTER);
4520 %}
4521
4522 // 32 bit integer valid for vector add sub immediate
4523 operand immIAddSubV()
4524 %{
4525 predicate(Assembler::operand_valid_for_sve_add_sub_immediate((int64_t)n->get_int()));
4526 match(ConI);
4527
4528 op_cost(0);
4529 format %{ %}
4530 interface(CONST_INTER);
4531 %}
4532
4533 // 32 bit unsigned integer valid for logical immediate
4534
4535 operand immBLog()
4536 %{
4537 predicate(Assembler::operand_valid_for_sve_logical_immediate(BitsPerByte, (uint64_t)n->get_int()));
4538 match(ConI);
4539
4540 op_cost(0);
4541 format %{ %}
4542 interface(CONST_INTER);
4543 %}
4544
4545 operand immSLog()
4546 %{
4547 predicate(Assembler::operand_valid_for_sve_logical_immediate(BitsPerShort, (uint64_t)n->get_int()));
4548 match(ConI);
4549
4550 op_cost(0);
4551 format %{ %}
4552 interface(CONST_INTER);
4553 %}
4554
4555 operand immILog()
4556 %{
4557 predicate(Assembler::operand_valid_for_logical_immediate(/*is32*/true, (uint64_t)n->get_int()));
4558 match(ConI);
4559
4560 op_cost(0);
4561 format %{ %}
4562 interface(CONST_INTER);
4563 %}
4564
4565 // Integer operands 64 bit
4566 // 64 bit immediate
4567 operand immL()
4568 %{
4569 match(ConL);
4570
4571 op_cost(0);
4572 format %{ %}
4573 interface(CONST_INTER);
4574 %}
4575
4576 // 64 bit zero
4577 operand immL0()
4578 %{
4579 predicate(n->get_long() == 0);
4580 match(ConL);
4581
4582 op_cost(0);
4583 format %{ %}
4584 interface(CONST_INTER);
4585 %}
4586
4587 // 64 bit unit increment
4588 operand immL_1()
4589 %{
4590 predicate(n->get_long() == 1);
4591 match(ConL);
4592
4593 op_cost(0);
4594 format %{ %}
4595 interface(CONST_INTER);
4596 %}
4597
4598 // 64 bit unit decrement
4599 operand immL_M1()
4600 %{
4601 predicate(n->get_long() == -1);
4602 match(ConL);
4603
4604 op_cost(0);
4605 format %{ %}
4606 interface(CONST_INTER);
4607 %}
4608
4609 // 32 bit offset of pc in thread anchor
4610
4611 operand immL_pc_off()
4612 %{
4613 predicate(n->get_long() == in_bytes(JavaThread::frame_anchor_offset()) +
4614 in_bytes(JavaFrameAnchor::last_Java_pc_offset()));
4615 match(ConL);
4616
4617 op_cost(0);
4618 format %{ %}
4619 interface(CONST_INTER);
4620 %}
4621
4622 // 64 bit integer valid for add sub immediate
4623 operand immLAddSub()
4624 %{
4625 predicate(Assembler::operand_valid_for_add_sub_immediate(n->get_long()));
4626 match(ConL);
4627 op_cost(0);
4628 format %{ %}
4629 interface(CONST_INTER);
4630 %}
4631
4632 // 64 bit integer valid for addv subv immediate
4633 operand immLAddSubV()
4634 %{
4635 predicate(Assembler::operand_valid_for_sve_add_sub_immediate(n->get_long()));
4636 match(ConL);
4637
4638 op_cost(0);
4639 format %{ %}
4640 interface(CONST_INTER);
4641 %}
4642
4643 // 64 bit integer valid for logical immediate
4644 operand immLLog()
4645 %{
4646 predicate(Assembler::operand_valid_for_logical_immediate(/*is32*/false, (uint64_t)n->get_long()));
4647 match(ConL);
4648 op_cost(0);
4649 format %{ %}
4650 interface(CONST_INTER);
4651 %}
4652
4653 // Long Immediate: low 32-bit mask
4654 operand immL_32bits()
4655 %{
4656 predicate(n->get_long() == 0xFFFFFFFFL);
4657 match(ConL);
4658 op_cost(0);
4659 format %{ %}
4660 interface(CONST_INTER);
4661 %}
4662
4663 // Pointer operands
4664 // Pointer Immediate
4665 operand immP()
4666 %{
4667 match(ConP);
4668
4669 op_cost(0);
4670 format %{ %}
4671 interface(CONST_INTER);
4672 %}
4673
4674 // NULL Pointer Immediate
4675 operand immP0()
4676 %{
4677 predicate(n->get_ptr() == 0);
4678 match(ConP);
4679
4680 op_cost(0);
4681 format %{ %}
4682 interface(CONST_INTER);
4683 %}
4684
4685 // Pointer Immediate One
4686 // this is used in object initialization (initial object header)
4687 operand immP_1()
4688 %{
4689 predicate(n->get_ptr() == 1);
4690 match(ConP);
4691
4692 op_cost(0);
4693 format %{ %}
4694 interface(CONST_INTER);
4695 %}
4696
4697 // Card Table Byte Map Base
4698 operand immByteMapBase()
4699 %{
4700 // Get base of card map
4701 predicate(BarrierSet::barrier_set()->is_a(BarrierSet::CardTableBarrierSet) &&
4702 (CardTable::CardValue*)n->get_ptr() == ((CardTableBarrierSet*)(BarrierSet::barrier_set()))->card_table()->byte_map_base());
4703 match(ConP);
4704
4705 op_cost(0);
4706 format %{ %}
4707 interface(CONST_INTER);
4708 %}
4709
4710 // Pointer Immediate Minus One
4711 // this is used when we want to write the current PC to the thread anchor
4712 operand immP_M1()
4713 %{
4714 predicate(n->get_ptr() == -1);
4715 match(ConP);
4716
4717 op_cost(0);
4718 format %{ %}
4719 interface(CONST_INTER);
4720 %}
4721
4722 // Pointer Immediate Minus Two
4723 // this is used when we want to write the current PC to the thread anchor
4724 operand immP_M2()
4725 %{
4726 predicate(n->get_ptr() == -2);
4727 match(ConP);
4728
4729 op_cost(0);
4730 format %{ %}
4731 interface(CONST_INTER);
4732 %}
4733
4734 // Float and Double operands
4735 // Double Immediate
4736 operand immD()
4737 %{
4738 match(ConD);
4739 op_cost(0);
4740 format %{ %}
4741 interface(CONST_INTER);
4742 %}
4743
4744 // Double Immediate: +0.0d
4745 operand immD0()
4746 %{
4747 predicate(jlong_cast(n->getd()) == 0);
4748 match(ConD);
4749
4750 op_cost(0);
4751 format %{ %}
4752 interface(CONST_INTER);
4753 %}
4754
4755 // constant 'double +0.0'.
4756 operand immDPacked()
4757 %{
4758 predicate(Assembler::operand_valid_for_float_immediate(n->getd()));
4759 match(ConD);
4760 op_cost(0);
4761 format %{ %}
4762 interface(CONST_INTER);
4763 %}
4764
4765 // Float Immediate
4766 operand immF()
4767 %{
4768 match(ConF);
4769 op_cost(0);
4770 format %{ %}
4771 interface(CONST_INTER);
4772 %}
4773
4774 // Float Immediate: +0.0f.
4775 operand immF0()
4776 %{
4777 predicate(jint_cast(n->getf()) == 0);
4778 match(ConF);
4779
4780 op_cost(0);
4781 format %{ %}
4782 interface(CONST_INTER);
4783 %}
4784
4785 //
4786 operand immFPacked()
4787 %{
4788 predicate(Assembler::operand_valid_for_float_immediate((double)n->getf()));
4789 match(ConF);
4790 op_cost(0);
4791 format %{ %}
4792 interface(CONST_INTER);
4793 %}
4794
4795 // Narrow pointer operands
4796 // Narrow Pointer Immediate
4797 operand immN()
4798 %{
4799 match(ConN);
4800
4801 op_cost(0);
4802 format %{ %}
4803 interface(CONST_INTER);
4804 %}
4805
4806 // Narrow NULL Pointer Immediate
4807 operand immN0()
4808 %{
4809 predicate(n->get_narrowcon() == 0);
4810 match(ConN);
4811
4812 op_cost(0);
4813 format %{ %}
4814 interface(CONST_INTER);
4815 %}
4816
4817 operand immNKlass()
4818 %{
4819 match(ConNKlass);
4820
4821 op_cost(0);
4822 format %{ %}
4823 interface(CONST_INTER);
4824 %}
4825
4826 // Integer 32 bit Register Operands
4827 // Integer 32 bitRegister (excludes SP)
4828 operand iRegI()
4829 %{
4830 constraint(ALLOC_IN_RC(any_reg32));
4831 match(RegI);
4832 match(iRegINoSp);
4833 op_cost(0);
4834 format %{ %}
4835 interface(REG_INTER);
4836 %}
4837
4838 // Integer 32 bit Register not Special
4839 operand iRegINoSp()
4840 %{
4841 constraint(ALLOC_IN_RC(no_special_reg32));
4842 match(RegI);
4843 op_cost(0);
4844 format %{ %}
4845 interface(REG_INTER);
4846 %}
4847
4848 // Integer 64 bit Register Operands
4849 // Integer 64 bit Register (includes SP)
4850 operand iRegL()
4851 %{
4852 constraint(ALLOC_IN_RC(any_reg));
4853 match(RegL);
4854 match(iRegLNoSp);
4855 op_cost(0);
4856 format %{ %}
4857 interface(REG_INTER);
4858 %}
4859
4860 // Integer 64 bit Register not Special
4861 operand iRegLNoSp()
4862 %{
4863 constraint(ALLOC_IN_RC(no_special_reg));
4864 match(RegL);
4865 match(iRegL_R0);
4866 format %{ %}
4867 interface(REG_INTER);
4868 %}
4869
4870 // Pointer Register Operands
4871 // Pointer Register
4872 operand iRegP()
4873 %{
4874 constraint(ALLOC_IN_RC(ptr_reg));
4875 match(RegP);
4876 match(iRegPNoSp);
4877 match(iRegP_R0);
4878 //match(iRegP_R2);
4879 //match(iRegP_R4);
4880 //match(iRegP_R5);
4881 match(thread_RegP);
4882 op_cost(0);
4883 format %{ %}
4884 interface(REG_INTER);
4885 %}
4886
4887 // Pointer 64 bit Register not Special
4888 operand iRegPNoSp()
4889 %{
4890 constraint(ALLOC_IN_RC(no_special_ptr_reg));
4891 match(RegP);
4892 // match(iRegP);
4893 // match(iRegP_R0);
4894 // match(iRegP_R2);
4895 // match(iRegP_R4);
4896 // match(iRegP_R5);
4897 // match(thread_RegP);
4898 op_cost(0);
4899 format %{ %}
4900 interface(REG_INTER);
4901 %}
4902
4903 // Pointer 64 bit Register R0 only
4904 operand iRegP_R0()
4905 %{
4906 constraint(ALLOC_IN_RC(r0_reg));
4907 match(RegP);
4908 // match(iRegP);
4909 match(iRegPNoSp);
4910 op_cost(0);
4911 format %{ %}
4912 interface(REG_INTER);
4913 %}
4914
4915 // Pointer 64 bit Register R1 only
4916 operand iRegP_R1()
4917 %{
4918 constraint(ALLOC_IN_RC(r1_reg));
4919 match(RegP);
4920 // match(iRegP);
4921 match(iRegPNoSp);
4922 op_cost(0);
4923 format %{ %}
4924 interface(REG_INTER);
4925 %}
4926
4927 // Pointer 64 bit Register R2 only
4928 operand iRegP_R2()
4929 %{
4930 constraint(ALLOC_IN_RC(r2_reg));
4931 match(RegP);
4932 // match(iRegP);
4933 match(iRegPNoSp);
4934 op_cost(0);
4935 format %{ %}
4936 interface(REG_INTER);
4937 %}
4938
4939 // Pointer 64 bit Register R3 only
4940 operand iRegP_R3()
4941 %{
4942 constraint(ALLOC_IN_RC(r3_reg));
4943 match(RegP);
4944 // match(iRegP);
4945 match(iRegPNoSp);
4946 op_cost(0);
4947 format %{ %}
4948 interface(REG_INTER);
4949 %}
4950
4951 // Pointer 64 bit Register R4 only
4952 operand iRegP_R4()
4953 %{
4954 constraint(ALLOC_IN_RC(r4_reg));
4955 match(RegP);
4956 // match(iRegP);
4957 match(iRegPNoSp);
4958 op_cost(0);
4959 format %{ %}
4960 interface(REG_INTER);
4961 %}
4962
4963 // Pointer 64 bit Register R5 only
4964 operand iRegP_R5()
4965 %{
4966 constraint(ALLOC_IN_RC(r5_reg));
4967 match(RegP);
4968 // match(iRegP);
4969 match(iRegPNoSp);
4970 op_cost(0);
4971 format %{ %}
4972 interface(REG_INTER);
4973 %}
4974
4975 // Pointer 64 bit Register R10 only
4976 operand iRegP_R10()
4977 %{
4978 constraint(ALLOC_IN_RC(r10_reg));
4979 match(RegP);
4980 // match(iRegP);
4981 match(iRegPNoSp);
4982 op_cost(0);
4983 format %{ %}
4984 interface(REG_INTER);
4985 %}
4986
4987 // Long 64 bit Register R0 only
4988 operand iRegL_R0()
4989 %{
4990 constraint(ALLOC_IN_RC(r0_reg));
4991 match(RegL);
4992 match(iRegLNoSp);
4993 op_cost(0);
4994 format %{ %}
4995 interface(REG_INTER);
4996 %}
4997
4998 // Long 64 bit Register R2 only
4999 operand iRegL_R2()
5000 %{
5001 constraint(ALLOC_IN_RC(r2_reg));
5002 match(RegL);
5003 match(iRegLNoSp);
5004 op_cost(0);
5005 format %{ %}
5006 interface(REG_INTER);
5007 %}
5008
5009 // Long 64 bit Register R3 only
5010 operand iRegL_R3()
5011 %{
5012 constraint(ALLOC_IN_RC(r3_reg));
5013 match(RegL);
5014 match(iRegLNoSp);
5015 op_cost(0);
5016 format %{ %}
5017 interface(REG_INTER);
5018 %}
5019
5020 // Long 64 bit Register R11 only
5021 operand iRegL_R11()
5022 %{
5023 constraint(ALLOC_IN_RC(r11_reg));
5024 match(RegL);
5025 match(iRegLNoSp);
5026 op_cost(0);
5027 format %{ %}
5028 interface(REG_INTER);
5029 %}
5030
5031 // Pointer 64 bit Register FP only
5032 operand iRegP_FP()
5033 %{
5034 constraint(ALLOC_IN_RC(fp_reg));
5035 match(RegP);
5036 // match(iRegP);
5037 op_cost(0);
5038 format %{ %}
5039 interface(REG_INTER);
5040 %}
5041
5042 // Register R0 only
5043 operand iRegI_R0()
5044 %{
5045 constraint(ALLOC_IN_RC(int_r0_reg));
5046 match(RegI);
5047 match(iRegINoSp);
5048 op_cost(0);
5049 format %{ %}
5050 interface(REG_INTER);
5051 %}
5052
5053 // Register R2 only
5054 operand iRegI_R2()
5055 %{
5056 constraint(ALLOC_IN_RC(int_r2_reg));
5057 match(RegI);
5058 match(iRegINoSp);
5059 op_cost(0);
5060 format %{ %}
5061 interface(REG_INTER);
5062 %}
5063
5064 // Register R3 only
5065 operand iRegI_R3()
5066 %{
5067 constraint(ALLOC_IN_RC(int_r3_reg));
5068 match(RegI);
5069 match(iRegINoSp);
5070 op_cost(0);
5071 format %{ %}
5072 interface(REG_INTER);
5073 %}
5074
5075
5076 // Register R4 only
5077 operand iRegI_R4()
5078 %{
5079 constraint(ALLOC_IN_RC(int_r4_reg));
5080 match(RegI);
5081 match(iRegINoSp);
5082 op_cost(0);
5083 format %{ %}
5084 interface(REG_INTER);
5085 %}
5086
5087
5088 // Pointer Register Operands
5089 // Narrow Pointer Register
5090 operand iRegN()
5091 %{
5092 constraint(ALLOC_IN_RC(any_reg32));
5093 match(RegN);
5094 match(iRegNNoSp);
5095 op_cost(0);
5096 format %{ %}
5097 interface(REG_INTER);
5098 %}
5099
5100 operand iRegN_R0()
5101 %{
5102 constraint(ALLOC_IN_RC(r0_reg));
5103 match(iRegN);
5104 op_cost(0);
5105 format %{ %}
5106 interface(REG_INTER);
5107 %}
5108
5109 operand iRegN_R2()
5110 %{
5111 constraint(ALLOC_IN_RC(r2_reg));
5112 match(iRegN);
5113 op_cost(0);
5114 format %{ %}
5115 interface(REG_INTER);
5116 %}
5117
5118 operand iRegN_R3()
5119 %{
5120 constraint(ALLOC_IN_RC(r3_reg));
5121 match(iRegN);
5122 op_cost(0);
5123 format %{ %}
5124 interface(REG_INTER);
5125 %}
5126
5127 // Integer 64 bit Register not Special
5128 operand iRegNNoSp()
5129 %{
5130 constraint(ALLOC_IN_RC(no_special_reg32));
5131 match(RegN);
5132 op_cost(0);
5133 format %{ %}
5134 interface(REG_INTER);
5135 %}
5136
5137 // heap base register -- used for encoding immN0
5138
5139 operand iRegIHeapbase()
5140 %{
5141 constraint(ALLOC_IN_RC(heapbase_reg));
5142 match(RegI);
5143 op_cost(0);
5144 format %{ %}
5145 interface(REG_INTER);
5146 %}
5147
5148 // Float Register
5149 // Float register operands
5150 operand vRegF()
5151 %{
5152 constraint(ALLOC_IN_RC(float_reg));
5153 match(RegF);
5154
5155 op_cost(0);
5156 format %{ %}
5157 interface(REG_INTER);
5158 %}
5159
5160 // Double Register
5161 // Double register operands
5162 operand vRegD()
5163 %{
5164 constraint(ALLOC_IN_RC(double_reg));
5165 match(RegD);
5166
5167 op_cost(0);
5168 format %{ %}
5169 interface(REG_INTER);
5170 %}
5171
5172 // Generic vector class. This will be used for
5173 // all vector operands, including NEON and SVE.
5174 operand vReg()
5175 %{
5176 constraint(ALLOC_IN_RC(dynamic));
5177 match(VecA);
5178 match(VecD);
5179 match(VecX);
5180
5181 op_cost(0);
5182 format %{ %}
5183 interface(REG_INTER);
5184 %}
5185
5186 operand vecA()
5187 %{
5188 constraint(ALLOC_IN_RC(vectora_reg));
5189 match(VecA);
5190
5191 op_cost(0);
5192 format %{ %}
5193 interface(REG_INTER);
5194 %}
5195
5196 operand vecD()
5197 %{
5198 constraint(ALLOC_IN_RC(vectord_reg));
5199 match(VecD);
5200
5201 op_cost(0);
5202 format %{ %}
5203 interface(REG_INTER);
5204 %}
5205
5206 operand vecX()
5207 %{
5208 constraint(ALLOC_IN_RC(vectorx_reg));
5209 match(VecX);
5210
5211 op_cost(0);
5212 format %{ %}
5213 interface(REG_INTER);
5214 %}
5215
5216 operand vRegD_V0()
5217 %{
5218 constraint(ALLOC_IN_RC(v0_reg));
5219 match(RegD);
5220 op_cost(0);
5221 format %{ %}
5222 interface(REG_INTER);
5223 %}
5224
5225 operand vRegD_V1()
5226 %{
5227 constraint(ALLOC_IN_RC(v1_reg));
5228 match(RegD);
5229 op_cost(0);
5230 format %{ %}
5231 interface(REG_INTER);
5232 %}
5233
5234 operand vRegD_V2()
5235 %{
5236 constraint(ALLOC_IN_RC(v2_reg));
5237 match(RegD);
5238 op_cost(0);
5239 format %{ %}
5240 interface(REG_INTER);
5241 %}
5242
5243 operand vRegD_V3()
5244 %{
5245 constraint(ALLOC_IN_RC(v3_reg));
5246 match(RegD);
5247 op_cost(0);
5248 format %{ %}
5249 interface(REG_INTER);
5250 %}
5251
5252 operand vRegD_V4()
5253 %{
5254 constraint(ALLOC_IN_RC(v4_reg));
5255 match(RegD);
5256 op_cost(0);
5257 format %{ %}
5258 interface(REG_INTER);
5259 %}
5260
5261 operand vRegD_V5()
5262 %{
5263 constraint(ALLOC_IN_RC(v5_reg));
5264 match(RegD);
5265 op_cost(0);
5266 format %{ %}
5267 interface(REG_INTER);
5268 %}
5269
5270 operand vRegD_V6()
5271 %{
5272 constraint(ALLOC_IN_RC(v6_reg));
5273 match(RegD);
5274 op_cost(0);
5275 format %{ %}
5276 interface(REG_INTER);
5277 %}
5278
5279 operand vRegD_V7()
5280 %{
5281 constraint(ALLOC_IN_RC(v7_reg));
5282 match(RegD);
5283 op_cost(0);
5284 format %{ %}
5285 interface(REG_INTER);
5286 %}
5287
5288 operand vRegD_V8()
5289 %{
5290 constraint(ALLOC_IN_RC(v8_reg));
5291 match(RegD);
5292 op_cost(0);
5293 format %{ %}
5294 interface(REG_INTER);
5295 %}
5296
5297 operand vRegD_V9()
5298 %{
5299 constraint(ALLOC_IN_RC(v9_reg));
5300 match(RegD);
5301 op_cost(0);
5302 format %{ %}
5303 interface(REG_INTER);
5304 %}
5305
5306 operand vRegD_V10()
5307 %{
5308 constraint(ALLOC_IN_RC(v10_reg));
5309 match(RegD);
5310 op_cost(0);
5311 format %{ %}
5312 interface(REG_INTER);
5313 %}
5314
5315 operand vRegD_V11()
5316 %{
5317 constraint(ALLOC_IN_RC(v11_reg));
5318 match(RegD);
5319 op_cost(0);
5320 format %{ %}
5321 interface(REG_INTER);
5322 %}
5323
5324 operand vRegD_V12()
5325 %{
5326 constraint(ALLOC_IN_RC(v12_reg));
5327 match(RegD);
5328 op_cost(0);
5329 format %{ %}
5330 interface(REG_INTER);
5331 %}
5332
5333 operand vRegD_V13()
5334 %{
5335 constraint(ALLOC_IN_RC(v13_reg));
5336 match(RegD);
5337 op_cost(0);
5338 format %{ %}
5339 interface(REG_INTER);
5340 %}
5341
5342 operand vRegD_V14()
5343 %{
5344 constraint(ALLOC_IN_RC(v14_reg));
5345 match(RegD);
5346 op_cost(0);
5347 format %{ %}
5348 interface(REG_INTER);
5349 %}
5350
5351 operand vRegD_V15()
5352 %{
5353 constraint(ALLOC_IN_RC(v15_reg));
5354 match(RegD);
5355 op_cost(0);
5356 format %{ %}
5357 interface(REG_INTER);
5358 %}
5359
5360 operand vRegD_V16()
5361 %{
5362 constraint(ALLOC_IN_RC(v16_reg));
5363 match(RegD);
5364 op_cost(0);
5365 format %{ %}
5366 interface(REG_INTER);
5367 %}
5368
5369 operand vRegD_V17()
5370 %{
5371 constraint(ALLOC_IN_RC(v17_reg));
5372 match(RegD);
5373 op_cost(0);
5374 format %{ %}
5375 interface(REG_INTER);
5376 %}
5377
5378 operand vRegD_V18()
5379 %{
5380 constraint(ALLOC_IN_RC(v18_reg));
5381 match(RegD);
5382 op_cost(0);
5383 format %{ %}
5384 interface(REG_INTER);
5385 %}
5386
5387 operand vRegD_V19()
5388 %{
5389 constraint(ALLOC_IN_RC(v19_reg));
5390 match(RegD);
5391 op_cost(0);
5392 format %{ %}
5393 interface(REG_INTER);
5394 %}
5395
5396 operand vRegD_V20()
5397 %{
5398 constraint(ALLOC_IN_RC(v20_reg));
5399 match(RegD);
5400 op_cost(0);
5401 format %{ %}
5402 interface(REG_INTER);
5403 %}
5404
5405 operand vRegD_V21()
5406 %{
5407 constraint(ALLOC_IN_RC(v21_reg));
5408 match(RegD);
5409 op_cost(0);
5410 format %{ %}
5411 interface(REG_INTER);
5412 %}
5413
5414 operand vRegD_V22()
5415 %{
5416 constraint(ALLOC_IN_RC(v22_reg));
5417 match(RegD);
5418 op_cost(0);
5419 format %{ %}
5420 interface(REG_INTER);
5421 %}
5422
5423 operand vRegD_V23()
5424 %{
5425 constraint(ALLOC_IN_RC(v23_reg));
5426 match(RegD);
5427 op_cost(0);
5428 format %{ %}
5429 interface(REG_INTER);
5430 %}
5431
5432 operand vRegD_V24()
5433 %{
5434 constraint(ALLOC_IN_RC(v24_reg));
5435 match(RegD);
5436 op_cost(0);
5437 format %{ %}
5438 interface(REG_INTER);
5439 %}
5440
5441 operand vRegD_V25()
5442 %{
5443 constraint(ALLOC_IN_RC(v25_reg));
5444 match(RegD);
5445 op_cost(0);
5446 format %{ %}
5447 interface(REG_INTER);
5448 %}
5449
5450 operand vRegD_V26()
5451 %{
5452 constraint(ALLOC_IN_RC(v26_reg));
5453 match(RegD);
5454 op_cost(0);
5455 format %{ %}
5456 interface(REG_INTER);
5457 %}
5458
5459 operand vRegD_V27()
5460 %{
5461 constraint(ALLOC_IN_RC(v27_reg));
5462 match(RegD);
5463 op_cost(0);
5464 format %{ %}
5465 interface(REG_INTER);
5466 %}
5467
5468 operand vRegD_V28()
5469 %{
5470 constraint(ALLOC_IN_RC(v28_reg));
5471 match(RegD);
5472 op_cost(0);
5473 format %{ %}
5474 interface(REG_INTER);
5475 %}
5476
5477 operand vRegD_V29()
5478 %{
5479 constraint(ALLOC_IN_RC(v29_reg));
5480 match(RegD);
5481 op_cost(0);
5482 format %{ %}
5483 interface(REG_INTER);
5484 %}
5485
5486 operand vRegD_V30()
5487 %{
5488 constraint(ALLOC_IN_RC(v30_reg));
5489 match(RegD);
5490 op_cost(0);
5491 format %{ %}
5492 interface(REG_INTER);
5493 %}
5494
5495 operand vRegD_V31()
5496 %{
5497 constraint(ALLOC_IN_RC(v31_reg));
5498 match(RegD);
5499 op_cost(0);
5500 format %{ %}
5501 interface(REG_INTER);
5502 %}
5503
5504 operand pReg()
5505 %{
5506 constraint(ALLOC_IN_RC(pr_reg));
5507 match(RegVectMask);
5508 match(pRegGov);
5509 op_cost(0);
5510 format %{ %}
5511 interface(REG_INTER);
5512 %}
5513
5514 operand pRegGov()
5515 %{
5516 constraint(ALLOC_IN_RC(gov_pr));
5517 match(RegVectMask);
5518 op_cost(0);
5519 format %{ %}
5520 interface(REG_INTER);
5521 %}
5522
5523 operand pRegGov_P0()
5524 %{
5525 constraint(ALLOC_IN_RC(p0_reg));
5526 match(RegVectMask);
5527 op_cost(0);
5528 format %{ %}
5529 interface(REG_INTER);
5530 %}
5531
5532 operand pRegGov_P1()
5533 %{
5534 constraint(ALLOC_IN_RC(p1_reg));
5535 match(RegVectMask);
5536 op_cost(0);
5537 format %{ %}
5538 interface(REG_INTER);
5539 %}
5540
5541 // Flags register, used as output of signed compare instructions
5542
5543 // note that on AArch64 we also use this register as the output for
5544 // for floating point compare instructions (CmpF CmpD). this ensures
5545 // that ordered inequality tests use GT, GE, LT or LE none of which
5546 // pass through cases where the result is unordered i.e. one or both
5547 // inputs to the compare is a NaN. this means that the ideal code can
5548 // replace e.g. a GT with an LE and not end up capturing the NaN case
5549 // (where the comparison should always fail). EQ and NE tests are
5550 // always generated in ideal code so that unordered folds into the NE
5551 // case, matching the behaviour of AArch64 NE.
5552 //
5553 // This differs from x86 where the outputs of FP compares use a
5554 // special FP flags registers and where compares based on this
5555 // register are distinguished into ordered inequalities (cmpOpUCF) and
5556 // EQ/NEQ tests (cmpOpUCF2). x86 has to special case the latter tests
5557 // to explicitly handle the unordered case in branches. x86 also has
5558 // to include extra CMoveX rules to accept a cmpOpUCF input.
5559
5560 operand rFlagsReg()
5561 %{
5562 constraint(ALLOC_IN_RC(int_flags));
5563 match(RegFlags);
5564
5565 op_cost(0);
5566 format %{ "RFLAGS" %}
5567 interface(REG_INTER);
5568 %}
5569
5570 // Flags register, used as output of unsigned compare instructions
5571 operand rFlagsRegU()
5572 %{
5573 constraint(ALLOC_IN_RC(int_flags));
5574 match(RegFlags);
5575
5576 op_cost(0);
5577 format %{ "RFLAGSU" %}
5578 interface(REG_INTER);
5579 %}
5580
5581 // Special Registers
5582
5583 // Method Register
5584 operand inline_cache_RegP(iRegP reg)
5585 %{
5586 constraint(ALLOC_IN_RC(method_reg)); // inline_cache_reg
5587 match(reg);
5588 match(iRegPNoSp);
5589 op_cost(0);
5590 format %{ %}
5591 interface(REG_INTER);
5592 %}
5593
5594 // Thread Register
5595 operand thread_RegP(iRegP reg)
5596 %{
5597 constraint(ALLOC_IN_RC(thread_reg)); // link_reg
5598 match(reg);
5599 op_cost(0);
5600 format %{ %}
5601 interface(REG_INTER);
5602 %}
5603
5604 operand lr_RegP(iRegP reg)
5605 %{
5606 constraint(ALLOC_IN_RC(lr_reg)); // link_reg
5607 match(reg);
5608 op_cost(0);
5609 format %{ %}
5610 interface(REG_INTER);
5611 %}
5612
5613 //----------Memory Operands----------------------------------------------------
5614
5615 operand indirect(iRegP reg)
5616 %{
5617 constraint(ALLOC_IN_RC(ptr_reg));
5618 match(reg);
5619 op_cost(0);
5620 format %{ "[$reg]" %}
5621 interface(MEMORY_INTER) %{
5622 base($reg);
5623 index(0xffffffff);
5624 scale(0x0);
5625 disp(0x0);
5626 %}
5627 %}
5628
5629 operand indIndexScaledI2L(iRegP reg, iRegI ireg, immIScale scale)
5630 %{
5631 constraint(ALLOC_IN_RC(ptr_reg));
5632 predicate(size_fits_all_mem_uses(n->as_AddP(), n->in(AddPNode::Offset)->in(2)->get_int()));
5633 match(AddP reg (LShiftL (ConvI2L ireg) scale));
5634 op_cost(0);
5635 format %{ "$reg, $ireg sxtw($scale), 0, I2L" %}
5636 interface(MEMORY_INTER) %{
5637 base($reg);
5638 index($ireg);
5639 scale($scale);
5640 disp(0x0);
5641 %}
5642 %}
5643
5644 operand indIndexScaled(iRegP reg, iRegL lreg, immIScale scale)
5645 %{
5646 constraint(ALLOC_IN_RC(ptr_reg));
5647 predicate(size_fits_all_mem_uses(n->as_AddP(), n->in(AddPNode::Offset)->in(2)->get_int()));
5648 match(AddP reg (LShiftL lreg scale));
5649 op_cost(0);
5650 format %{ "$reg, $lreg lsl($scale)" %}
5651 interface(MEMORY_INTER) %{
5652 base($reg);
5653 index($lreg);
5654 scale($scale);
5655 disp(0x0);
5656 %}
5657 %}
5658
5659 operand indIndexI2L(iRegP reg, iRegI ireg)
5660 %{
5661 constraint(ALLOC_IN_RC(ptr_reg));
5662 match(AddP reg (ConvI2L ireg));
5663 op_cost(0);
5664 format %{ "$reg, $ireg, 0, I2L" %}
5665 interface(MEMORY_INTER) %{
5666 base($reg);
5667 index($ireg);
5668 scale(0x0);
5669 disp(0x0);
5670 %}
5671 %}
5672
5673 operand indIndex(iRegP reg, iRegL lreg)
5674 %{
5675 constraint(ALLOC_IN_RC(ptr_reg));
5676 match(AddP reg lreg);
5677 op_cost(0);
5678 format %{ "$reg, $lreg" %}
5679 interface(MEMORY_INTER) %{
5680 base($reg);
5681 index($lreg);
5682 scale(0x0);
5683 disp(0x0);
5684 %}
5685 %}
5686
5687 operand indOffI(iRegP reg, immIOffset off)
5688 %{
5689 constraint(ALLOC_IN_RC(ptr_reg));
5690 match(AddP reg off);
5691 op_cost(0);
5692 format %{ "[$reg, $off]" %}
5693 interface(MEMORY_INTER) %{
5694 base($reg);
5695 index(0xffffffff);
5696 scale(0x0);
5697 disp($off);
5698 %}
5699 %}
5700
5701 operand indOffI1(iRegP reg, immIOffset1 off)
5702 %{
5703 constraint(ALLOC_IN_RC(ptr_reg));
5704 match(AddP reg off);
5705 op_cost(0);
5706 format %{ "[$reg, $off]" %}
5707 interface(MEMORY_INTER) %{
5708 base($reg);
5709 index(0xffffffff);
5710 scale(0x0);
5711 disp($off);
5712 %}
5713 %}
5714
5715 operand indOffI2(iRegP reg, immIOffset2 off)
5716 %{
5717 constraint(ALLOC_IN_RC(ptr_reg));
5718 match(AddP reg off);
5719 op_cost(0);
5720 format %{ "[$reg, $off]" %}
5721 interface(MEMORY_INTER) %{
5722 base($reg);
5723 index(0xffffffff);
5724 scale(0x0);
5725 disp($off);
5726 %}
5727 %}
5728
5729 operand indOffI4(iRegP reg, immIOffset4 off)
5730 %{
5731 constraint(ALLOC_IN_RC(ptr_reg));
5732 match(AddP reg off);
5733 op_cost(0);
5734 format %{ "[$reg, $off]" %}
5735 interface(MEMORY_INTER) %{
5736 base($reg);
5737 index(0xffffffff);
5738 scale(0x0);
5739 disp($off);
5740 %}
5741 %}
5742
5743 operand indOffI8(iRegP reg, immIOffset8 off)
5744 %{
5745 constraint(ALLOC_IN_RC(ptr_reg));
5746 match(AddP reg off);
5747 op_cost(0);
5748 format %{ "[$reg, $off]" %}
5749 interface(MEMORY_INTER) %{
5750 base($reg);
5751 index(0xffffffff);
5752 scale(0x0);
5753 disp($off);
5754 %}
5755 %}
5756
5757 operand indOffI16(iRegP reg, immIOffset16 off)
5758 %{
5759 constraint(ALLOC_IN_RC(ptr_reg));
5760 match(AddP reg off);
5761 op_cost(0);
5762 format %{ "[$reg, $off]" %}
5763 interface(MEMORY_INTER) %{
5764 base($reg);
5765 index(0xffffffff);
5766 scale(0x0);
5767 disp($off);
5768 %}
5769 %}
5770
5771 operand indOffL(iRegP reg, immLoffset off)
5772 %{
5773 constraint(ALLOC_IN_RC(ptr_reg));
5774 match(AddP reg off);
5775 op_cost(0);
5776 format %{ "[$reg, $off]" %}
5777 interface(MEMORY_INTER) %{
5778 base($reg);
5779 index(0xffffffff);
5780 scale(0x0);
5781 disp($off);
5782 %}
5783 %}
5784
5785 operand indOffL1(iRegP reg, immLoffset1 off)
5786 %{
5787 constraint(ALLOC_IN_RC(ptr_reg));
5788 match(AddP reg off);
5789 op_cost(0);
5790 format %{ "[$reg, $off]" %}
5791 interface(MEMORY_INTER) %{
5792 base($reg);
5793 index(0xffffffff);
5794 scale(0x0);
5795 disp($off);
5796 %}
5797 %}
5798
5799 operand indOffL2(iRegP reg, immLoffset2 off)
5800 %{
5801 constraint(ALLOC_IN_RC(ptr_reg));
5802 match(AddP reg off);
5803 op_cost(0);
5804 format %{ "[$reg, $off]" %}
5805 interface(MEMORY_INTER) %{
5806 base($reg);
5807 index(0xffffffff);
5808 scale(0x0);
5809 disp($off);
5810 %}
5811 %}
5812
5813 operand indOffL4(iRegP reg, immLoffset4 off)
5814 %{
5815 constraint(ALLOC_IN_RC(ptr_reg));
5816 match(AddP reg off);
5817 op_cost(0);
5818 format %{ "[$reg, $off]" %}
5819 interface(MEMORY_INTER) %{
5820 base($reg);
5821 index(0xffffffff);
5822 scale(0x0);
5823 disp($off);
5824 %}
5825 %}
5826
5827 operand indOffL8(iRegP reg, immLoffset8 off)
5828 %{
5829 constraint(ALLOC_IN_RC(ptr_reg));
5830 match(AddP reg off);
5831 op_cost(0);
5832 format %{ "[$reg, $off]" %}
5833 interface(MEMORY_INTER) %{
5834 base($reg);
5835 index(0xffffffff);
5836 scale(0x0);
5837 disp($off);
5838 %}
5839 %}
5840
5841 operand indOffL16(iRegP reg, immLoffset16 off)
5842 %{
5843 constraint(ALLOC_IN_RC(ptr_reg));
5844 match(AddP reg off);
5845 op_cost(0);
5846 format %{ "[$reg, $off]" %}
5847 interface(MEMORY_INTER) %{
5848 base($reg);
5849 index(0xffffffff);
5850 scale(0x0);
5851 disp($off);
5852 %}
5853 %}
5854
5855 operand indirectN(iRegN reg)
5856 %{
5857 predicate(CompressedOops::shift() == 0);
5858 constraint(ALLOC_IN_RC(ptr_reg));
5859 match(DecodeN reg);
5860 op_cost(0);
5861 format %{ "[$reg]\t# narrow" %}
5862 interface(MEMORY_INTER) %{
5863 base($reg);
5864 index(0xffffffff);
5865 scale(0x0);
5866 disp(0x0);
5867 %}
5868 %}
5869
5870 operand indIndexScaledI2LN(iRegN reg, iRegI ireg, immIScale scale)
5871 %{
5872 predicate(CompressedOops::shift() == 0 && size_fits_all_mem_uses(n->as_AddP(), n->in(AddPNode::Offset)->in(2)->get_int()));
5873 constraint(ALLOC_IN_RC(ptr_reg));
5874 match(AddP (DecodeN reg) (LShiftL (ConvI2L ireg) scale));
5875 op_cost(0);
5876 format %{ "$reg, $ireg sxtw($scale), 0, I2L\t# narrow" %}
5877 interface(MEMORY_INTER) %{
5878 base($reg);
5879 index($ireg);
5880 scale($scale);
5881 disp(0x0);
5882 %}
5883 %}
5884
5885 operand indIndexScaledN(iRegN reg, iRegL lreg, immIScale scale)
5886 %{
5887 predicate(CompressedOops::shift() == 0 && size_fits_all_mem_uses(n->as_AddP(), n->in(AddPNode::Offset)->in(2)->get_int()));
5888 constraint(ALLOC_IN_RC(ptr_reg));
5889 match(AddP (DecodeN reg) (LShiftL lreg scale));
5890 op_cost(0);
5891 format %{ "$reg, $lreg lsl($scale)\t# narrow" %}
5892 interface(MEMORY_INTER) %{
5893 base($reg);
5894 index($lreg);
5895 scale($scale);
5896 disp(0x0);
5897 %}
5898 %}
5899
5900 operand indIndexI2LN(iRegN reg, iRegI ireg)
5901 %{
5902 predicate(CompressedOops::shift() == 0);
5903 constraint(ALLOC_IN_RC(ptr_reg));
5904 match(AddP (DecodeN reg) (ConvI2L ireg));
5905 op_cost(0);
5906 format %{ "$reg, $ireg, 0, I2L\t# narrow" %}
5907 interface(MEMORY_INTER) %{
5908 base($reg);
5909 index($ireg);
5910 scale(0x0);
5911 disp(0x0);
5912 %}
5913 %}
5914
5915 operand indIndexN(iRegN reg, iRegL lreg)
5916 %{
5917 predicate(CompressedOops::shift() == 0);
5918 constraint(ALLOC_IN_RC(ptr_reg));
5919 match(AddP (DecodeN reg) lreg);
5920 op_cost(0);
5921 format %{ "$reg, $lreg\t# narrow" %}
5922 interface(MEMORY_INTER) %{
5923 base($reg);
5924 index($lreg);
5925 scale(0x0);
5926 disp(0x0);
5927 %}
5928 %}
5929
5930 operand indOffIN(iRegN reg, immIOffset off)
5931 %{
5932 predicate(CompressedOops::shift() == 0);
5933 constraint(ALLOC_IN_RC(ptr_reg));
5934 match(AddP (DecodeN reg) off);
5935 op_cost(0);
5936 format %{ "[$reg, $off]\t# narrow" %}
5937 interface(MEMORY_INTER) %{
5938 base($reg);
5939 index(0xffffffff);
5940 scale(0x0);
5941 disp($off);
5942 %}
5943 %}
5944
5945 operand indOffLN(iRegN reg, immLoffset off)
5946 %{
5947 predicate(CompressedOops::shift() == 0);
5948 constraint(ALLOC_IN_RC(ptr_reg));
5949 match(AddP (DecodeN reg) off);
5950 op_cost(0);
5951 format %{ "[$reg, $off]\t# narrow" %}
5952 interface(MEMORY_INTER) %{
5953 base($reg);
5954 index(0xffffffff);
5955 scale(0x0);
5956 disp($off);
5957 %}
5958 %}
5959
5960
5961
5962 // AArch64 opto stubs need to write to the pc slot in the thread anchor
5963 operand thread_anchor_pc(thread_RegP reg, immL_pc_off off)
5964 %{
5965 constraint(ALLOC_IN_RC(ptr_reg));
5966 match(AddP reg off);
5967 op_cost(0);
5968 format %{ "[$reg, $off]" %}
5969 interface(MEMORY_INTER) %{
5970 base($reg);
5971 index(0xffffffff);
5972 scale(0x0);
5973 disp($off);
5974 %}
5975 %}
5976
5977 //----------Special Memory Operands--------------------------------------------
5978 // Stack Slot Operand - This operand is used for loading and storing temporary
5979 // values on the stack where a match requires a value to
5980 // flow through memory.
5981 operand stackSlotP(sRegP reg)
5982 %{
5983 constraint(ALLOC_IN_RC(stack_slots));
5984 op_cost(100);
5985 // No match rule because this operand is only generated in matching
5986 // match(RegP);
5987 format %{ "[$reg]" %}
5988 interface(MEMORY_INTER) %{
5989 base(0x1e); // RSP
5990 index(0x0); // No Index
5991 scale(0x0); // No Scale
5992 disp($reg); // Stack Offset
5993 %}
5994 %}
5995
5996 operand stackSlotI(sRegI reg)
5997 %{
5998 constraint(ALLOC_IN_RC(stack_slots));
5999 // No match rule because this operand is only generated in matching
6000 // match(RegI);
6001 format %{ "[$reg]" %}
6002 interface(MEMORY_INTER) %{
6003 base(0x1e); // RSP
6004 index(0x0); // No Index
6005 scale(0x0); // No Scale
6006 disp($reg); // Stack Offset
6007 %}
6008 %}
6009
6010 operand stackSlotF(sRegF reg)
6011 %{
6012 constraint(ALLOC_IN_RC(stack_slots));
6013 // No match rule because this operand is only generated in matching
6014 // match(RegF);
6015 format %{ "[$reg]" %}
6016 interface(MEMORY_INTER) %{
6017 base(0x1e); // RSP
6018 index(0x0); // No Index
6019 scale(0x0); // No Scale
6020 disp($reg); // Stack Offset
6021 %}
6022 %}
6023
6024 operand stackSlotD(sRegD reg)
6025 %{
6026 constraint(ALLOC_IN_RC(stack_slots));
6027 // No match rule because this operand is only generated in matching
6028 // match(RegD);
6029 format %{ "[$reg]" %}
6030 interface(MEMORY_INTER) %{
6031 base(0x1e); // RSP
6032 index(0x0); // No Index
6033 scale(0x0); // No Scale
6034 disp($reg); // Stack Offset
6035 %}
6036 %}
6037
6038 operand stackSlotL(sRegL reg)
6039 %{
6040 constraint(ALLOC_IN_RC(stack_slots));
6041 // No match rule because this operand is only generated in matching
6042 // match(RegL);
6043 format %{ "[$reg]" %}
6044 interface(MEMORY_INTER) %{
6045 base(0x1e); // RSP
6046 index(0x0); // No Index
6047 scale(0x0); // No Scale
6048 disp($reg); // Stack Offset
6049 %}
6050 %}
6051
6052 // Operands for expressing Control Flow
6053 // NOTE: Label is a predefined operand which should not be redefined in
6054 // the AD file. It is generically handled within the ADLC.
6055
6056 //----------Conditional Branch Operands----------------------------------------
6057 // Comparison Op - This is the operation of the comparison, and is limited to
6058 // the following set of codes:
6059 // L (<), LE (<=), G (>), GE (>=), E (==), NE (!=)
6060 //
6061 // Other attributes of the comparison, such as unsignedness, are specified
6062 // by the comparison instruction that sets a condition code flags register.
6063 // That result is represented by a flags operand whose subtype is appropriate
6064 // to the unsignedness (etc.) of the comparison.
6065 //
6066 // Later, the instruction which matches both the Comparison Op (a Bool) and
6067 // the flags (produced by the Cmp) specifies the coding of the comparison op
6068 // by matching a specific subtype of Bool operand below, such as cmpOpU.
6069
6070 // used for signed integral comparisons and fp comparisons
6071
6072 operand cmpOp()
6073 %{
6074 match(Bool);
6075
6076 format %{ "" %}
6077 interface(COND_INTER) %{
6078 equal(0x0, "eq");
6079 not_equal(0x1, "ne");
6080 less(0xb, "lt");
6081 greater_equal(0xa, "ge");
6082 less_equal(0xd, "le");
6083 greater(0xc, "gt");
6084 overflow(0x6, "vs");
6085 no_overflow(0x7, "vc");
6086 %}
6087 %}
6088
6089 // used for unsigned integral comparisons
6090
6091 operand cmpOpU()
6092 %{
6093 match(Bool);
6094
6095 format %{ "" %}
6096 interface(COND_INTER) %{
6097 equal(0x0, "eq");
6098 not_equal(0x1, "ne");
6099 less(0x3, "lo");
6100 greater_equal(0x2, "hs");
6101 less_equal(0x9, "ls");
6102 greater(0x8, "hi");
6103 overflow(0x6, "vs");
6104 no_overflow(0x7, "vc");
6105 %}
6106 %}
6107
6108 // used for certain integral comparisons which can be
6109 // converted to cbxx or tbxx instructions
6110
6111 operand cmpOpEqNe()
6112 %{
6113 match(Bool);
6114 op_cost(0);
6115 predicate(n->as_Bool()->_test._test == BoolTest::ne
6116 || n->as_Bool()->_test._test == BoolTest::eq);
6117
6118 format %{ "" %}
6119 interface(COND_INTER) %{
6120 equal(0x0, "eq");
6121 not_equal(0x1, "ne");
6122 less(0xb, "lt");
6123 greater_equal(0xa, "ge");
6124 less_equal(0xd, "le");
6125 greater(0xc, "gt");
6126 overflow(0x6, "vs");
6127 no_overflow(0x7, "vc");
6128 %}
6129 %}
6130
6131 // used for certain integral comparisons which can be
6132 // converted to cbxx or tbxx instructions
6133
6134 operand cmpOpLtGe()
6135 %{
6136 match(Bool);
6137 op_cost(0);
6138
6139 predicate(n->as_Bool()->_test._test == BoolTest::lt
6140 || n->as_Bool()->_test._test == BoolTest::ge);
6141
6142 format %{ "" %}
6143 interface(COND_INTER) %{
6144 equal(0x0, "eq");
6145 not_equal(0x1, "ne");
6146 less(0xb, "lt");
6147 greater_equal(0xa, "ge");
6148 less_equal(0xd, "le");
6149 greater(0xc, "gt");
6150 overflow(0x6, "vs");
6151 no_overflow(0x7, "vc");
6152 %}
6153 %}
6154
6155 // used for certain unsigned integral comparisons which can be
6156 // converted to cbxx or tbxx instructions
6157
6158 operand cmpOpUEqNeLtGe()
6159 %{
6160 match(Bool);
6161 op_cost(0);
6162
6163 predicate(n->as_Bool()->_test._test == BoolTest::eq
6164 || n->as_Bool()->_test._test == BoolTest::ne
6165 || n->as_Bool()->_test._test == BoolTest::lt
6166 || n->as_Bool()->_test._test == BoolTest::ge);
6167
6168 format %{ "" %}
6169 interface(COND_INTER) %{
6170 equal(0x0, "eq");
6171 not_equal(0x1, "ne");
6172 less(0xb, "lt");
6173 greater_equal(0xa, "ge");
6174 less_equal(0xd, "le");
6175 greater(0xc, "gt");
6176 overflow(0x6, "vs");
6177 no_overflow(0x7, "vc");
6178 %}
6179 %}
6180
6181 // Special operand allowing long args to int ops to be truncated for free
6182
6183 operand iRegL2I(iRegL reg) %{
6184
6185 op_cost(0);
6186
6187 match(ConvL2I reg);
6188
6189 format %{ "l2i($reg)" %}
6190
6191 interface(REG_INTER)
6192 %}
6193
6194 opclass vmem2(indirect, indIndex, indOffI2, indOffL2);
6195 opclass vmem4(indirect, indIndex, indOffI4, indOffL4);
6196 opclass vmem8(indirect, indIndex, indOffI8, indOffL8);
6197 opclass vmem16(indirect, indIndex, indOffI16, indOffL16);
6198
6199 //----------OPERAND CLASSES----------------------------------------------------
6200 // Operand Classes are groups of operands that are used as to simplify
6201 // instruction definitions by not requiring the AD writer to specify
6202 // separate instructions for every form of operand when the
6203 // instruction accepts multiple operand types with the same basic
6204 // encoding and format. The classic case of this is memory operands.
6205
6206 // memory is used to define read/write location for load/store
6207 // instruction defs. we can turn a memory op into an Address
6208
6209 opclass memory1(indirect, indIndexScaled, indIndexScaledI2L, indIndexI2L, indIndex, indOffI1, indOffL1,
6210 indirectN, indIndexScaledN, indIndexScaledI2LN, indIndexI2LN, indIndexN);
6211
6212 opclass memory2(indirect, indIndexScaled, indIndexScaledI2L, indIndexI2L, indIndex, indOffI2, indOffL2,
6213 indirectN, indIndexScaledN, indIndexScaledI2LN, indIndexI2LN, indIndexN);
6214
6215 opclass memory4(indirect, indIndexScaled, indIndexScaledI2L, indIndexI2L, indIndex, indOffI4, indOffL4,
6216 indirectN, indIndexScaledN, indIndexScaledI2LN, indIndexI2LN, indIndexN, indOffIN, indOffLN);
6217
6218 opclass memory8(indirect, indIndexScaled, indIndexScaledI2L, indIndexI2L, indIndex, indOffI8, indOffL8,
6219 indirectN, indIndexScaledN, indIndexScaledI2LN, indIndexI2LN, indIndexN, indOffIN, indOffLN);
6220
6221 // All of the memory operands. For the pipeline description.
6222 opclass memory(indirect, indIndexScaled, indIndexScaledI2L, indIndexI2L, indIndex,
6223 indOffI1, indOffL1, indOffI2, indOffL2, indOffI4, indOffL4, indOffI8, indOffL8,
6224 indirectN, indIndexScaledN, indIndexScaledI2LN, indIndexI2LN, indIndexN, indOffIN, indOffLN);
6225
6226
6227 // iRegIorL2I is used for src inputs in rules for 32 bit int (I)
6228 // operations. it allows the src to be either an iRegI or a (ConvL2I
6229 // iRegL). in the latter case the l2i normally planted for a ConvL2I
6230 // can be elided because the 32-bit instruction will just employ the
6231 // lower 32 bits anyway.
6232 //
6233 // n.b. this does not elide all L2I conversions. if the truncated
6234 // value is consumed by more than one operation then the ConvL2I
6235 // cannot be bundled into the consuming nodes so an l2i gets planted
6236 // (actually a movw $dst $src) and the downstream instructions consume
6237 // the result of the l2i as an iRegI input. That's a shame since the
6238 // movw is actually redundant but its not too costly.
6239
6240 opclass iRegIorL2I(iRegI, iRegL2I);
6241
6242 //----------PIPELINE-----------------------------------------------------------
6243 // Rules which define the behavior of the target architectures pipeline.
6244
6245 // For specific pipelines, eg A53, define the stages of that pipeline
6246 //pipe_desc(ISS, EX1, EX2, WR);
6247 #define ISS S0
6248 #define EX1 S1
6249 #define EX2 S2
6250 #define WR S3
6251
6252 // Integer ALU reg operation
6253 pipeline %{
6254
6255 attributes %{
6256 // ARM instructions are of fixed length
6257 fixed_size_instructions; // Fixed size instructions TODO does
6258 max_instructions_per_bundle = 4; // A53 = 2, A57 = 4
6259 // ARM instructions come in 32-bit word units
6260 instruction_unit_size = 4; // An instruction is 4 bytes long
6261 instruction_fetch_unit_size = 64; // The processor fetches one line
6262 instruction_fetch_units = 1; // of 64 bytes
6263
6264 // List of nop instructions
6265 nops( MachNop );
6266 %}
6267
6268 // We don't use an actual pipeline model so don't care about resources
6269 // or description. we do use pipeline classes to introduce fixed
6270 // latencies
6271
6272 //----------RESOURCES----------------------------------------------------------
6273 // Resources are the functional units available to the machine
6274
6275 resources( INS0, INS1, INS01 = INS0 | INS1,
6276 ALU0, ALU1, ALU = ALU0 | ALU1,
6277 MAC,
6278 DIV,
6279 BRANCH,
6280 LDST,
6281 NEON_FP);
6282
6283 //----------PIPELINE DESCRIPTION-----------------------------------------------
6284 // Pipeline Description specifies the stages in the machine's pipeline
6285
6286 // Define the pipeline as a generic 6 stage pipeline
6287 pipe_desc(S0, S1, S2, S3, S4, S5);
6288
6289 //----------PIPELINE CLASSES---------------------------------------------------
6290 // Pipeline Classes describe the stages in which input and output are
6291 // referenced by the hardware pipeline.
6292
6293 pipe_class fp_dop_reg_reg_s(vRegF dst, vRegF src1, vRegF src2)
6294 %{
6295 single_instruction;
6296 src1 : S1(read);
6297 src2 : S2(read);
6298 dst : S5(write);
6299 INS01 : ISS;
6300 NEON_FP : S5;
6301 %}
6302
6303 pipe_class fp_dop_reg_reg_d(vRegD dst, vRegD src1, vRegD src2)
6304 %{
6305 single_instruction;
6306 src1 : S1(read);
6307 src2 : S2(read);
6308 dst : S5(write);
6309 INS01 : ISS;
6310 NEON_FP : S5;
6311 %}
6312
6313 pipe_class fp_uop_s(vRegF dst, vRegF src)
6314 %{
6315 single_instruction;
6316 src : S1(read);
6317 dst : S5(write);
6318 INS01 : ISS;
6319 NEON_FP : S5;
6320 %}
6321
6322 pipe_class fp_uop_d(vRegD dst, vRegD src)
6323 %{
6324 single_instruction;
6325 src : S1(read);
6326 dst : S5(write);
6327 INS01 : ISS;
6328 NEON_FP : S5;
6329 %}
6330
6331 pipe_class fp_d2f(vRegF dst, vRegD src)
6332 %{
6333 single_instruction;
6334 src : S1(read);
6335 dst : S5(write);
6336 INS01 : ISS;
6337 NEON_FP : S5;
6338 %}
6339
6340 pipe_class fp_f2d(vRegD dst, vRegF src)
6341 %{
6342 single_instruction;
6343 src : S1(read);
6344 dst : S5(write);
6345 INS01 : ISS;
6346 NEON_FP : S5;
6347 %}
6348
6349 pipe_class fp_f2i(iRegINoSp dst, vRegF src)
6350 %{
6351 single_instruction;
6352 src : S1(read);
6353 dst : S5(write);
6354 INS01 : ISS;
6355 NEON_FP : S5;
6356 %}
6357
6358 pipe_class fp_f2l(iRegLNoSp dst, vRegF src)
6359 %{
6360 single_instruction;
6361 src : S1(read);
6362 dst : S5(write);
6363 INS01 : ISS;
6364 NEON_FP : S5;
6365 %}
6366
6367 pipe_class fp_i2f(vRegF dst, iRegIorL2I src)
6368 %{
6369 single_instruction;
6370 src : S1(read);
6371 dst : S5(write);
6372 INS01 : ISS;
6373 NEON_FP : S5;
6374 %}
6375
6376 pipe_class fp_l2f(vRegF dst, iRegL src)
6377 %{
6378 single_instruction;
6379 src : S1(read);
6380 dst : S5(write);
6381 INS01 : ISS;
6382 NEON_FP : S5;
6383 %}
6384
6385 pipe_class fp_d2i(iRegINoSp dst, vRegD src)
6386 %{
6387 single_instruction;
6388 src : S1(read);
6389 dst : S5(write);
6390 INS01 : ISS;
6391 NEON_FP : S5;
6392 %}
6393
6394 pipe_class fp_d2l(iRegLNoSp dst, vRegD src)
6395 %{
6396 single_instruction;
6397 src : S1(read);
6398 dst : S5(write);
6399 INS01 : ISS;
6400 NEON_FP : S5;
6401 %}
6402
6403 pipe_class fp_i2d(vRegD dst, iRegIorL2I src)
6404 %{
6405 single_instruction;
6406 src : S1(read);
6407 dst : S5(write);
6408 INS01 : ISS;
6409 NEON_FP : S5;
6410 %}
6411
6412 pipe_class fp_l2d(vRegD dst, iRegIorL2I src)
6413 %{
6414 single_instruction;
6415 src : S1(read);
6416 dst : S5(write);
6417 INS01 : ISS;
6418 NEON_FP : S5;
6419 %}
6420
6421 pipe_class fp_div_s(vRegF dst, vRegF src1, vRegF src2)
6422 %{
6423 single_instruction;
6424 src1 : S1(read);
6425 src2 : S2(read);
6426 dst : S5(write);
6427 INS0 : ISS;
6428 NEON_FP : S5;
6429 %}
6430
6431 pipe_class fp_div_d(vRegD dst, vRegD src1, vRegD src2)
6432 %{
6433 single_instruction;
6434 src1 : S1(read);
6435 src2 : S2(read);
6436 dst : S5(write);
6437 INS0 : ISS;
6438 NEON_FP : S5;
6439 %}
6440
6441 pipe_class fp_cond_reg_reg_s(vRegF dst, vRegF src1, vRegF src2, rFlagsReg cr)
6442 %{
6443 single_instruction;
6444 cr : S1(read);
6445 src1 : S1(read);
6446 src2 : S1(read);
6447 dst : S3(write);
6448 INS01 : ISS;
6449 NEON_FP : S3;
6450 %}
6451
6452 pipe_class fp_cond_reg_reg_d(vRegD dst, vRegD src1, vRegD src2, rFlagsReg cr)
6453 %{
6454 single_instruction;
6455 cr : S1(read);
6456 src1 : S1(read);
6457 src2 : S1(read);
6458 dst : S3(write);
6459 INS01 : ISS;
6460 NEON_FP : S3;
6461 %}
6462
6463 pipe_class fp_imm_s(vRegF dst)
6464 %{
6465 single_instruction;
6466 dst : S3(write);
6467 INS01 : ISS;
6468 NEON_FP : S3;
6469 %}
6470
6471 pipe_class fp_imm_d(vRegD dst)
6472 %{
6473 single_instruction;
6474 dst : S3(write);
6475 INS01 : ISS;
6476 NEON_FP : S3;
6477 %}
6478
6479 pipe_class fp_load_constant_s(vRegF dst)
6480 %{
6481 single_instruction;
6482 dst : S4(write);
6483 INS01 : ISS;
6484 NEON_FP : S4;
6485 %}
6486
6487 pipe_class fp_load_constant_d(vRegD dst)
6488 %{
6489 single_instruction;
6490 dst : S4(write);
6491 INS01 : ISS;
6492 NEON_FP : S4;
6493 %}
6494
6495 //------- Integer ALU operations --------------------------
6496
6497 // Integer ALU reg-reg operation
6498 // Operands needed in EX1, result generated in EX2
6499 // Eg. ADD x0, x1, x2
6500 pipe_class ialu_reg_reg(iRegI dst, iRegI src1, iRegI src2)
6501 %{
6502 single_instruction;
6503 dst : EX2(write);
6504 src1 : EX1(read);
6505 src2 : EX1(read);
6506 INS01 : ISS; // Dual issue as instruction 0 or 1
6507 ALU : EX2;
6508 %}
6509
6510 // Integer ALU reg-reg operation with constant shift
6511 // Shifted register must be available in LATE_ISS instead of EX1
6512 // Eg. ADD x0, x1, x2, LSL #2
6513 pipe_class ialu_reg_reg_shift(iRegI dst, iRegI src1, iRegI src2, immI shift)
6514 %{
6515 single_instruction;
6516 dst : EX2(write);
6517 src1 : EX1(read);
6518 src2 : ISS(read);
6519 INS01 : ISS;
6520 ALU : EX2;
6521 %}
6522
6523 // Integer ALU reg operation with constant shift
6524 // Eg. LSL x0, x1, #shift
6525 pipe_class ialu_reg_shift(iRegI dst, iRegI src1)
6526 %{
6527 single_instruction;
6528 dst : EX2(write);
6529 src1 : ISS(read);
6530 INS01 : ISS;
6531 ALU : EX2;
6532 %}
6533
6534 // Integer ALU reg-reg operation with variable shift
6535 // Both operands must be available in LATE_ISS instead of EX1
6536 // Result is available in EX1 instead of EX2
6537 // Eg. LSLV x0, x1, x2
6538 pipe_class ialu_reg_reg_vshift(iRegI dst, iRegI src1, iRegI src2)
6539 %{
6540 single_instruction;
6541 dst : EX1(write);
6542 src1 : ISS(read);
6543 src2 : ISS(read);
6544 INS01 : ISS;
6545 ALU : EX1;
6546 %}
6547
6548 // Integer ALU reg-reg operation with extract
6549 // As for _vshift above, but result generated in EX2
6550 // Eg. EXTR x0, x1, x2, #N
6551 pipe_class ialu_reg_reg_extr(iRegI dst, iRegI src1, iRegI src2)
6552 %{
6553 single_instruction;
6554 dst : EX2(write);
6555 src1 : ISS(read);
6556 src2 : ISS(read);
6557 INS1 : ISS; // Can only dual issue as Instruction 1
6558 ALU : EX1;
6559 %}
6560
6561 // Integer ALU reg operation
6562 // Eg. NEG x0, x1
6563 pipe_class ialu_reg(iRegI dst, iRegI src)
6564 %{
6565 single_instruction;
6566 dst : EX2(write);
6567 src : EX1(read);
6568 INS01 : ISS;
6569 ALU : EX2;
6570 %}
6571
6572 // Integer ALU reg mmediate operation
6573 // Eg. ADD x0, x1, #N
6574 pipe_class ialu_reg_imm(iRegI dst, iRegI src1)
6575 %{
6576 single_instruction;
6577 dst : EX2(write);
6578 src1 : EX1(read);
6579 INS01 : ISS;
6580 ALU : EX2;
6581 %}
6582
6583 // Integer ALU immediate operation (no source operands)
6584 // Eg. MOV x0, #N
6585 pipe_class ialu_imm(iRegI dst)
6586 %{
6587 single_instruction;
6588 dst : EX1(write);
6589 INS01 : ISS;
6590 ALU : EX1;
6591 %}
6592
6593 //------- Compare operation -------------------------------
6594
6595 // Compare reg-reg
6596 // Eg. CMP x0, x1
6597 pipe_class icmp_reg_reg(rFlagsReg cr, iRegI op1, iRegI op2)
6598 %{
6599 single_instruction;
6600 // fixed_latency(16);
6601 cr : EX2(write);
6602 op1 : EX1(read);
6603 op2 : EX1(read);
6604 INS01 : ISS;
6605 ALU : EX2;
6606 %}
6607
6608 // Compare reg-reg
6609 // Eg. CMP x0, #N
6610 pipe_class icmp_reg_imm(rFlagsReg cr, iRegI op1)
6611 %{
6612 single_instruction;
6613 // fixed_latency(16);
6614 cr : EX2(write);
6615 op1 : EX1(read);
6616 INS01 : ISS;
6617 ALU : EX2;
6618 %}
6619
6620 //------- Conditional instructions ------------------------
6621
6622 // Conditional no operands
6623 // Eg. CSINC x0, zr, zr, <cond>
6624 pipe_class icond_none(iRegI dst, rFlagsReg cr)
6625 %{
6626 single_instruction;
6627 cr : EX1(read);
6628 dst : EX2(write);
6629 INS01 : ISS;
6630 ALU : EX2;
6631 %}
6632
6633 // Conditional 2 operand
6634 // EG. CSEL X0, X1, X2, <cond>
6635 pipe_class icond_reg_reg(iRegI dst, iRegI src1, iRegI src2, rFlagsReg cr)
6636 %{
6637 single_instruction;
6638 cr : EX1(read);
6639 src1 : EX1(read);
6640 src2 : EX1(read);
6641 dst : EX2(write);
6642 INS01 : ISS;
6643 ALU : EX2;
6644 %}
6645
6646 // Conditional 2 operand
6647 // EG. CSEL X0, X1, X2, <cond>
6648 pipe_class icond_reg(iRegI dst, iRegI src, rFlagsReg cr)
6649 %{
6650 single_instruction;
6651 cr : EX1(read);
6652 src : EX1(read);
6653 dst : EX2(write);
6654 INS01 : ISS;
6655 ALU : EX2;
6656 %}
6657
6658 //------- Multiply pipeline operations --------------------
6659
6660 // Multiply reg-reg
6661 // Eg. MUL w0, w1, w2
6662 pipe_class imul_reg_reg(iRegI dst, iRegI src1, iRegI src2)
6663 %{
6664 single_instruction;
6665 dst : WR(write);
6666 src1 : ISS(read);
6667 src2 : ISS(read);
6668 INS01 : ISS;
6669 MAC : WR;
6670 %}
6671
6672 // Multiply accumulate
6673 // Eg. MADD w0, w1, w2, w3
6674 pipe_class imac_reg_reg(iRegI dst, iRegI src1, iRegI src2, iRegI src3)
6675 %{
6676 single_instruction;
6677 dst : WR(write);
6678 src1 : ISS(read);
6679 src2 : ISS(read);
6680 src3 : ISS(read);
6681 INS01 : ISS;
6682 MAC : WR;
6683 %}
6684
6685 // Eg. MUL w0, w1, w2
6686 pipe_class lmul_reg_reg(iRegI dst, iRegI src1, iRegI src2)
6687 %{
6688 single_instruction;
6689 fixed_latency(3); // Maximum latency for 64 bit mul
6690 dst : WR(write);
6691 src1 : ISS(read);
6692 src2 : ISS(read);
6693 INS01 : ISS;
6694 MAC : WR;
6695 %}
6696
6697 // Multiply accumulate
6698 // Eg. MADD w0, w1, w2, w3
6699 pipe_class lmac_reg_reg(iRegI dst, iRegI src1, iRegI src2, iRegI src3)
6700 %{
6701 single_instruction;
6702 fixed_latency(3); // Maximum latency for 64 bit mul
6703 dst : WR(write);
6704 src1 : ISS(read);
6705 src2 : ISS(read);
6706 src3 : ISS(read);
6707 INS01 : ISS;
6708 MAC : WR;
6709 %}
6710
6711 //------- Divide pipeline operations --------------------
6712
6713 // Eg. SDIV w0, w1, w2
6714 pipe_class idiv_reg_reg(iRegI dst, iRegI src1, iRegI src2)
6715 %{
6716 single_instruction;
6717 fixed_latency(8); // Maximum latency for 32 bit divide
6718 dst : WR(write);
6719 src1 : ISS(read);
6720 src2 : ISS(read);
6721 INS0 : ISS; // Can only dual issue as instruction 0
6722 DIV : WR;
6723 %}
6724
6725 // Eg. SDIV x0, x1, x2
6726 pipe_class ldiv_reg_reg(iRegI dst, iRegI src1, iRegI src2)
6727 %{
6728 single_instruction;
6729 fixed_latency(16); // Maximum latency for 64 bit divide
6730 dst : WR(write);
6731 src1 : ISS(read);
6732 src2 : ISS(read);
6733 INS0 : ISS; // Can only dual issue as instruction 0
6734 DIV : WR;
6735 %}
6736
6737 //------- Load pipeline operations ------------------------
6738
6739 // Load - prefetch
6740 // Eg. PFRM <mem>
6741 pipe_class iload_prefetch(memory mem)
6742 %{
6743 single_instruction;
6744 mem : ISS(read);
6745 INS01 : ISS;
6746 LDST : WR;
6747 %}
6748
6749 // Load - reg, mem
6750 // Eg. LDR x0, <mem>
6751 pipe_class iload_reg_mem(iRegI dst, memory mem)
6752 %{
6753 single_instruction;
6754 dst : WR(write);
6755 mem : ISS(read);
6756 INS01 : ISS;
6757 LDST : WR;
6758 %}
6759
6760 // Load - reg, reg
6761 // Eg. LDR x0, [sp, x1]
6762 pipe_class iload_reg_reg(iRegI dst, iRegI src)
6763 %{
6764 single_instruction;
6765 dst : WR(write);
6766 src : ISS(read);
6767 INS01 : ISS;
6768 LDST : WR;
6769 %}
6770
6771 //------- Store pipeline operations -----------------------
6772
6773 // Store - zr, mem
6774 // Eg. STR zr, <mem>
6775 pipe_class istore_mem(memory mem)
6776 %{
6777 single_instruction;
6778 mem : ISS(read);
6779 INS01 : ISS;
6780 LDST : WR;
6781 %}
6782
6783 // Store - reg, mem
6784 // Eg. STR x0, <mem>
6785 pipe_class istore_reg_mem(iRegI src, memory mem)
6786 %{
6787 single_instruction;
6788 mem : ISS(read);
6789 src : EX2(read);
6790 INS01 : ISS;
6791 LDST : WR;
6792 %}
6793
6794 // Store - reg, reg
6795 // Eg. STR x0, [sp, x1]
6796 pipe_class istore_reg_reg(iRegI dst, iRegI src)
6797 %{
6798 single_instruction;
6799 dst : ISS(read);
6800 src : EX2(read);
6801 INS01 : ISS;
6802 LDST : WR;
6803 %}
6804
6805 //------- Store pipeline operations -----------------------
6806
6807 // Branch
6808 pipe_class pipe_branch()
6809 %{
6810 single_instruction;
6811 INS01 : ISS;
6812 BRANCH : EX1;
6813 %}
6814
6815 // Conditional branch
6816 pipe_class pipe_branch_cond(rFlagsReg cr)
6817 %{
6818 single_instruction;
6819 cr : EX1(read);
6820 INS01 : ISS;
6821 BRANCH : EX1;
6822 %}
6823
6824 // Compare & Branch
6825 // EG. CBZ/CBNZ
6826 pipe_class pipe_cmp_branch(iRegI op1)
6827 %{
6828 single_instruction;
6829 op1 : EX1(read);
6830 INS01 : ISS;
6831 BRANCH : EX1;
6832 %}
6833
6834 //------- Synchronisation operations ----------------------
6835
6836 // Any operation requiring serialization.
6837 // EG. DMB/Atomic Ops/Load Acquire/Str Release
6838 pipe_class pipe_serial()
6839 %{
6840 single_instruction;
6841 force_serialization;
6842 fixed_latency(16);
6843 INS01 : ISS(2); // Cannot dual issue with any other instruction
6844 LDST : WR;
6845 %}
6846
6847 // Generic big/slow expanded idiom - also serialized
6848 pipe_class pipe_slow()
6849 %{
6850 instruction_count(10);
6851 multiple_bundles;
6852 force_serialization;
6853 fixed_latency(16);
6854 INS01 : ISS(2); // Cannot dual issue with any other instruction
6855 LDST : WR;
6856 %}
6857
6858 // Empty pipeline class
6859 pipe_class pipe_class_empty()
6860 %{
6861 single_instruction;
6862 fixed_latency(0);
6863 %}
6864
6865 // Default pipeline class.
6866 pipe_class pipe_class_default()
6867 %{
6868 single_instruction;
6869 fixed_latency(2);
6870 %}
6871
6872 // Pipeline class for compares.
6873 pipe_class pipe_class_compare()
6874 %{
6875 single_instruction;
6876 fixed_latency(16);
6877 %}
6878
6879 // Pipeline class for memory operations.
6880 pipe_class pipe_class_memory()
6881 %{
6882 single_instruction;
6883 fixed_latency(16);
6884 %}
6885
6886 // Pipeline class for call.
6887 pipe_class pipe_class_call()
6888 %{
6889 single_instruction;
6890 fixed_latency(100);
6891 %}
6892
6893 // Define the class for the Nop node.
6894 define %{
6895 MachNop = pipe_class_empty;
6896 %}
6897
6898 %}
6899 //----------INSTRUCTIONS-------------------------------------------------------
6900 //
6901 // match -- States which machine-independent subtree may be replaced
6902 // by this instruction.
6903 // ins_cost -- The estimated cost of this instruction is used by instruction
6904 // selection to identify a minimum cost tree of machine
6905 // instructions that matches a tree of machine-independent
6906 // instructions.
6907 // format -- A string providing the disassembly for this instruction.
6908 // The value of an instruction's operand may be inserted
6909 // by referring to it with a '$' prefix.
6910 // opcode -- Three instruction opcodes may be provided. These are referred
6911 // to within an encode class as $primary, $secondary, and $tertiary
6912 // rrspectively. The primary opcode is commonly used to
6913 // indicate the type of machine instruction, while secondary
6914 // and tertiary are often used for prefix options or addressing
6915 // modes.
6916 // ins_encode -- A list of encode classes with parameters. The encode class
6917 // name must have been defined in an 'enc_class' specification
6918 // in the encode section of the architecture description.
6919
6920 // ============================================================================
6921 // Memory (Load/Store) Instructions
6922
6923 // Load Instructions
6924
6925 // Load Byte (8 bit signed)
6926 instruct loadB(iRegINoSp dst, memory1 mem)
6927 %{
6928 match(Set dst (LoadB mem));
6929 predicate(!needs_acquiring_load(n));
6930
6931 ins_cost(4 * INSN_COST);
6932 format %{ "ldrsbw $dst, $mem\t# byte" %}
6933
6934 ins_encode(aarch64_enc_ldrsbw(dst, mem));
6935
6936 ins_pipe(iload_reg_mem);
6937 %}
6938
6939 // Load Byte (8 bit signed) into long
6940 instruct loadB2L(iRegLNoSp dst, memory1 mem)
6941 %{
6942 match(Set dst (ConvI2L (LoadB mem)));
6943 predicate(!needs_acquiring_load(n->in(1)));
6944
6945 ins_cost(4 * INSN_COST);
6946 format %{ "ldrsb $dst, $mem\t# byte" %}
6947
6948 ins_encode(aarch64_enc_ldrsb(dst, mem));
6949
6950 ins_pipe(iload_reg_mem);
6951 %}
6952
6953 // Load Byte (8 bit unsigned)
6954 instruct loadUB(iRegINoSp dst, memory1 mem)
6955 %{
6956 match(Set dst (LoadUB mem));
6957 predicate(!needs_acquiring_load(n));
6958
6959 ins_cost(4 * INSN_COST);
6960 format %{ "ldrbw $dst, $mem\t# byte" %}
6961
6962 ins_encode(aarch64_enc_ldrb(dst, mem));
6963
6964 ins_pipe(iload_reg_mem);
6965 %}
6966
6967 // Load Byte (8 bit unsigned) into long
6968 instruct loadUB2L(iRegLNoSp dst, memory1 mem)
6969 %{
6970 match(Set dst (ConvI2L (LoadUB mem)));
6971 predicate(!needs_acquiring_load(n->in(1)));
6972
6973 ins_cost(4 * INSN_COST);
6974 format %{ "ldrb $dst, $mem\t# byte" %}
6975
6976 ins_encode(aarch64_enc_ldrb(dst, mem));
6977
6978 ins_pipe(iload_reg_mem);
6979 %}
6980
6981 // Load Short (16 bit signed)
6982 instruct loadS(iRegINoSp dst, memory2 mem)
6983 %{
6984 match(Set dst (LoadS mem));
6985 predicate(!needs_acquiring_load(n));
6986
6987 ins_cost(4 * INSN_COST);
6988 format %{ "ldrshw $dst, $mem\t# short" %}
6989
6990 ins_encode(aarch64_enc_ldrshw(dst, mem));
6991
6992 ins_pipe(iload_reg_mem);
6993 %}
6994
6995 // Load Short (16 bit signed) into long
6996 instruct loadS2L(iRegLNoSp dst, memory2 mem)
6997 %{
6998 match(Set dst (ConvI2L (LoadS mem)));
6999 predicate(!needs_acquiring_load(n->in(1)));
7000
7001 ins_cost(4 * INSN_COST);
7002 format %{ "ldrsh $dst, $mem\t# short" %}
7003
7004 ins_encode(aarch64_enc_ldrsh(dst, mem));
7005
7006 ins_pipe(iload_reg_mem);
7007 %}
7008
7009 // Load Char (16 bit unsigned)
7010 instruct loadUS(iRegINoSp dst, memory2 mem)
7011 %{
7012 match(Set dst (LoadUS mem));
7013 predicate(!needs_acquiring_load(n));
7014
7015 ins_cost(4 * INSN_COST);
7016 format %{ "ldrh $dst, $mem\t# short" %}
7017
7018 ins_encode(aarch64_enc_ldrh(dst, mem));
7019
7020 ins_pipe(iload_reg_mem);
7021 %}
7022
7023 // Load Short/Char (16 bit unsigned) into long
7024 instruct loadUS2L(iRegLNoSp dst, memory2 mem)
7025 %{
7026 match(Set dst (ConvI2L (LoadUS mem)));
7027 predicate(!needs_acquiring_load(n->in(1)));
7028
7029 ins_cost(4 * INSN_COST);
7030 format %{ "ldrh $dst, $mem\t# short" %}
7031
7032 ins_encode(aarch64_enc_ldrh(dst, mem));
7033
7034 ins_pipe(iload_reg_mem);
7035 %}
7036
7037 // Load Integer (32 bit signed)
7038 instruct loadI(iRegINoSp dst, memory4 mem)
7039 %{
7040 match(Set dst (LoadI mem));
7041 predicate(!needs_acquiring_load(n));
7042
7043 ins_cost(4 * INSN_COST);
7044 format %{ "ldrw $dst, $mem\t# int" %}
7045
7046 ins_encode(aarch64_enc_ldrw(dst, mem));
7047
7048 ins_pipe(iload_reg_mem);
7049 %}
7050
7051 // Load Integer (32 bit signed) into long
7052 instruct loadI2L(iRegLNoSp dst, memory4 mem)
7053 %{
7054 match(Set dst (ConvI2L (LoadI mem)));
7055 predicate(!needs_acquiring_load(n->in(1)));
7056
7057 ins_cost(4 * INSN_COST);
7058 format %{ "ldrsw $dst, $mem\t# int" %}
7059
7060 ins_encode(aarch64_enc_ldrsw(dst, mem));
7061
7062 ins_pipe(iload_reg_mem);
7063 %}
7064
7065 // Load Integer (32 bit unsigned) into long
7066 instruct loadUI2L(iRegLNoSp dst, memory4 mem, immL_32bits mask)
7067 %{
7068 match(Set dst (AndL (ConvI2L (LoadI mem)) mask));
7069 predicate(!needs_acquiring_load(n->in(1)->in(1)->as_Load()));
7070
7071 ins_cost(4 * INSN_COST);
7072 format %{ "ldrw $dst, $mem\t# int" %}
7073
7074 ins_encode(aarch64_enc_ldrw(dst, mem));
7075
7076 ins_pipe(iload_reg_mem);
7077 %}
7078
7079 // Load Long (64 bit signed)
7080 instruct loadL(iRegLNoSp dst, memory8 mem)
7081 %{
7082 match(Set dst (LoadL mem));
7083 predicate(!needs_acquiring_load(n));
7084
7085 ins_cost(4 * INSN_COST);
7086 format %{ "ldr $dst, $mem\t# int" %}
7087
7088 ins_encode(aarch64_enc_ldr(dst, mem));
7089
7090 ins_pipe(iload_reg_mem);
7091 %}
7092
7093 // Load Range
7094 instruct loadRange(iRegINoSp dst, memory4 mem)
7095 %{
7096 match(Set dst (LoadRange mem));
7097
7098 ins_cost(4 * INSN_COST);
7099 format %{ "ldrw $dst, $mem\t# range" %}
7100
7101 ins_encode(aarch64_enc_ldrw(dst, mem));
7102
7103 ins_pipe(iload_reg_mem);
7104 %}
7105
7106 // Load Pointer
7107 instruct loadP(iRegPNoSp dst, memory8 mem)
7108 %{
7109 match(Set dst (LoadP mem));
7110 predicate(!needs_acquiring_load(n) && (n->as_Load()->barrier_data() == 0));
7111
7112 ins_cost(4 * INSN_COST);
7113 format %{ "ldr $dst, $mem\t# ptr" %}
7114
7115 ins_encode(aarch64_enc_ldr(dst, mem));
7116
7117 ins_pipe(iload_reg_mem);
7118 %}
7119
7120 // Load Compressed Pointer
7121 instruct loadN(iRegNNoSp dst, memory4 mem)
7122 %{
7123 match(Set dst (LoadN mem));
7124 predicate(!needs_acquiring_load(n));
7125
7126 ins_cost(4 * INSN_COST);
7127 format %{ "ldrw $dst, $mem\t# compressed ptr" %}
7128
7129 ins_encode(aarch64_enc_ldrw(dst, mem));
7130
7131 ins_pipe(iload_reg_mem);
7132 %}
7133
7134 // Load Klass Pointer
7135 instruct loadKlass(iRegPNoSp dst, memory8 mem)
7136 %{
7137 match(Set dst (LoadKlass mem));
7138 predicate(!needs_acquiring_load(n));
7139
7140 ins_cost(4 * INSN_COST);
7141 format %{ "ldr $dst, $mem\t# class" %}
7142
7143 ins_encode(aarch64_enc_ldr(dst, mem));
7144
7145 ins_pipe(iload_reg_mem);
7146 %}
7147
7148 // Load Narrow Klass Pointer
7149 instruct loadNKlass(iRegNNoSp dst, memory4 mem, rFlagsReg cr)
7150 %{
7151 match(Set dst (LoadNKlass mem));
7152 effect(TEMP_DEF dst, KILL cr);
7153 predicate(!needs_acquiring_load(n));
7154
7155 ins_cost(4 * INSN_COST);
7156 format %{ "ldrw $dst, $mem\t# compressed class ptr" %}
7157 ins_encode %{
7158 assert($mem$$disp == oopDesc::klass_offset_in_bytes(), "expect correct offset");
7159 assert($mem$$index$$Register == noreg, "expect no index");
7160 __ load_nklass($dst$$Register, $mem$$base$$Register);
7161 %}
7162 ins_pipe(pipe_slow);
7163 %}
7164
7165 // Load Float
7166 instruct loadF(vRegF dst, memory4 mem)
7167 %{
7168 match(Set dst (LoadF mem));
7169 predicate(!needs_acquiring_load(n));
7170
7171 ins_cost(4 * INSN_COST);
7172 format %{ "ldrs $dst, $mem\t# float" %}
7173
7174 ins_encode( aarch64_enc_ldrs(dst, mem) );
7175
7176 ins_pipe(pipe_class_memory);
7177 %}
7178
7179 // Load Double
7180 instruct loadD(vRegD dst, memory8 mem)
7181 %{
7182 match(Set dst (LoadD mem));
7183 predicate(!needs_acquiring_load(n));
7184
7185 ins_cost(4 * INSN_COST);
7186 format %{ "ldrd $dst, $mem\t# double" %}
7187
7188 ins_encode( aarch64_enc_ldrd(dst, mem) );
7189
7190 ins_pipe(pipe_class_memory);
7191 %}
7192
7193
7194 // Load Int Constant
7195 instruct loadConI(iRegINoSp dst, immI src)
7196 %{
7197 match(Set dst src);
7198
7199 ins_cost(INSN_COST);
7200 format %{ "mov $dst, $src\t# int" %}
7201
7202 ins_encode( aarch64_enc_movw_imm(dst, src) );
7203
7204 ins_pipe(ialu_imm);
7205 %}
7206
7207 // Load Long Constant
7208 instruct loadConL(iRegLNoSp dst, immL src)
7209 %{
7210 match(Set dst src);
7211
7212 ins_cost(INSN_COST);
7213 format %{ "mov $dst, $src\t# long" %}
7214
7215 ins_encode( aarch64_enc_mov_imm(dst, src) );
7216
7217 ins_pipe(ialu_imm);
7218 %}
7219
7220 // Load Pointer Constant
7221
7222 instruct loadConP(iRegPNoSp dst, immP con)
7223 %{
7224 match(Set dst con);
7225
7226 ins_cost(INSN_COST * 4);
7227 format %{
7228 "mov $dst, $con\t# ptr\n\t"
7229 %}
7230
7231 ins_encode(aarch64_enc_mov_p(dst, con));
7232
7233 ins_pipe(ialu_imm);
7234 %}
7235
7236 // Load Null Pointer Constant
7237
7238 instruct loadConP0(iRegPNoSp dst, immP0 con)
7239 %{
7240 match(Set dst con);
7241
7242 ins_cost(INSN_COST);
7243 format %{ "mov $dst, $con\t# NULL ptr" %}
7244
7245 ins_encode(aarch64_enc_mov_p0(dst, con));
7246
7247 ins_pipe(ialu_imm);
7248 %}
7249
7250 // Load Pointer Constant One
7251
7252 instruct loadConP1(iRegPNoSp dst, immP_1 con)
7253 %{
7254 match(Set dst con);
7255
7256 ins_cost(INSN_COST);
7257 format %{ "mov $dst, $con\t# NULL ptr" %}
7258
7259 ins_encode(aarch64_enc_mov_p1(dst, con));
7260
7261 ins_pipe(ialu_imm);
7262 %}
7263
7264 // Load Byte Map Base Constant
7265
7266 instruct loadByteMapBase(iRegPNoSp dst, immByteMapBase con)
7267 %{
7268 match(Set dst con);
7269
7270 ins_cost(INSN_COST);
7271 format %{ "adr $dst, $con\t# Byte Map Base" %}
7272
7273 ins_encode(aarch64_enc_mov_byte_map_base(dst, con));
7274
7275 ins_pipe(ialu_imm);
7276 %}
7277
7278 // Load Narrow Pointer Constant
7279
7280 instruct loadConN(iRegNNoSp dst, immN con)
7281 %{
7282 match(Set dst con);
7283
7284 ins_cost(INSN_COST * 4);
7285 format %{ "mov $dst, $con\t# compressed ptr" %}
7286
7287 ins_encode(aarch64_enc_mov_n(dst, con));
7288
7289 ins_pipe(ialu_imm);
7290 %}
7291
7292 // Load Narrow Null Pointer Constant
7293
7294 instruct loadConN0(iRegNNoSp dst, immN0 con)
7295 %{
7296 match(Set dst con);
7297
7298 ins_cost(INSN_COST);
7299 format %{ "mov $dst, $con\t# compressed NULL ptr" %}
7300
7301 ins_encode(aarch64_enc_mov_n0(dst, con));
7302
7303 ins_pipe(ialu_imm);
7304 %}
7305
7306 // Load Narrow Klass Constant
7307
7308 instruct loadConNKlass(iRegNNoSp dst, immNKlass con)
7309 %{
7310 match(Set dst con);
7311
7312 ins_cost(INSN_COST);
7313 format %{ "mov $dst, $con\t# compressed klass ptr" %}
7314
7315 ins_encode(aarch64_enc_mov_nk(dst, con));
7316
7317 ins_pipe(ialu_imm);
7318 %}
7319
7320 // Load Packed Float Constant
7321
7322 instruct loadConF_packed(vRegF dst, immFPacked con) %{
7323 match(Set dst con);
7324 ins_cost(INSN_COST * 4);
7325 format %{ "fmovs $dst, $con"%}
7326 ins_encode %{
7327 __ fmovs(as_FloatRegister($dst$$reg), (double)$con$$constant);
7328 %}
7329
7330 ins_pipe(fp_imm_s);
7331 %}
7332
7333 // Load Float Constant
7334
7335 instruct loadConF(vRegF dst, immF con) %{
7336 match(Set dst con);
7337
7338 ins_cost(INSN_COST * 4);
7339
7340 format %{
7341 "ldrs $dst, [$constantaddress]\t# load from constant table: float=$con\n\t"
7342 %}
7343
7344 ins_encode %{
7345 __ ldrs(as_FloatRegister($dst$$reg), $constantaddress($con));
7346 %}
7347
7348 ins_pipe(fp_load_constant_s);
7349 %}
7350
7351 // Load Packed Double Constant
7352
7353 instruct loadConD_packed(vRegD dst, immDPacked con) %{
7354 match(Set dst con);
7355 ins_cost(INSN_COST);
7356 format %{ "fmovd $dst, $con"%}
7357 ins_encode %{
7358 __ fmovd(as_FloatRegister($dst$$reg), $con$$constant);
7359 %}
7360
7361 ins_pipe(fp_imm_d);
7362 %}
7363
7364 // Load Double Constant
7365
7366 instruct loadConD(vRegD dst, immD con) %{
7367 match(Set dst con);
7368
7369 ins_cost(INSN_COST * 5);
7370 format %{
7371 "ldrd $dst, [$constantaddress]\t# load from constant table: float=$con\n\t"
7372 %}
7373
7374 ins_encode %{
7375 __ ldrd(as_FloatRegister($dst$$reg), $constantaddress($con));
7376 %}
7377
7378 ins_pipe(fp_load_constant_d);
7379 %}
7380
7381 // Store Instructions
7382
7383 // Store CMS card-mark Immediate
7384 instruct storeimmCM0(immI0 zero, memory1 mem)
7385 %{
7386 match(Set mem (StoreCM mem zero));
7387
7388 ins_cost(INSN_COST);
7389 format %{ "storestore (elided)\n\t"
7390 "strb zr, $mem\t# byte" %}
7391
7392 ins_encode(aarch64_enc_strb0(mem));
7393
7394 ins_pipe(istore_mem);
7395 %}
7396
7397 // Store CMS card-mark Immediate with intervening StoreStore
7398 // needed when using CMS with no conditional card marking
7399 instruct storeimmCM0_ordered(immI0 zero, memory1 mem)
7400 %{
7401 match(Set mem (StoreCM mem zero));
7402
7403 ins_cost(INSN_COST * 2);
7404 format %{ "storestore\n\t"
7405 "dmb ishst"
7406 "\n\tstrb zr, $mem\t# byte" %}
7407
7408 ins_encode(aarch64_enc_strb0_ordered(mem));
7409
7410 ins_pipe(istore_mem);
7411 %}
7412
7413 // Store Byte
7414 instruct storeB(iRegIorL2I src, memory1 mem)
7415 %{
7416 match(Set mem (StoreB mem src));
7417 predicate(!needs_releasing_store(n));
7418
7419 ins_cost(INSN_COST);
7420 format %{ "strb $src, $mem\t# byte" %}
7421
7422 ins_encode(aarch64_enc_strb(src, mem));
7423
7424 ins_pipe(istore_reg_mem);
7425 %}
7426
7427
7428 instruct storeimmB0(immI0 zero, memory1 mem)
7429 %{
7430 match(Set mem (StoreB mem zero));
7431 predicate(!needs_releasing_store(n));
7432
7433 ins_cost(INSN_COST);
7434 format %{ "strb rscractch2, $mem\t# byte" %}
7435
7436 ins_encode(aarch64_enc_strb0(mem));
7437
7438 ins_pipe(istore_mem);
7439 %}
7440
7441 // Store Char/Short
7442 instruct storeC(iRegIorL2I src, memory2 mem)
7443 %{
7444 match(Set mem (StoreC mem src));
7445 predicate(!needs_releasing_store(n));
7446
7447 ins_cost(INSN_COST);
7448 format %{ "strh $src, $mem\t# short" %}
7449
7450 ins_encode(aarch64_enc_strh(src, mem));
7451
7452 ins_pipe(istore_reg_mem);
7453 %}
7454
7455 instruct storeimmC0(immI0 zero, memory2 mem)
7456 %{
7457 match(Set mem (StoreC mem zero));
7458 predicate(!needs_releasing_store(n));
7459
7460 ins_cost(INSN_COST);
7461 format %{ "strh zr, $mem\t# short" %}
7462
7463 ins_encode(aarch64_enc_strh0(mem));
7464
7465 ins_pipe(istore_mem);
7466 %}
7467
7468 // Store Integer
7469
7470 instruct storeI(iRegIorL2I src, memory4 mem)
7471 %{
7472 match(Set mem(StoreI mem src));
7473 predicate(!needs_releasing_store(n));
7474
7475 ins_cost(INSN_COST);
7476 format %{ "strw $src, $mem\t# int" %}
7477
7478 ins_encode(aarch64_enc_strw(src, mem));
7479
7480 ins_pipe(istore_reg_mem);
7481 %}
7482
7483 instruct storeimmI0(immI0 zero, memory4 mem)
7484 %{
7485 match(Set mem(StoreI mem zero));
7486 predicate(!needs_releasing_store(n));
7487
7488 ins_cost(INSN_COST);
7489 format %{ "strw zr, $mem\t# int" %}
7490
7491 ins_encode(aarch64_enc_strw0(mem));
7492
7493 ins_pipe(istore_mem);
7494 %}
7495
7496 // Store Long (64 bit signed)
7497 instruct storeL(iRegL src, memory8 mem)
7498 %{
7499 match(Set mem (StoreL mem src));
7500 predicate(!needs_releasing_store(n));
7501
7502 ins_cost(INSN_COST);
7503 format %{ "str $src, $mem\t# int" %}
7504
7505 ins_encode(aarch64_enc_str(src, mem));
7506
7507 ins_pipe(istore_reg_mem);
7508 %}
7509
7510 // Store Long (64 bit signed)
7511 instruct storeimmL0(immL0 zero, memory8 mem)
7512 %{
7513 match(Set mem (StoreL mem zero));
7514 predicate(!needs_releasing_store(n));
7515
7516 ins_cost(INSN_COST);
7517 format %{ "str zr, $mem\t# int" %}
7518
7519 ins_encode(aarch64_enc_str0(mem));
7520
7521 ins_pipe(istore_mem);
7522 %}
7523
7524 // Store Pointer
7525 instruct storeP(iRegP src, memory8 mem)
7526 %{
7527 match(Set mem (StoreP mem src));
7528 predicate(!needs_releasing_store(n));
7529
7530 ins_cost(INSN_COST);
7531 format %{ "str $src, $mem\t# ptr" %}
7532
7533 ins_encode(aarch64_enc_str(src, mem));
7534
7535 ins_pipe(istore_reg_mem);
7536 %}
7537
7538 // Store Pointer
7539 instruct storeimmP0(immP0 zero, memory8 mem)
7540 %{
7541 match(Set mem (StoreP mem zero));
7542 predicate(!needs_releasing_store(n));
7543
7544 ins_cost(INSN_COST);
7545 format %{ "str zr, $mem\t# ptr" %}
7546
7547 ins_encode(aarch64_enc_str0(mem));
7548
7549 ins_pipe(istore_mem);
7550 %}
7551
7552 // Store Compressed Pointer
7553 instruct storeN(iRegN src, memory4 mem)
7554 %{
7555 match(Set mem (StoreN mem src));
7556 predicate(!needs_releasing_store(n));
7557
7558 ins_cost(INSN_COST);
7559 format %{ "strw $src, $mem\t# compressed ptr" %}
7560
7561 ins_encode(aarch64_enc_strw(src, mem));
7562
7563 ins_pipe(istore_reg_mem);
7564 %}
7565
7566 instruct storeImmN0(immN0 zero, memory4 mem)
7567 %{
7568 match(Set mem (StoreN mem zero));
7569 predicate(!needs_releasing_store(n));
7570
7571 ins_cost(INSN_COST);
7572 format %{ "strw zr, $mem\t# compressed ptr" %}
7573
7574 ins_encode(aarch64_enc_strw0(mem));
7575
7576 ins_pipe(istore_mem);
7577 %}
7578
7579 // Store Float
7580 instruct storeF(vRegF src, memory4 mem)
7581 %{
7582 match(Set mem (StoreF mem src));
7583 predicate(!needs_releasing_store(n));
7584
7585 ins_cost(INSN_COST);
7586 format %{ "strs $src, $mem\t# float" %}
7587
7588 ins_encode( aarch64_enc_strs(src, mem) );
7589
7590 ins_pipe(pipe_class_memory);
7591 %}
7592
7593 // TODO
7594 // implement storeImmF0 and storeFImmPacked
7595
7596 // Store Double
7597 instruct storeD(vRegD src, memory8 mem)
7598 %{
7599 match(Set mem (StoreD mem src));
7600 predicate(!needs_releasing_store(n));
7601
7602 ins_cost(INSN_COST);
7603 format %{ "strd $src, $mem\t# double" %}
7604
7605 ins_encode( aarch64_enc_strd(src, mem) );
7606
7607 ins_pipe(pipe_class_memory);
7608 %}
7609
7610 // Store Compressed Klass Pointer
7611 instruct storeNKlass(iRegN src, memory4 mem)
7612 %{
7613 predicate(!needs_releasing_store(n));
7614 match(Set mem (StoreNKlass mem src));
7615
7616 ins_cost(INSN_COST);
7617 format %{ "strw $src, $mem\t# compressed klass ptr" %}
7618
7619 ins_encode(aarch64_enc_strw(src, mem));
7620
7621 ins_pipe(istore_reg_mem);
7622 %}
7623
7624 // TODO
7625 // implement storeImmD0 and storeDImmPacked
7626
7627 // prefetch instructions
7628 // Must be safe to execute with invalid address (cannot fault).
7629
7630 instruct prefetchalloc( memory8 mem ) %{
7631 match(PrefetchAllocation mem);
7632
7633 ins_cost(INSN_COST);
7634 format %{ "prfm $mem, PSTL1KEEP\t# Prefetch into level 1 cache write keep" %}
7635
7636 ins_encode( aarch64_enc_prefetchw(mem) );
7637
7638 ins_pipe(iload_prefetch);
7639 %}
7640
7641 // ---------------- volatile loads and stores ----------------
7642
7643 // Load Byte (8 bit signed)
7644 instruct loadB_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
7645 %{
7646 match(Set dst (LoadB mem));
7647
7648 ins_cost(VOLATILE_REF_COST);
7649 format %{ "ldarsb $dst, $mem\t# byte" %}
7650
7651 ins_encode(aarch64_enc_ldarsb(dst, mem));
7652
7653 ins_pipe(pipe_serial);
7654 %}
7655
7656 // Load Byte (8 bit signed) into long
7657 instruct loadB2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
7658 %{
7659 match(Set dst (ConvI2L (LoadB mem)));
7660
7661 ins_cost(VOLATILE_REF_COST);
7662 format %{ "ldarsb $dst, $mem\t# byte" %}
7663
7664 ins_encode(aarch64_enc_ldarsb(dst, mem));
7665
7666 ins_pipe(pipe_serial);
7667 %}
7668
7669 // Load Byte (8 bit unsigned)
7670 instruct loadUB_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
7671 %{
7672 match(Set dst (LoadUB mem));
7673
7674 ins_cost(VOLATILE_REF_COST);
7675 format %{ "ldarb $dst, $mem\t# byte" %}
7676
7677 ins_encode(aarch64_enc_ldarb(dst, mem));
7678
7679 ins_pipe(pipe_serial);
7680 %}
7681
7682 // Load Byte (8 bit unsigned) into long
7683 instruct loadUB2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
7684 %{
7685 match(Set dst (ConvI2L (LoadUB mem)));
7686
7687 ins_cost(VOLATILE_REF_COST);
7688 format %{ "ldarb $dst, $mem\t# byte" %}
7689
7690 ins_encode(aarch64_enc_ldarb(dst, mem));
7691
7692 ins_pipe(pipe_serial);
7693 %}
7694
7695 // Load Short (16 bit signed)
7696 instruct loadS_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
7697 %{
7698 match(Set dst (LoadS mem));
7699
7700 ins_cost(VOLATILE_REF_COST);
7701 format %{ "ldarshw $dst, $mem\t# short" %}
7702
7703 ins_encode(aarch64_enc_ldarshw(dst, mem));
7704
7705 ins_pipe(pipe_serial);
7706 %}
7707
7708 instruct loadUS_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
7709 %{
7710 match(Set dst (LoadUS mem));
7711
7712 ins_cost(VOLATILE_REF_COST);
7713 format %{ "ldarhw $dst, $mem\t# short" %}
7714
7715 ins_encode(aarch64_enc_ldarhw(dst, mem));
7716
7717 ins_pipe(pipe_serial);
7718 %}
7719
7720 // Load Short/Char (16 bit unsigned) into long
7721 instruct loadUS2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
7722 %{
7723 match(Set dst (ConvI2L (LoadUS mem)));
7724
7725 ins_cost(VOLATILE_REF_COST);
7726 format %{ "ldarh $dst, $mem\t# short" %}
7727
7728 ins_encode(aarch64_enc_ldarh(dst, mem));
7729
7730 ins_pipe(pipe_serial);
7731 %}
7732
7733 // Load Short/Char (16 bit signed) into long
7734 instruct loadS2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
7735 %{
7736 match(Set dst (ConvI2L (LoadS mem)));
7737
7738 ins_cost(VOLATILE_REF_COST);
7739 format %{ "ldarh $dst, $mem\t# short" %}
7740
7741 ins_encode(aarch64_enc_ldarsh(dst, mem));
7742
7743 ins_pipe(pipe_serial);
7744 %}
7745
7746 // Load Integer (32 bit signed)
7747 instruct loadI_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
7748 %{
7749 match(Set dst (LoadI mem));
7750
7751 ins_cost(VOLATILE_REF_COST);
7752 format %{ "ldarw $dst, $mem\t# int" %}
7753
7754 ins_encode(aarch64_enc_ldarw(dst, mem));
7755
7756 ins_pipe(pipe_serial);
7757 %}
7758
7759 // Load Integer (32 bit unsigned) into long
7760 instruct loadUI2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem, immL_32bits mask)
7761 %{
7762 match(Set dst (AndL (ConvI2L (LoadI mem)) mask));
7763
7764 ins_cost(VOLATILE_REF_COST);
7765 format %{ "ldarw $dst, $mem\t# int" %}
7766
7767 ins_encode(aarch64_enc_ldarw(dst, mem));
7768
7769 ins_pipe(pipe_serial);
7770 %}
7771
7772 // Load Long (64 bit signed)
7773 instruct loadL_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
7774 %{
7775 match(Set dst (LoadL mem));
7776
7777 ins_cost(VOLATILE_REF_COST);
7778 format %{ "ldar $dst, $mem\t# int" %}
7779
7780 ins_encode(aarch64_enc_ldar(dst, mem));
7781
7782 ins_pipe(pipe_serial);
7783 %}
7784
7785 // Load Pointer
7786 instruct loadP_volatile(iRegPNoSp dst, /* sync_memory*/indirect mem)
7787 %{
7788 match(Set dst (LoadP mem));
7789 predicate(n->as_Load()->barrier_data() == 0);
7790
7791 ins_cost(VOLATILE_REF_COST);
7792 format %{ "ldar $dst, $mem\t# ptr" %}
7793
7794 ins_encode(aarch64_enc_ldar(dst, mem));
7795
7796 ins_pipe(pipe_serial);
7797 %}
7798
7799 // Load Compressed Pointer
7800 instruct loadN_volatile(iRegNNoSp dst, /* sync_memory*/indirect mem)
7801 %{
7802 match(Set dst (LoadN mem));
7803
7804 ins_cost(VOLATILE_REF_COST);
7805 format %{ "ldarw $dst, $mem\t# compressed ptr" %}
7806
7807 ins_encode(aarch64_enc_ldarw(dst, mem));
7808
7809 ins_pipe(pipe_serial);
7810 %}
7811
7812 // Load Float
7813 instruct loadF_volatile(vRegF dst, /* sync_memory*/indirect mem)
7814 %{
7815 match(Set dst (LoadF mem));
7816
7817 ins_cost(VOLATILE_REF_COST);
7818 format %{ "ldars $dst, $mem\t# float" %}
7819
7820 ins_encode( aarch64_enc_fldars(dst, mem) );
7821
7822 ins_pipe(pipe_serial);
7823 %}
7824
7825 // Load Double
7826 instruct loadD_volatile(vRegD dst, /* sync_memory*/indirect mem)
7827 %{
7828 match(Set dst (LoadD mem));
7829
7830 ins_cost(VOLATILE_REF_COST);
7831 format %{ "ldard $dst, $mem\t# double" %}
7832
7833 ins_encode( aarch64_enc_fldard(dst, mem) );
7834
7835 ins_pipe(pipe_serial);
7836 %}
7837
7838 // Store Byte
7839 instruct storeB_volatile(iRegIorL2I src, /* sync_memory*/indirect mem)
7840 %{
7841 match(Set mem (StoreB mem src));
7842
7843 ins_cost(VOLATILE_REF_COST);
7844 format %{ "stlrb $src, $mem\t# byte" %}
7845
7846 ins_encode(aarch64_enc_stlrb(src, mem));
7847
7848 ins_pipe(pipe_class_memory);
7849 %}
7850
7851 instruct storeimmB0_volatile(immI0 zero, /* sync_memory*/indirect mem)
7852 %{
7853 match(Set mem (StoreB mem zero));
7854
7855 ins_cost(VOLATILE_REF_COST);
7856 format %{ "stlrb zr, $mem\t# byte" %}
7857
7858 ins_encode(aarch64_enc_stlrb0(mem));
7859
7860 ins_pipe(pipe_class_memory);
7861 %}
7862
7863 // Store Char/Short
7864 instruct storeC_volatile(iRegIorL2I src, /* sync_memory*/indirect mem)
7865 %{
7866 match(Set mem (StoreC mem src));
7867
7868 ins_cost(VOLATILE_REF_COST);
7869 format %{ "stlrh $src, $mem\t# short" %}
7870
7871 ins_encode(aarch64_enc_stlrh(src, mem));
7872
7873 ins_pipe(pipe_class_memory);
7874 %}
7875
7876 instruct storeimmC0_volatile(immI0 zero, /* sync_memory*/indirect mem)
7877 %{
7878 match(Set mem (StoreC mem zero));
7879
7880 ins_cost(VOLATILE_REF_COST);
7881 format %{ "stlrh zr, $mem\t# short" %}
7882
7883 ins_encode(aarch64_enc_stlrh0(mem));
7884
7885 ins_pipe(pipe_class_memory);
7886 %}
7887
7888 // Store Integer
7889
7890 instruct storeI_volatile(iRegIorL2I src, /* sync_memory*/indirect mem)
7891 %{
7892 match(Set mem(StoreI mem src));
7893
7894 ins_cost(VOLATILE_REF_COST);
7895 format %{ "stlrw $src, $mem\t# int" %}
7896
7897 ins_encode(aarch64_enc_stlrw(src, mem));
7898
7899 ins_pipe(pipe_class_memory);
7900 %}
7901
7902 instruct storeimmI0_volatile(immI0 zero, /* sync_memory*/indirect mem)
7903 %{
7904 match(Set mem(StoreI mem zero));
7905
7906 ins_cost(VOLATILE_REF_COST);
7907 format %{ "stlrw zr, $mem\t# int" %}
7908
7909 ins_encode(aarch64_enc_stlrw0(mem));
7910
7911 ins_pipe(pipe_class_memory);
7912 %}
7913
7914 // Store Long (64 bit signed)
7915 instruct storeL_volatile(iRegL src, /* sync_memory*/indirect mem)
7916 %{
7917 match(Set mem (StoreL mem src));
7918
7919 ins_cost(VOLATILE_REF_COST);
7920 format %{ "stlr $src, $mem\t# int" %}
7921
7922 ins_encode(aarch64_enc_stlr(src, mem));
7923
7924 ins_pipe(pipe_class_memory);
7925 %}
7926
7927 instruct storeimmL0_volatile(immL0 zero, /* sync_memory*/indirect mem)
7928 %{
7929 match(Set mem (StoreL mem zero));
7930
7931 ins_cost(VOLATILE_REF_COST);
7932 format %{ "stlr zr, $mem\t# int" %}
7933
7934 ins_encode(aarch64_enc_stlr0(mem));
7935
7936 ins_pipe(pipe_class_memory);
7937 %}
7938
7939 // Store Pointer
7940 instruct storeP_volatile(iRegP src, /* sync_memory*/indirect mem)
7941 %{
7942 match(Set mem (StoreP mem src));
7943
7944 ins_cost(VOLATILE_REF_COST);
7945 format %{ "stlr $src, $mem\t# ptr" %}
7946
7947 ins_encode(aarch64_enc_stlr(src, mem));
7948
7949 ins_pipe(pipe_class_memory);
7950 %}
7951
7952 instruct storeimmP0_volatile(immP0 zero, /* sync_memory*/indirect mem)
7953 %{
7954 match(Set mem (StoreP mem zero));
7955
7956 ins_cost(VOLATILE_REF_COST);
7957 format %{ "stlr zr, $mem\t# ptr" %}
7958
7959 ins_encode(aarch64_enc_stlr0(mem));
7960
7961 ins_pipe(pipe_class_memory);
7962 %}
7963
7964 // Store Compressed Pointer
7965 instruct storeN_volatile(iRegN src, /* sync_memory*/indirect mem)
7966 %{
7967 match(Set mem (StoreN mem src));
7968
7969 ins_cost(VOLATILE_REF_COST);
7970 format %{ "stlrw $src, $mem\t# compressed ptr" %}
7971
7972 ins_encode(aarch64_enc_stlrw(src, mem));
7973
7974 ins_pipe(pipe_class_memory);
7975 %}
7976
7977 instruct storeimmN0_volatile(immN0 zero, /* sync_memory*/indirect mem)
7978 %{
7979 match(Set mem (StoreN mem zero));
7980
7981 ins_cost(VOLATILE_REF_COST);
7982 format %{ "stlrw zr, $mem\t# compressed ptr" %}
7983
7984 ins_encode(aarch64_enc_stlrw0(mem));
7985
7986 ins_pipe(pipe_class_memory);
7987 %}
7988
7989 // Store Float
7990 instruct storeF_volatile(vRegF src, /* sync_memory*/indirect mem)
7991 %{
7992 match(Set mem (StoreF mem src));
7993
7994 ins_cost(VOLATILE_REF_COST);
7995 format %{ "stlrs $src, $mem\t# float" %}
7996
7997 ins_encode( aarch64_enc_fstlrs(src, mem) );
7998
7999 ins_pipe(pipe_class_memory);
8000 %}
8001
8002 // TODO
8003 // implement storeImmF0 and storeFImmPacked
8004
8005 // Store Double
8006 instruct storeD_volatile(vRegD src, /* sync_memory*/indirect mem)
8007 %{
8008 match(Set mem (StoreD mem src));
8009
8010 ins_cost(VOLATILE_REF_COST);
8011 format %{ "stlrd $src, $mem\t# double" %}
8012
8013 ins_encode( aarch64_enc_fstlrd(src, mem) );
8014
8015 ins_pipe(pipe_class_memory);
8016 %}
8017
8018 // ---------------- end of volatile loads and stores ----------------
8019
8020 instruct cacheWB(indirect addr)
8021 %{
8022 predicate(VM_Version::supports_data_cache_line_flush());
8023 match(CacheWB addr);
8024
8025 ins_cost(100);
8026 format %{"cache wb $addr" %}
8027 ins_encode %{
8028 assert($addr->index_position() < 0, "should be");
8029 assert($addr$$disp == 0, "should be");
8030 __ cache_wb(Address($addr$$base$$Register, 0));
8031 %}
8032 ins_pipe(pipe_slow); // XXX
8033 %}
8034
8035 instruct cacheWBPreSync()
8036 %{
8037 predicate(VM_Version::supports_data_cache_line_flush());
8038 match(CacheWBPreSync);
8039
8040 ins_cost(100);
8041 format %{"cache wb presync" %}
8042 ins_encode %{
8043 __ cache_wbsync(true);
8044 %}
8045 ins_pipe(pipe_slow); // XXX
8046 %}
8047
8048 instruct cacheWBPostSync()
8049 %{
8050 predicate(VM_Version::supports_data_cache_line_flush());
8051 match(CacheWBPostSync);
8052
8053 ins_cost(100);
8054 format %{"cache wb postsync" %}
8055 ins_encode %{
8056 __ cache_wbsync(false);
8057 %}
8058 ins_pipe(pipe_slow); // XXX
8059 %}
8060
8061 // ============================================================================
8062 // BSWAP Instructions
8063
8064 instruct bytes_reverse_int(iRegINoSp dst, iRegIorL2I src) %{
8065 match(Set dst (ReverseBytesI src));
8066
8067 ins_cost(INSN_COST);
8068 format %{ "revw $dst, $src" %}
8069
8070 ins_encode %{
8071 __ revw(as_Register($dst$$reg), as_Register($src$$reg));
8072 %}
8073
8074 ins_pipe(ialu_reg);
8075 %}
8076
8077 instruct bytes_reverse_long(iRegLNoSp dst, iRegL src) %{
8078 match(Set dst (ReverseBytesL src));
8079
8080 ins_cost(INSN_COST);
8081 format %{ "rev $dst, $src" %}
8082
8083 ins_encode %{
8084 __ rev(as_Register($dst$$reg), as_Register($src$$reg));
8085 %}
8086
8087 ins_pipe(ialu_reg);
8088 %}
8089
8090 instruct bytes_reverse_unsigned_short(iRegINoSp dst, iRegIorL2I src) %{
8091 match(Set dst (ReverseBytesUS src));
8092
8093 ins_cost(INSN_COST);
8094 format %{ "rev16w $dst, $src" %}
8095
8096 ins_encode %{
8097 __ rev16w(as_Register($dst$$reg), as_Register($src$$reg));
8098 %}
8099
8100 ins_pipe(ialu_reg);
8101 %}
8102
8103 instruct bytes_reverse_short(iRegINoSp dst, iRegIorL2I src) %{
8104 match(Set dst (ReverseBytesS src));
8105
8106 ins_cost(INSN_COST);
8107 format %{ "rev16w $dst, $src\n\t"
8108 "sbfmw $dst, $dst, #0, #15" %}
8109
8110 ins_encode %{
8111 __ rev16w(as_Register($dst$$reg), as_Register($src$$reg));
8112 __ sbfmw(as_Register($dst$$reg), as_Register($dst$$reg), 0U, 15U);
8113 %}
8114
8115 ins_pipe(ialu_reg);
8116 %}
8117
8118 // ============================================================================
8119 // Zero Count Instructions
8120
8121 instruct countLeadingZerosI(iRegINoSp dst, iRegIorL2I src) %{
8122 match(Set dst (CountLeadingZerosI src));
8123
8124 ins_cost(INSN_COST);
8125 format %{ "clzw $dst, $src" %}
8126 ins_encode %{
8127 __ clzw(as_Register($dst$$reg), as_Register($src$$reg));
8128 %}
8129
8130 ins_pipe(ialu_reg);
8131 %}
8132
8133 instruct countLeadingZerosL(iRegINoSp dst, iRegL src) %{
8134 match(Set dst (CountLeadingZerosL src));
8135
8136 ins_cost(INSN_COST);
8137 format %{ "clz $dst, $src" %}
8138 ins_encode %{
8139 __ clz(as_Register($dst$$reg), as_Register($src$$reg));
8140 %}
8141
8142 ins_pipe(ialu_reg);
8143 %}
8144
8145 instruct countTrailingZerosI(iRegINoSp dst, iRegIorL2I src) %{
8146 match(Set dst (CountTrailingZerosI src));
8147
8148 ins_cost(INSN_COST * 2);
8149 format %{ "rbitw $dst, $src\n\t"
8150 "clzw $dst, $dst" %}
8151 ins_encode %{
8152 __ rbitw(as_Register($dst$$reg), as_Register($src$$reg));
8153 __ clzw(as_Register($dst$$reg), as_Register($dst$$reg));
8154 %}
8155
8156 ins_pipe(ialu_reg);
8157 %}
8158
8159 instruct countTrailingZerosL(iRegINoSp dst, iRegL src) %{
8160 match(Set dst (CountTrailingZerosL src));
8161
8162 ins_cost(INSN_COST * 2);
8163 format %{ "rbit $dst, $src\n\t"
8164 "clz $dst, $dst" %}
8165 ins_encode %{
8166 __ rbit(as_Register($dst$$reg), as_Register($src$$reg));
8167 __ clz(as_Register($dst$$reg), as_Register($dst$$reg));
8168 %}
8169
8170 ins_pipe(ialu_reg);
8171 %}
8172
8173 //---------- Population Count Instructions -------------------------------------
8174 //
8175
8176 instruct popCountI(iRegINoSp dst, iRegIorL2I src, vRegF tmp) %{
8177 match(Set dst (PopCountI src));
8178 effect(TEMP tmp);
8179 ins_cost(INSN_COST * 13);
8180
8181 format %{ "movw $src, $src\n\t"
8182 "mov $tmp, $src\t# vector (1D)\n\t"
8183 "cnt $tmp, $tmp\t# vector (8B)\n\t"
8184 "addv $tmp, $tmp\t# vector (8B)\n\t"
8185 "mov $dst, $tmp\t# vector (1D)" %}
8186 ins_encode %{
8187 __ movw($src$$Register, $src$$Register); // ensure top 32 bits 0
8188 __ mov($tmp$$FloatRegister, __ D, 0, $src$$Register);
8189 __ cnt($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
8190 __ addv($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
8191 __ mov($dst$$Register, $tmp$$FloatRegister, __ D, 0);
8192 %}
8193
8194 ins_pipe(pipe_class_default);
8195 %}
8196
8197 instruct popCountI_mem(iRegINoSp dst, memory4 mem, vRegF tmp) %{
8198 match(Set dst (PopCountI (LoadI mem)));
8199 effect(TEMP tmp);
8200 ins_cost(INSN_COST * 13);
8201
8202 format %{ "ldrs $tmp, $mem\n\t"
8203 "cnt $tmp, $tmp\t# vector (8B)\n\t"
8204 "addv $tmp, $tmp\t# vector (8B)\n\t"
8205 "mov $dst, $tmp\t# vector (1D)" %}
8206 ins_encode %{
8207 FloatRegister tmp_reg = as_FloatRegister($tmp$$reg);
8208 loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrs, tmp_reg, $mem->opcode(),
8209 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
8210 __ cnt($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
8211 __ addv($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
8212 __ mov($dst$$Register, $tmp$$FloatRegister, __ D, 0);
8213 %}
8214
8215 ins_pipe(pipe_class_default);
8216 %}
8217
8218 // Note: Long.bitCount(long) returns an int.
8219 instruct popCountL(iRegINoSp dst, iRegL src, vRegD tmp) %{
8220 match(Set dst (PopCountL src));
8221 effect(TEMP tmp);
8222 ins_cost(INSN_COST * 13);
8223
8224 format %{ "mov $tmp, $src\t# vector (1D)\n\t"
8225 "cnt $tmp, $tmp\t# vector (8B)\n\t"
8226 "addv $tmp, $tmp\t# vector (8B)\n\t"
8227 "mov $dst, $tmp\t# vector (1D)" %}
8228 ins_encode %{
8229 __ mov($tmp$$FloatRegister, __ D, 0, $src$$Register);
8230 __ cnt($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
8231 __ addv($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
8232 __ mov($dst$$Register, $tmp$$FloatRegister, __ D, 0);
8233 %}
8234
8235 ins_pipe(pipe_class_default);
8236 %}
8237
8238 instruct popCountL_mem(iRegINoSp dst, memory8 mem, vRegD tmp) %{
8239 match(Set dst (PopCountL (LoadL mem)));
8240 effect(TEMP tmp);
8241 ins_cost(INSN_COST * 13);
8242
8243 format %{ "ldrd $tmp, $mem\n\t"
8244 "cnt $tmp, $tmp\t# vector (8B)\n\t"
8245 "addv $tmp, $tmp\t# vector (8B)\n\t"
8246 "mov $dst, $tmp\t# vector (1D)" %}
8247 ins_encode %{
8248 FloatRegister tmp_reg = as_FloatRegister($tmp$$reg);
8249 loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrd, tmp_reg, $mem->opcode(),
8250 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
8251 __ cnt($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
8252 __ addv($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
8253 __ mov($dst$$Register, $tmp$$FloatRegister, __ D, 0);
8254 %}
8255
8256 ins_pipe(pipe_class_default);
8257 %}
8258
8259 // ============================================================================
8260 // MemBar Instruction
8261
8262 instruct load_fence() %{
8263 match(LoadFence);
8264 ins_cost(VOLATILE_REF_COST);
8265
8266 format %{ "load_fence" %}
8267
8268 ins_encode %{
8269 __ membar(Assembler::LoadLoad|Assembler::LoadStore);
8270 %}
8271 ins_pipe(pipe_serial);
8272 %}
8273
8274 instruct unnecessary_membar_acquire() %{
8275 predicate(unnecessary_acquire(n));
8276 match(MemBarAcquire);
8277 ins_cost(0);
8278
8279 format %{ "membar_acquire (elided)" %}
8280
8281 ins_encode %{
8282 __ block_comment("membar_acquire (elided)");
8283 %}
8284
8285 ins_pipe(pipe_class_empty);
8286 %}
8287
8288 instruct membar_acquire() %{
8289 match(MemBarAcquire);
8290 ins_cost(VOLATILE_REF_COST);
8291
8292 format %{ "membar_acquire\n\t"
8293 "dmb ish" %}
8294
8295 ins_encode %{
8296 __ block_comment("membar_acquire");
8297 __ membar(Assembler::LoadLoad|Assembler::LoadStore);
8298 %}
8299
8300 ins_pipe(pipe_serial);
8301 %}
8302
8303
8304 instruct membar_acquire_lock() %{
8305 match(MemBarAcquireLock);
8306 ins_cost(VOLATILE_REF_COST);
8307
8308 format %{ "membar_acquire_lock (elided)" %}
8309
8310 ins_encode %{
8311 __ block_comment("membar_acquire_lock (elided)");
8312 %}
8313
8314 ins_pipe(pipe_serial);
8315 %}
8316
8317 instruct store_fence() %{
8318 match(StoreFence);
8319 ins_cost(VOLATILE_REF_COST);
8320
8321 format %{ "store_fence" %}
8322
8323 ins_encode %{
8324 __ membar(Assembler::LoadStore|Assembler::StoreStore);
8325 %}
8326 ins_pipe(pipe_serial);
8327 %}
8328
8329 instruct unnecessary_membar_release() %{
8330 predicate(unnecessary_release(n));
8331 match(MemBarRelease);
8332 ins_cost(0);
8333
8334 format %{ "membar_release (elided)" %}
8335
8336 ins_encode %{
8337 __ block_comment("membar_release (elided)");
8338 %}
8339 ins_pipe(pipe_serial);
8340 %}
8341
8342 instruct membar_release() %{
8343 match(MemBarRelease);
8344 ins_cost(VOLATILE_REF_COST);
8345
8346 format %{ "membar_release\n\t"
8347 "dmb ish" %}
8348
8349 ins_encode %{
8350 __ block_comment("membar_release");
8351 __ membar(Assembler::LoadStore|Assembler::StoreStore);
8352 %}
8353 ins_pipe(pipe_serial);
8354 %}
8355
8356 instruct membar_storestore() %{
8357 match(MemBarStoreStore);
8358 match(StoreStoreFence);
8359 ins_cost(VOLATILE_REF_COST);
8360
8361 format %{ "MEMBAR-store-store" %}
8362
8363 ins_encode %{
8364 __ membar(Assembler::StoreStore);
8365 %}
8366 ins_pipe(pipe_serial);
8367 %}
8368
8369 instruct membar_release_lock() %{
8370 match(MemBarReleaseLock);
8371 ins_cost(VOLATILE_REF_COST);
8372
8373 format %{ "membar_release_lock (elided)" %}
8374
8375 ins_encode %{
8376 __ block_comment("membar_release_lock (elided)");
8377 %}
8378
8379 ins_pipe(pipe_serial);
8380 %}
8381
8382 instruct unnecessary_membar_volatile() %{
8383 predicate(unnecessary_volatile(n));
8384 match(MemBarVolatile);
8385 ins_cost(0);
8386
8387 format %{ "membar_volatile (elided)" %}
8388
8389 ins_encode %{
8390 __ block_comment("membar_volatile (elided)");
8391 %}
8392
8393 ins_pipe(pipe_serial);
8394 %}
8395
8396 instruct membar_volatile() %{
8397 match(MemBarVolatile);
8398 ins_cost(VOLATILE_REF_COST*100);
8399
8400 format %{ "membar_volatile\n\t"
8401 "dmb ish"%}
8402
8403 ins_encode %{
8404 __ block_comment("membar_volatile");
8405 __ membar(Assembler::StoreLoad);
8406 %}
8407
8408 ins_pipe(pipe_serial);
8409 %}
8410
8411 // ============================================================================
8412 // Cast/Convert Instructions
8413
8414 instruct castX2P(iRegPNoSp dst, iRegL src) %{
8415 match(Set dst (CastX2P src));
8416
8417 ins_cost(INSN_COST);
8418 format %{ "mov $dst, $src\t# long -> ptr" %}
8419
8420 ins_encode %{
8421 if ($dst$$reg != $src$$reg) {
8422 __ mov(as_Register($dst$$reg), as_Register($src$$reg));
8423 }
8424 %}
8425
8426 ins_pipe(ialu_reg);
8427 %}
8428
8429 instruct castP2X(iRegLNoSp dst, iRegP src) %{
8430 match(Set dst (CastP2X src));
8431
8432 ins_cost(INSN_COST);
8433 format %{ "mov $dst, $src\t# ptr -> long" %}
8434
8435 ins_encode %{
8436 if ($dst$$reg != $src$$reg) {
8437 __ mov(as_Register($dst$$reg), as_Register($src$$reg));
8438 }
8439 %}
8440
8441 ins_pipe(ialu_reg);
8442 %}
8443
8444 // Convert oop into int for vectors alignment masking
8445 instruct convP2I(iRegINoSp dst, iRegP src) %{
8446 match(Set dst (ConvL2I (CastP2X src)));
8447
8448 ins_cost(INSN_COST);
8449 format %{ "movw $dst, $src\t# ptr -> int" %}
8450 ins_encode %{
8451 __ movw($dst$$Register, $src$$Register);
8452 %}
8453
8454 ins_pipe(ialu_reg);
8455 %}
8456
8457 // Convert compressed oop into int for vectors alignment masking
8458 // in case of 32bit oops (heap < 4Gb).
8459 instruct convN2I(iRegINoSp dst, iRegN src)
8460 %{
8461 predicate(CompressedOops::shift() == 0);
8462 match(Set dst (ConvL2I (CastP2X (DecodeN src))));
8463
8464 ins_cost(INSN_COST);
8465 format %{ "mov dst, $src\t# compressed ptr -> int" %}
8466 ins_encode %{
8467 __ movw($dst$$Register, $src$$Register);
8468 %}
8469
8470 ins_pipe(ialu_reg);
8471 %}
8472
8473
8474 // Convert oop pointer into compressed form
8475 instruct encodeHeapOop(iRegNNoSp dst, iRegP src, rFlagsReg cr) %{
8476 predicate(n->bottom_type()->make_ptr()->ptr() != TypePtr::NotNull);
8477 match(Set dst (EncodeP src));
8478 effect(KILL cr);
8479 ins_cost(INSN_COST * 3);
8480 format %{ "encode_heap_oop $dst, $src" %}
8481 ins_encode %{
8482 Register s = $src$$Register;
8483 Register d = $dst$$Register;
8484 __ encode_heap_oop(d, s);
8485 %}
8486 ins_pipe(ialu_reg);
8487 %}
8488
8489 instruct encodeHeapOop_not_null(iRegNNoSp dst, iRegP src, rFlagsReg cr) %{
8490 predicate(n->bottom_type()->make_ptr()->ptr() == TypePtr::NotNull);
8491 match(Set dst (EncodeP src));
8492 ins_cost(INSN_COST * 3);
8493 format %{ "encode_heap_oop_not_null $dst, $src" %}
8494 ins_encode %{
8495 __ encode_heap_oop_not_null($dst$$Register, $src$$Register);
8496 %}
8497 ins_pipe(ialu_reg);
8498 %}
8499
8500 instruct decodeHeapOop(iRegPNoSp dst, iRegN src, rFlagsReg cr) %{
8501 predicate(n->bottom_type()->is_ptr()->ptr() != TypePtr::NotNull &&
8502 n->bottom_type()->is_ptr()->ptr() != TypePtr::Constant);
8503 match(Set dst (DecodeN src));
8504 ins_cost(INSN_COST * 3);
8505 format %{ "decode_heap_oop $dst, $src" %}
8506 ins_encode %{
8507 Register s = $src$$Register;
8508 Register d = $dst$$Register;
8509 __ decode_heap_oop(d, s);
8510 %}
8511 ins_pipe(ialu_reg);
8512 %}
8513
8514 instruct decodeHeapOop_not_null(iRegPNoSp dst, iRegN src, rFlagsReg cr) %{
8515 predicate(n->bottom_type()->is_ptr()->ptr() == TypePtr::NotNull ||
8516 n->bottom_type()->is_ptr()->ptr() == TypePtr::Constant);
8517 match(Set dst (DecodeN src));
8518 ins_cost(INSN_COST * 3);
8519 format %{ "decode_heap_oop_not_null $dst, $src" %}
8520 ins_encode %{
8521 Register s = $src$$Register;
8522 Register d = $dst$$Register;
8523 __ decode_heap_oop_not_null(d, s);
8524 %}
8525 ins_pipe(ialu_reg);
8526 %}
8527
8528 // n.b. AArch64 implementations of encode_klass_not_null and
8529 // decode_klass_not_null do not modify the flags register so, unlike
8530 // Intel, we don't kill CR as a side effect here
8531
8532 instruct encodeKlass_not_null(iRegNNoSp dst, iRegP src) %{
8533 match(Set dst (EncodePKlass src));
8534
8535 ins_cost(INSN_COST * 3);
8536 format %{ "encode_klass_not_null $dst,$src" %}
8537
8538 ins_encode %{
8539 Register src_reg = as_Register($src$$reg);
8540 Register dst_reg = as_Register($dst$$reg);
8541 __ encode_klass_not_null(dst_reg, src_reg);
8542 %}
8543
8544 ins_pipe(ialu_reg);
8545 %}
8546
8547 instruct decodeKlass_not_null(iRegPNoSp dst, iRegN src) %{
8548 match(Set dst (DecodeNKlass src));
8549
8550 ins_cost(INSN_COST * 3);
8551 format %{ "decode_klass_not_null $dst,$src" %}
8552
8553 ins_encode %{
8554 Register src_reg = as_Register($src$$reg);
8555 Register dst_reg = as_Register($dst$$reg);
8556 if (dst_reg != src_reg) {
8557 __ decode_klass_not_null(dst_reg, src_reg);
8558 } else {
8559 __ decode_klass_not_null(dst_reg);
8560 }
8561 %}
8562
8563 ins_pipe(ialu_reg);
8564 %}
8565
8566 instruct checkCastPP(iRegPNoSp dst)
8567 %{
8568 match(Set dst (CheckCastPP dst));
8569
8570 size(0);
8571 format %{ "# checkcastPP of $dst" %}
8572 ins_encode(/* empty encoding */);
8573 ins_pipe(pipe_class_empty);
8574 %}
8575
8576 instruct castPP(iRegPNoSp dst)
8577 %{
8578 match(Set dst (CastPP dst));
8579
8580 size(0);
8581 format %{ "# castPP of $dst" %}
8582 ins_encode(/* empty encoding */);
8583 ins_pipe(pipe_class_empty);
8584 %}
8585
8586 instruct castII(iRegI dst)
8587 %{
8588 match(Set dst (CastII dst));
8589
8590 size(0);
8591 format %{ "# castII of $dst" %}
8592 ins_encode(/* empty encoding */);
8593 ins_cost(0);
8594 ins_pipe(pipe_class_empty);
8595 %}
8596
8597 instruct castLL(iRegL dst)
8598 %{
8599 match(Set dst (CastLL dst));
8600
8601 size(0);
8602 format %{ "# castLL of $dst" %}
8603 ins_encode(/* empty encoding */);
8604 ins_cost(0);
8605 ins_pipe(pipe_class_empty);
8606 %}
8607
8608 instruct castFF(vRegF dst)
8609 %{
8610 match(Set dst (CastFF dst));
8611
8612 size(0);
8613 format %{ "# castFF of $dst" %}
8614 ins_encode(/* empty encoding */);
8615 ins_cost(0);
8616 ins_pipe(pipe_class_empty);
8617 %}
8618
8619 instruct castDD(vRegD dst)
8620 %{
8621 match(Set dst (CastDD dst));
8622
8623 size(0);
8624 format %{ "# castDD of $dst" %}
8625 ins_encode(/* empty encoding */);
8626 ins_cost(0);
8627 ins_pipe(pipe_class_empty);
8628 %}
8629
8630 instruct castVV(vReg dst)
8631 %{
8632 match(Set dst (CastVV dst));
8633
8634 size(0);
8635 format %{ "# castVV of $dst" %}
8636 ins_encode(/* empty encoding */);
8637 ins_cost(0);
8638 ins_pipe(pipe_class_empty);
8639 %}
8640
8641 instruct castVVMask(pRegGov dst)
8642 %{
8643 match(Set dst (CastVV dst));
8644
8645 size(0);
8646 format %{ "# castVV of $dst" %}
8647 ins_encode(/* empty encoding */);
8648 ins_cost(0);
8649 ins_pipe(pipe_class_empty);
8650 %}
8651
8652 // ============================================================================
8653 // Atomic operation instructions
8654 //
8655
8656 // standard CompareAndSwapX when we are using barriers
8657 // these have higher priority than the rules selected by a predicate
8658
8659 // XXX No flag versions for CompareAndSwap{I,L,P,N} because matcher
8660 // can't match them
8661
8662 instruct compareAndSwapB(iRegINoSp res, indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr) %{
8663
8664 match(Set res (CompareAndSwapB mem (Binary oldval newval)));
8665 ins_cost(2 * VOLATILE_REF_COST);
8666
8667 effect(KILL cr);
8668
8669 format %{
8670 "cmpxchgb $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval"
8671 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8672 %}
8673
8674 ins_encode(aarch64_enc_cmpxchgb(mem, oldval, newval),
8675 aarch64_enc_cset_eq(res));
8676
8677 ins_pipe(pipe_slow);
8678 %}
8679
8680 instruct compareAndSwapS(iRegINoSp res, indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr) %{
8681
8682 match(Set res (CompareAndSwapS mem (Binary oldval newval)));
8683 ins_cost(2 * VOLATILE_REF_COST);
8684
8685 effect(KILL cr);
8686
8687 format %{
8688 "cmpxchgs $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval"
8689 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8690 %}
8691
8692 ins_encode(aarch64_enc_cmpxchgs(mem, oldval, newval),
8693 aarch64_enc_cset_eq(res));
8694
8695 ins_pipe(pipe_slow);
8696 %}
8697
8698 instruct compareAndSwapI(iRegINoSp res, indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr) %{
8699
8700 match(Set res (CompareAndSwapI mem (Binary oldval newval)));
8701 ins_cost(2 * VOLATILE_REF_COST);
8702
8703 effect(KILL cr);
8704
8705 format %{
8706 "cmpxchgw $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval"
8707 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8708 %}
8709
8710 ins_encode(aarch64_enc_cmpxchgw(mem, oldval, newval),
8711 aarch64_enc_cset_eq(res));
8712
8713 ins_pipe(pipe_slow);
8714 %}
8715
8716 instruct compareAndSwapL(iRegINoSp res, indirect mem, iRegLNoSp oldval, iRegLNoSp newval, rFlagsReg cr) %{
8717
8718 match(Set res (CompareAndSwapL mem (Binary oldval newval)));
8719 ins_cost(2 * VOLATILE_REF_COST);
8720
8721 effect(KILL cr);
8722
8723 format %{
8724 "cmpxchg $mem, $oldval, $newval\t# (long) if $mem == $oldval then $mem <-- $newval"
8725 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8726 %}
8727
8728 ins_encode(aarch64_enc_cmpxchg(mem, oldval, newval),
8729 aarch64_enc_cset_eq(res));
8730
8731 ins_pipe(pipe_slow);
8732 %}
8733
8734 instruct compareAndSwapP(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
8735
8736 match(Set res (CompareAndSwapP mem (Binary oldval newval)));
8737 predicate(n->as_LoadStore()->barrier_data() == 0);
8738 ins_cost(2 * VOLATILE_REF_COST);
8739
8740 effect(KILL cr);
8741
8742 format %{
8743 "cmpxchg $mem, $oldval, $newval\t# (ptr) if $mem == $oldval then $mem <-- $newval"
8744 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8745 %}
8746
8747 ins_encode(aarch64_enc_cmpxchg(mem, oldval, newval),
8748 aarch64_enc_cset_eq(res));
8749
8750 ins_pipe(pipe_slow);
8751 %}
8752
8753 instruct compareAndSwapN(iRegINoSp res, indirect mem, iRegNNoSp oldval, iRegNNoSp newval, rFlagsReg cr) %{
8754
8755 match(Set res (CompareAndSwapN mem (Binary oldval newval)));
8756 ins_cost(2 * VOLATILE_REF_COST);
8757
8758 effect(KILL cr);
8759
8760 format %{
8761 "cmpxchgw $mem, $oldval, $newval\t# (narrow oop) if $mem == $oldval then $mem <-- $newval"
8762 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8763 %}
8764
8765 ins_encode(aarch64_enc_cmpxchgw(mem, oldval, newval),
8766 aarch64_enc_cset_eq(res));
8767
8768 ins_pipe(pipe_slow);
8769 %}
8770
8771 // alternative CompareAndSwapX when we are eliding barriers
8772
8773 instruct compareAndSwapBAcq(iRegINoSp res, indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr) %{
8774
8775 predicate(needs_acquiring_load_exclusive(n));
8776 match(Set res (CompareAndSwapB mem (Binary oldval newval)));
8777 ins_cost(VOLATILE_REF_COST);
8778
8779 effect(KILL cr);
8780
8781 format %{
8782 "cmpxchgb_acq $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval"
8783 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8784 %}
8785
8786 ins_encode(aarch64_enc_cmpxchgb_acq(mem, oldval, newval),
8787 aarch64_enc_cset_eq(res));
8788
8789 ins_pipe(pipe_slow);
8790 %}
8791
8792 instruct compareAndSwapSAcq(iRegINoSp res, indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr) %{
8793
8794 predicate(needs_acquiring_load_exclusive(n));
8795 match(Set res (CompareAndSwapS mem (Binary oldval newval)));
8796 ins_cost(VOLATILE_REF_COST);
8797
8798 effect(KILL cr);
8799
8800 format %{
8801 "cmpxchgs_acq $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval"
8802 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8803 %}
8804
8805 ins_encode(aarch64_enc_cmpxchgs_acq(mem, oldval, newval),
8806 aarch64_enc_cset_eq(res));
8807
8808 ins_pipe(pipe_slow);
8809 %}
8810
8811 instruct compareAndSwapIAcq(iRegINoSp res, indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr) %{
8812
8813 predicate(needs_acquiring_load_exclusive(n));
8814 match(Set res (CompareAndSwapI mem (Binary oldval newval)));
8815 ins_cost(VOLATILE_REF_COST);
8816
8817 effect(KILL cr);
8818
8819 format %{
8820 "cmpxchgw_acq $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval"
8821 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8822 %}
8823
8824 ins_encode(aarch64_enc_cmpxchgw_acq(mem, oldval, newval),
8825 aarch64_enc_cset_eq(res));
8826
8827 ins_pipe(pipe_slow);
8828 %}
8829
8830 instruct compareAndSwapLAcq(iRegINoSp res, indirect mem, iRegLNoSp oldval, iRegLNoSp newval, rFlagsReg cr) %{
8831
8832 predicate(needs_acquiring_load_exclusive(n));
8833 match(Set res (CompareAndSwapL mem (Binary oldval newval)));
8834 ins_cost(VOLATILE_REF_COST);
8835
8836 effect(KILL cr);
8837
8838 format %{
8839 "cmpxchg_acq $mem, $oldval, $newval\t# (long) if $mem == $oldval then $mem <-- $newval"
8840 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8841 %}
8842
8843 ins_encode(aarch64_enc_cmpxchg_acq(mem, oldval, newval),
8844 aarch64_enc_cset_eq(res));
8845
8846 ins_pipe(pipe_slow);
8847 %}
8848
8849 instruct compareAndSwapPAcq(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
8850
8851 predicate(needs_acquiring_load_exclusive(n) && (n->as_LoadStore()->barrier_data() == 0));
8852 match(Set res (CompareAndSwapP mem (Binary oldval newval)));
8853 ins_cost(VOLATILE_REF_COST);
8854
8855 effect(KILL cr);
8856
8857 format %{
8858 "cmpxchg_acq $mem, $oldval, $newval\t# (ptr) if $mem == $oldval then $mem <-- $newval"
8859 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8860 %}
8861
8862 ins_encode(aarch64_enc_cmpxchg_acq(mem, oldval, newval),
8863 aarch64_enc_cset_eq(res));
8864
8865 ins_pipe(pipe_slow);
8866 %}
8867
8868 instruct compareAndSwapNAcq(iRegINoSp res, indirect mem, iRegNNoSp oldval, iRegNNoSp newval, rFlagsReg cr) %{
8869
8870 predicate(needs_acquiring_load_exclusive(n));
8871 match(Set res (CompareAndSwapN mem (Binary oldval newval)));
8872 ins_cost(VOLATILE_REF_COST);
8873
8874 effect(KILL cr);
8875
8876 format %{
8877 "cmpxchgw_acq $mem, $oldval, $newval\t# (narrow oop) if $mem == $oldval then $mem <-- $newval"
8878 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8879 %}
8880
8881 ins_encode(aarch64_enc_cmpxchgw_acq(mem, oldval, newval),
8882 aarch64_enc_cset_eq(res));
8883
8884 ins_pipe(pipe_slow);
8885 %}
8886
8887
8888 // ---------------------------------------------------------------------
8889
8890 // BEGIN This section of the file is automatically generated. Do not edit --------------
8891
8892 // Sundry CAS operations. Note that release is always true,
8893 // regardless of the memory ordering of the CAS. This is because we
8894 // need the volatile case to be sequentially consistent but there is
8895 // no trailing StoreLoad barrier emitted by C2. Unfortunately we
8896 // can't check the type of memory ordering here, so we always emit a
8897 // STLXR.
8898
8899 // This section is generated from aarch64_ad_cas.m4
8900
8901
8902
8903 // This pattern is generated automatically from cas.m4.
8904 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8905 instruct compareAndExchangeB(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
8906
8907 match(Set res (CompareAndExchangeB mem (Binary oldval newval)));
8908 ins_cost(2 * VOLATILE_REF_COST);
8909 effect(TEMP_DEF res, KILL cr);
8910 format %{
8911 "cmpxchgb $res = $mem, $oldval, $newval\t# (byte, weak) if $mem == $oldval then $mem <-- $newval"
8912 %}
8913 ins_encode %{
8914 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8915 Assembler::byte, /*acquire*/ false, /*release*/ true,
8916 /*weak*/ false, $res$$Register);
8917 __ sxtbw($res$$Register, $res$$Register);
8918 %}
8919 ins_pipe(pipe_slow);
8920 %}
8921
8922 // This pattern is generated automatically from cas.m4.
8923 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8924 instruct compareAndExchangeS(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
8925
8926 match(Set res (CompareAndExchangeS mem (Binary oldval newval)));
8927 ins_cost(2 * VOLATILE_REF_COST);
8928 effect(TEMP_DEF res, KILL cr);
8929 format %{
8930 "cmpxchgs $res = $mem, $oldval, $newval\t# (short, weak) if $mem == $oldval then $mem <-- $newval"
8931 %}
8932 ins_encode %{
8933 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8934 Assembler::halfword, /*acquire*/ false, /*release*/ true,
8935 /*weak*/ false, $res$$Register);
8936 __ sxthw($res$$Register, $res$$Register);
8937 %}
8938 ins_pipe(pipe_slow);
8939 %}
8940
8941 // This pattern is generated automatically from cas.m4.
8942 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8943 instruct compareAndExchangeI(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
8944
8945 match(Set res (CompareAndExchangeI mem (Binary oldval newval)));
8946 ins_cost(2 * VOLATILE_REF_COST);
8947 effect(TEMP_DEF res, KILL cr);
8948 format %{
8949 "cmpxchgw $res = $mem, $oldval, $newval\t# (int, weak) if $mem == $oldval then $mem <-- $newval"
8950 %}
8951 ins_encode %{
8952 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8953 Assembler::word, /*acquire*/ false, /*release*/ true,
8954 /*weak*/ false, $res$$Register);
8955 %}
8956 ins_pipe(pipe_slow);
8957 %}
8958
8959 // This pattern is generated automatically from cas.m4.
8960 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8961 instruct compareAndExchangeL(iRegLNoSp res, indirect mem, iRegL oldval, iRegL newval, rFlagsReg cr) %{
8962
8963 match(Set res (CompareAndExchangeL mem (Binary oldval newval)));
8964 ins_cost(2 * VOLATILE_REF_COST);
8965 effect(TEMP_DEF res, KILL cr);
8966 format %{
8967 "cmpxchg $res = $mem, $oldval, $newval\t# (long, weak) if $mem == $oldval then $mem <-- $newval"
8968 %}
8969 ins_encode %{
8970 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8971 Assembler::xword, /*acquire*/ false, /*release*/ true,
8972 /*weak*/ false, $res$$Register);
8973 %}
8974 ins_pipe(pipe_slow);
8975 %}
8976
8977 // This pattern is generated automatically from cas.m4.
8978 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8979 instruct compareAndExchangeN(iRegNNoSp res, indirect mem, iRegN oldval, iRegN newval, rFlagsReg cr) %{
8980
8981 match(Set res (CompareAndExchangeN mem (Binary oldval newval)));
8982 ins_cost(2 * VOLATILE_REF_COST);
8983 effect(TEMP_DEF res, KILL cr);
8984 format %{
8985 "cmpxchgw $res = $mem, $oldval, $newval\t# (narrow oop, weak) if $mem == $oldval then $mem <-- $newval"
8986 %}
8987 ins_encode %{
8988 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8989 Assembler::word, /*acquire*/ false, /*release*/ true,
8990 /*weak*/ false, $res$$Register);
8991 %}
8992 ins_pipe(pipe_slow);
8993 %}
8994
8995 // This pattern is generated automatically from cas.m4.
8996 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8997 instruct compareAndExchangeP(iRegPNoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
8998 predicate(n->as_LoadStore()->barrier_data() == 0);
8999 match(Set res (CompareAndExchangeP mem (Binary oldval newval)));
9000 ins_cost(2 * VOLATILE_REF_COST);
9001 effect(TEMP_DEF res, KILL cr);
9002 format %{
9003 "cmpxchg $res = $mem, $oldval, $newval\t# (ptr, weak) if $mem == $oldval then $mem <-- $newval"
9004 %}
9005 ins_encode %{
9006 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
9007 Assembler::xword, /*acquire*/ false, /*release*/ true,
9008 /*weak*/ false, $res$$Register);
9009 %}
9010 ins_pipe(pipe_slow);
9011 %}
9012
9013 // This pattern is generated automatically from cas.m4.
9014 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9015 instruct compareAndExchangeBAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
9016 predicate(needs_acquiring_load_exclusive(n));
9017 match(Set res (CompareAndExchangeB mem (Binary oldval newval)));
9018 ins_cost(VOLATILE_REF_COST);
9019 effect(TEMP_DEF res, KILL cr);
9020 format %{
9021 "cmpxchgb_acq $res = $mem, $oldval, $newval\t# (byte, weak) if $mem == $oldval then $mem <-- $newval"
9022 %}
9023 ins_encode %{
9024 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
9025 Assembler::byte, /*acquire*/ true, /*release*/ true,
9026 /*weak*/ false, $res$$Register);
9027 __ sxtbw($res$$Register, $res$$Register);
9028 %}
9029 ins_pipe(pipe_slow);
9030 %}
9031
9032 // This pattern is generated automatically from cas.m4.
9033 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9034 instruct compareAndExchangeSAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
9035 predicate(needs_acquiring_load_exclusive(n));
9036 match(Set res (CompareAndExchangeS mem (Binary oldval newval)));
9037 ins_cost(VOLATILE_REF_COST);
9038 effect(TEMP_DEF res, KILL cr);
9039 format %{
9040 "cmpxchgs_acq $res = $mem, $oldval, $newval\t# (short, weak) if $mem == $oldval then $mem <-- $newval"
9041 %}
9042 ins_encode %{
9043 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
9044 Assembler::halfword, /*acquire*/ true, /*release*/ true,
9045 /*weak*/ false, $res$$Register);
9046 __ sxthw($res$$Register, $res$$Register);
9047 %}
9048 ins_pipe(pipe_slow);
9049 %}
9050
9051 // This pattern is generated automatically from cas.m4.
9052 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9053 instruct compareAndExchangeIAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
9054 predicate(needs_acquiring_load_exclusive(n));
9055 match(Set res (CompareAndExchangeI mem (Binary oldval newval)));
9056 ins_cost(VOLATILE_REF_COST);
9057 effect(TEMP_DEF res, KILL cr);
9058 format %{
9059 "cmpxchgw_acq $res = $mem, $oldval, $newval\t# (int, weak) if $mem == $oldval then $mem <-- $newval"
9060 %}
9061 ins_encode %{
9062 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
9063 Assembler::word, /*acquire*/ true, /*release*/ true,
9064 /*weak*/ false, $res$$Register);
9065 %}
9066 ins_pipe(pipe_slow);
9067 %}
9068
9069 // This pattern is generated automatically from cas.m4.
9070 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9071 instruct compareAndExchangeLAcq(iRegLNoSp res, indirect mem, iRegL oldval, iRegL newval, rFlagsReg cr) %{
9072 predicate(needs_acquiring_load_exclusive(n));
9073 match(Set res (CompareAndExchangeL mem (Binary oldval newval)));
9074 ins_cost(VOLATILE_REF_COST);
9075 effect(TEMP_DEF res, KILL cr);
9076 format %{
9077 "cmpxchg_acq $res = $mem, $oldval, $newval\t# (long, weak) if $mem == $oldval then $mem <-- $newval"
9078 %}
9079 ins_encode %{
9080 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
9081 Assembler::xword, /*acquire*/ true, /*release*/ true,
9082 /*weak*/ false, $res$$Register);
9083 %}
9084 ins_pipe(pipe_slow);
9085 %}
9086
9087 // This pattern is generated automatically from cas.m4.
9088 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9089 instruct compareAndExchangeNAcq(iRegNNoSp res, indirect mem, iRegN oldval, iRegN newval, rFlagsReg cr) %{
9090 predicate(needs_acquiring_load_exclusive(n));
9091 match(Set res (CompareAndExchangeN mem (Binary oldval newval)));
9092 ins_cost(VOLATILE_REF_COST);
9093 effect(TEMP_DEF res, KILL cr);
9094 format %{
9095 "cmpxchgw_acq $res = $mem, $oldval, $newval\t# (narrow oop, weak) if $mem == $oldval then $mem <-- $newval"
9096 %}
9097 ins_encode %{
9098 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
9099 Assembler::word, /*acquire*/ true, /*release*/ true,
9100 /*weak*/ false, $res$$Register);
9101 %}
9102 ins_pipe(pipe_slow);
9103 %}
9104
9105 // This pattern is generated automatically from cas.m4.
9106 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9107 instruct compareAndExchangePAcq(iRegPNoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
9108 predicate(needs_acquiring_load_exclusive(n) && (n->as_LoadStore()->barrier_data() == 0));
9109 match(Set res (CompareAndExchangeP mem (Binary oldval newval)));
9110 ins_cost(VOLATILE_REF_COST);
9111 effect(TEMP_DEF res, KILL cr);
9112 format %{
9113 "cmpxchg_acq $res = $mem, $oldval, $newval\t# (ptr, weak) if $mem == $oldval then $mem <-- $newval"
9114 %}
9115 ins_encode %{
9116 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
9117 Assembler::xword, /*acquire*/ true, /*release*/ true,
9118 /*weak*/ false, $res$$Register);
9119 %}
9120 ins_pipe(pipe_slow);
9121 %}
9122
9123 // This pattern is generated automatically from cas.m4.
9124 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9125 instruct weakCompareAndSwapB(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
9126
9127 match(Set res (WeakCompareAndSwapB mem (Binary oldval newval)));
9128 ins_cost(2 * VOLATILE_REF_COST);
9129 effect(KILL cr);
9130 format %{
9131 "cmpxchgb $res = $mem, $oldval, $newval\t# (byte, weak) if $mem == $oldval then $mem <-- $newval"
9132 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
9133 %}
9134 ins_encode %{
9135 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
9136 Assembler::byte, /*acquire*/ false, /*release*/ true,
9137 /*weak*/ true, noreg);
9138 __ csetw($res$$Register, Assembler::EQ);
9139 %}
9140 ins_pipe(pipe_slow);
9141 %}
9142
9143 // This pattern is generated automatically from cas.m4.
9144 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9145 instruct weakCompareAndSwapS(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
9146
9147 match(Set res (WeakCompareAndSwapS mem (Binary oldval newval)));
9148 ins_cost(2 * VOLATILE_REF_COST);
9149 effect(KILL cr);
9150 format %{
9151 "cmpxchgs $res = $mem, $oldval, $newval\t# (short, weak) if $mem == $oldval then $mem <-- $newval"
9152 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
9153 %}
9154 ins_encode %{
9155 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
9156 Assembler::halfword, /*acquire*/ false, /*release*/ true,
9157 /*weak*/ true, noreg);
9158 __ csetw($res$$Register, Assembler::EQ);
9159 %}
9160 ins_pipe(pipe_slow);
9161 %}
9162
9163 // This pattern is generated automatically from cas.m4.
9164 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9165 instruct weakCompareAndSwapI(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
9166
9167 match(Set res (WeakCompareAndSwapI mem (Binary oldval newval)));
9168 ins_cost(2 * VOLATILE_REF_COST);
9169 effect(KILL cr);
9170 format %{
9171 "cmpxchgw $res = $mem, $oldval, $newval\t# (int, weak) if $mem == $oldval then $mem <-- $newval"
9172 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
9173 %}
9174 ins_encode %{
9175 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
9176 Assembler::word, /*acquire*/ false, /*release*/ true,
9177 /*weak*/ true, noreg);
9178 __ csetw($res$$Register, Assembler::EQ);
9179 %}
9180 ins_pipe(pipe_slow);
9181 %}
9182
9183 // This pattern is generated automatically from cas.m4.
9184 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9185 instruct weakCompareAndSwapL(iRegINoSp res, indirect mem, iRegL oldval, iRegL newval, rFlagsReg cr) %{
9186
9187 match(Set res (WeakCompareAndSwapL mem (Binary oldval newval)));
9188 ins_cost(2 * VOLATILE_REF_COST);
9189 effect(KILL cr);
9190 format %{
9191 "cmpxchg $res = $mem, $oldval, $newval\t# (long, weak) if $mem == $oldval then $mem <-- $newval"
9192 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
9193 %}
9194 ins_encode %{
9195 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
9196 Assembler::xword, /*acquire*/ false, /*release*/ true,
9197 /*weak*/ true, noreg);
9198 __ csetw($res$$Register, Assembler::EQ);
9199 %}
9200 ins_pipe(pipe_slow);
9201 %}
9202
9203 // This pattern is generated automatically from cas.m4.
9204 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9205 instruct weakCompareAndSwapN(iRegINoSp res, indirect mem, iRegN oldval, iRegN newval, rFlagsReg cr) %{
9206
9207 match(Set res (WeakCompareAndSwapN mem (Binary oldval newval)));
9208 ins_cost(2 * VOLATILE_REF_COST);
9209 effect(KILL cr);
9210 format %{
9211 "cmpxchgw $res = $mem, $oldval, $newval\t# (narrow oop, weak) if $mem == $oldval then $mem <-- $newval"
9212 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
9213 %}
9214 ins_encode %{
9215 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
9216 Assembler::word, /*acquire*/ false, /*release*/ true,
9217 /*weak*/ true, noreg);
9218 __ csetw($res$$Register, Assembler::EQ);
9219 %}
9220 ins_pipe(pipe_slow);
9221 %}
9222
9223 // This pattern is generated automatically from cas.m4.
9224 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9225 instruct weakCompareAndSwapP(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
9226 predicate(n->as_LoadStore()->barrier_data() == 0);
9227 match(Set res (WeakCompareAndSwapP mem (Binary oldval newval)));
9228 ins_cost(2 * VOLATILE_REF_COST);
9229 effect(KILL cr);
9230 format %{
9231 "cmpxchg $res = $mem, $oldval, $newval\t# (ptr, weak) if $mem == $oldval then $mem <-- $newval"
9232 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
9233 %}
9234 ins_encode %{
9235 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
9236 Assembler::xword, /*acquire*/ false, /*release*/ true,
9237 /*weak*/ true, noreg);
9238 __ csetw($res$$Register, Assembler::EQ);
9239 %}
9240 ins_pipe(pipe_slow);
9241 %}
9242
9243 // This pattern is generated automatically from cas.m4.
9244 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9245 instruct weakCompareAndSwapBAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
9246 predicate(needs_acquiring_load_exclusive(n));
9247 match(Set res (WeakCompareAndSwapB mem (Binary oldval newval)));
9248 ins_cost(VOLATILE_REF_COST);
9249 effect(KILL cr);
9250 format %{
9251 "cmpxchgb_acq $res = $mem, $oldval, $newval\t# (byte, weak) if $mem == $oldval then $mem <-- $newval"
9252 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
9253 %}
9254 ins_encode %{
9255 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
9256 Assembler::byte, /*acquire*/ true, /*release*/ true,
9257 /*weak*/ true, noreg);
9258 __ csetw($res$$Register, Assembler::EQ);
9259 %}
9260 ins_pipe(pipe_slow);
9261 %}
9262
9263 // This pattern is generated automatically from cas.m4.
9264 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9265 instruct weakCompareAndSwapSAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
9266 predicate(needs_acquiring_load_exclusive(n));
9267 match(Set res (WeakCompareAndSwapS mem (Binary oldval newval)));
9268 ins_cost(VOLATILE_REF_COST);
9269 effect(KILL cr);
9270 format %{
9271 "cmpxchgs_acq $res = $mem, $oldval, $newval\t# (short, weak) if $mem == $oldval then $mem <-- $newval"
9272 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
9273 %}
9274 ins_encode %{
9275 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
9276 Assembler::halfword, /*acquire*/ true, /*release*/ true,
9277 /*weak*/ true, noreg);
9278 __ csetw($res$$Register, Assembler::EQ);
9279 %}
9280 ins_pipe(pipe_slow);
9281 %}
9282
9283 // This pattern is generated automatically from cas.m4.
9284 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9285 instruct weakCompareAndSwapIAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
9286 predicate(needs_acquiring_load_exclusive(n));
9287 match(Set res (WeakCompareAndSwapI mem (Binary oldval newval)));
9288 ins_cost(VOLATILE_REF_COST);
9289 effect(KILL cr);
9290 format %{
9291 "cmpxchgw_acq $res = $mem, $oldval, $newval\t# (int, weak) if $mem == $oldval then $mem <-- $newval"
9292 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
9293 %}
9294 ins_encode %{
9295 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
9296 Assembler::word, /*acquire*/ true, /*release*/ true,
9297 /*weak*/ true, noreg);
9298 __ csetw($res$$Register, Assembler::EQ);
9299 %}
9300 ins_pipe(pipe_slow);
9301 %}
9302
9303 // This pattern is generated automatically from cas.m4.
9304 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9305 instruct weakCompareAndSwapLAcq(iRegINoSp res, indirect mem, iRegL oldval, iRegL newval, rFlagsReg cr) %{
9306 predicate(needs_acquiring_load_exclusive(n));
9307 match(Set res (WeakCompareAndSwapL mem (Binary oldval newval)));
9308 ins_cost(VOLATILE_REF_COST);
9309 effect(KILL cr);
9310 format %{
9311 "cmpxchg_acq $res = $mem, $oldval, $newval\t# (long, weak) if $mem == $oldval then $mem <-- $newval"
9312 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
9313 %}
9314 ins_encode %{
9315 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
9316 Assembler::xword, /*acquire*/ true, /*release*/ true,
9317 /*weak*/ true, noreg);
9318 __ csetw($res$$Register, Assembler::EQ);
9319 %}
9320 ins_pipe(pipe_slow);
9321 %}
9322
9323 // This pattern is generated automatically from cas.m4.
9324 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9325 instruct weakCompareAndSwapNAcq(iRegINoSp res, indirect mem, iRegN oldval, iRegN newval, rFlagsReg cr) %{
9326 predicate(needs_acquiring_load_exclusive(n));
9327 match(Set res (WeakCompareAndSwapN mem (Binary oldval newval)));
9328 ins_cost(VOLATILE_REF_COST);
9329 effect(KILL cr);
9330 format %{
9331 "cmpxchgw_acq $res = $mem, $oldval, $newval\t# (narrow oop, weak) if $mem == $oldval then $mem <-- $newval"
9332 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
9333 %}
9334 ins_encode %{
9335 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
9336 Assembler::word, /*acquire*/ true, /*release*/ true,
9337 /*weak*/ true, noreg);
9338 __ csetw($res$$Register, Assembler::EQ);
9339 %}
9340 ins_pipe(pipe_slow);
9341 %}
9342
9343 // This pattern is generated automatically from cas.m4.
9344 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9345 instruct weakCompareAndSwapPAcq(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
9346 predicate(needs_acquiring_load_exclusive(n) && (n->as_LoadStore()->barrier_data() == 0));
9347 match(Set res (WeakCompareAndSwapP mem (Binary oldval newval)));
9348 ins_cost(VOLATILE_REF_COST);
9349 effect(KILL cr);
9350 format %{
9351 "cmpxchg_acq $res = $mem, $oldval, $newval\t# (ptr, weak) if $mem == $oldval then $mem <-- $newval"
9352 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
9353 %}
9354 ins_encode %{
9355 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
9356 Assembler::xword, /*acquire*/ true, /*release*/ true,
9357 /*weak*/ true, noreg);
9358 __ csetw($res$$Register, Assembler::EQ);
9359 %}
9360 ins_pipe(pipe_slow);
9361 %}
9362
9363 // END This section of the file is automatically generated. Do not edit --------------
9364 // ---------------------------------------------------------------------
9365
9366 instruct get_and_setI(indirect mem, iRegI newv, iRegINoSp prev) %{
9367 match(Set prev (GetAndSetI mem newv));
9368 ins_cost(2 * VOLATILE_REF_COST);
9369 format %{ "atomic_xchgw $prev, $newv, [$mem]" %}
9370 ins_encode %{
9371 __ atomic_xchgw($prev$$Register, $newv$$Register, as_Register($mem$$base));
9372 %}
9373 ins_pipe(pipe_serial);
9374 %}
9375
9376 instruct get_and_setL(indirect mem, iRegL newv, iRegLNoSp prev) %{
9377 match(Set prev (GetAndSetL mem newv));
9378 ins_cost(2 * VOLATILE_REF_COST);
9379 format %{ "atomic_xchg $prev, $newv, [$mem]" %}
9380 ins_encode %{
9381 __ atomic_xchg($prev$$Register, $newv$$Register, as_Register($mem$$base));
9382 %}
9383 ins_pipe(pipe_serial);
9384 %}
9385
9386 instruct get_and_setN(indirect mem, iRegN newv, iRegINoSp prev) %{
9387 match(Set prev (GetAndSetN mem newv));
9388 ins_cost(2 * VOLATILE_REF_COST);
9389 format %{ "atomic_xchgw $prev, $newv, [$mem]" %}
9390 ins_encode %{
9391 __ atomic_xchgw($prev$$Register, $newv$$Register, as_Register($mem$$base));
9392 %}
9393 ins_pipe(pipe_serial);
9394 %}
9395
9396 instruct get_and_setP(indirect mem, iRegP newv, iRegPNoSp prev) %{
9397 predicate(n->as_LoadStore()->barrier_data() == 0);
9398 match(Set prev (GetAndSetP mem newv));
9399 ins_cost(2 * VOLATILE_REF_COST);
9400 format %{ "atomic_xchg $prev, $newv, [$mem]" %}
9401 ins_encode %{
9402 __ atomic_xchg($prev$$Register, $newv$$Register, as_Register($mem$$base));
9403 %}
9404 ins_pipe(pipe_serial);
9405 %}
9406
9407 instruct get_and_setIAcq(indirect mem, iRegI newv, iRegINoSp prev) %{
9408 predicate(needs_acquiring_load_exclusive(n));
9409 match(Set prev (GetAndSetI mem newv));
9410 ins_cost(VOLATILE_REF_COST);
9411 format %{ "atomic_xchgw_acq $prev, $newv, [$mem]" %}
9412 ins_encode %{
9413 __ atomic_xchgalw($prev$$Register, $newv$$Register, as_Register($mem$$base));
9414 %}
9415 ins_pipe(pipe_serial);
9416 %}
9417
9418 instruct get_and_setLAcq(indirect mem, iRegL newv, iRegLNoSp prev) %{
9419 predicate(needs_acquiring_load_exclusive(n));
9420 match(Set prev (GetAndSetL mem newv));
9421 ins_cost(VOLATILE_REF_COST);
9422 format %{ "atomic_xchg_acq $prev, $newv, [$mem]" %}
9423 ins_encode %{
9424 __ atomic_xchgal($prev$$Register, $newv$$Register, as_Register($mem$$base));
9425 %}
9426 ins_pipe(pipe_serial);
9427 %}
9428
9429 instruct get_and_setNAcq(indirect mem, iRegN newv, iRegINoSp prev) %{
9430 predicate(needs_acquiring_load_exclusive(n));
9431 match(Set prev (GetAndSetN mem newv));
9432 ins_cost(VOLATILE_REF_COST);
9433 format %{ "atomic_xchgw_acq $prev, $newv, [$mem]" %}
9434 ins_encode %{
9435 __ atomic_xchgalw($prev$$Register, $newv$$Register, as_Register($mem$$base));
9436 %}
9437 ins_pipe(pipe_serial);
9438 %}
9439
9440 instruct get_and_setPAcq(indirect mem, iRegP newv, iRegPNoSp prev) %{
9441 predicate(needs_acquiring_load_exclusive(n) && (n->as_LoadStore()->barrier_data() == 0));
9442 match(Set prev (GetAndSetP mem newv));
9443 ins_cost(VOLATILE_REF_COST);
9444 format %{ "atomic_xchg_acq $prev, $newv, [$mem]" %}
9445 ins_encode %{
9446 __ atomic_xchgal($prev$$Register, $newv$$Register, as_Register($mem$$base));
9447 %}
9448 ins_pipe(pipe_serial);
9449 %}
9450
9451
9452 instruct get_and_addL(indirect mem, iRegLNoSp newval, iRegL incr) %{
9453 match(Set newval (GetAndAddL mem incr));
9454 ins_cost(2 * VOLATILE_REF_COST + 1);
9455 format %{ "get_and_addL $newval, [$mem], $incr" %}
9456 ins_encode %{
9457 __ atomic_add($newval$$Register, $incr$$Register, as_Register($mem$$base));
9458 %}
9459 ins_pipe(pipe_serial);
9460 %}
9461
9462 instruct get_and_addL_no_res(indirect mem, Universe dummy, iRegL incr) %{
9463 predicate(n->as_LoadStore()->result_not_used());
9464 match(Set dummy (GetAndAddL mem incr));
9465 ins_cost(2 * VOLATILE_REF_COST);
9466 format %{ "get_and_addL [$mem], $incr" %}
9467 ins_encode %{
9468 __ atomic_add(noreg, $incr$$Register, as_Register($mem$$base));
9469 %}
9470 ins_pipe(pipe_serial);
9471 %}
9472
9473 instruct get_and_addLi(indirect mem, iRegLNoSp newval, immLAddSub incr) %{
9474 match(Set newval (GetAndAddL mem incr));
9475 ins_cost(2 * VOLATILE_REF_COST + 1);
9476 format %{ "get_and_addL $newval, [$mem], $incr" %}
9477 ins_encode %{
9478 __ atomic_add($newval$$Register, $incr$$constant, as_Register($mem$$base));
9479 %}
9480 ins_pipe(pipe_serial);
9481 %}
9482
9483 instruct get_and_addLi_no_res(indirect mem, Universe dummy, immLAddSub incr) %{
9484 predicate(n->as_LoadStore()->result_not_used());
9485 match(Set dummy (GetAndAddL mem incr));
9486 ins_cost(2 * VOLATILE_REF_COST);
9487 format %{ "get_and_addL [$mem], $incr" %}
9488 ins_encode %{
9489 __ atomic_add(noreg, $incr$$constant, as_Register($mem$$base));
9490 %}
9491 ins_pipe(pipe_serial);
9492 %}
9493
9494 instruct get_and_addI(indirect mem, iRegINoSp newval, iRegIorL2I incr) %{
9495 match(Set newval (GetAndAddI mem incr));
9496 ins_cost(2 * VOLATILE_REF_COST + 1);
9497 format %{ "get_and_addI $newval, [$mem], $incr" %}
9498 ins_encode %{
9499 __ atomic_addw($newval$$Register, $incr$$Register, as_Register($mem$$base));
9500 %}
9501 ins_pipe(pipe_serial);
9502 %}
9503
9504 instruct get_and_addI_no_res(indirect mem, Universe dummy, iRegIorL2I incr) %{
9505 predicate(n->as_LoadStore()->result_not_used());
9506 match(Set dummy (GetAndAddI mem incr));
9507 ins_cost(2 * VOLATILE_REF_COST);
9508 format %{ "get_and_addI [$mem], $incr" %}
9509 ins_encode %{
9510 __ atomic_addw(noreg, $incr$$Register, as_Register($mem$$base));
9511 %}
9512 ins_pipe(pipe_serial);
9513 %}
9514
9515 instruct get_and_addIi(indirect mem, iRegINoSp newval, immIAddSub incr) %{
9516 match(Set newval (GetAndAddI mem incr));
9517 ins_cost(2 * VOLATILE_REF_COST + 1);
9518 format %{ "get_and_addI $newval, [$mem], $incr" %}
9519 ins_encode %{
9520 __ atomic_addw($newval$$Register, $incr$$constant, as_Register($mem$$base));
9521 %}
9522 ins_pipe(pipe_serial);
9523 %}
9524
9525 instruct get_and_addIi_no_res(indirect mem, Universe dummy, immIAddSub incr) %{
9526 predicate(n->as_LoadStore()->result_not_used());
9527 match(Set dummy (GetAndAddI mem incr));
9528 ins_cost(2 * VOLATILE_REF_COST);
9529 format %{ "get_and_addI [$mem], $incr" %}
9530 ins_encode %{
9531 __ atomic_addw(noreg, $incr$$constant, as_Register($mem$$base));
9532 %}
9533 ins_pipe(pipe_serial);
9534 %}
9535
9536 instruct get_and_addLAcq(indirect mem, iRegLNoSp newval, iRegL incr) %{
9537 predicate(needs_acquiring_load_exclusive(n));
9538 match(Set newval (GetAndAddL mem incr));
9539 ins_cost(VOLATILE_REF_COST + 1);
9540 format %{ "get_and_addL_acq $newval, [$mem], $incr" %}
9541 ins_encode %{
9542 __ atomic_addal($newval$$Register, $incr$$Register, as_Register($mem$$base));
9543 %}
9544 ins_pipe(pipe_serial);
9545 %}
9546
9547 instruct get_and_addL_no_resAcq(indirect mem, Universe dummy, iRegL incr) %{
9548 predicate(n->as_LoadStore()->result_not_used() && needs_acquiring_load_exclusive(n));
9549 match(Set dummy (GetAndAddL mem incr));
9550 ins_cost(VOLATILE_REF_COST);
9551 format %{ "get_and_addL_acq [$mem], $incr" %}
9552 ins_encode %{
9553 __ atomic_addal(noreg, $incr$$Register, as_Register($mem$$base));
9554 %}
9555 ins_pipe(pipe_serial);
9556 %}
9557
9558 instruct get_and_addLiAcq(indirect mem, iRegLNoSp newval, immLAddSub incr) %{
9559 predicate(needs_acquiring_load_exclusive(n));
9560 match(Set newval (GetAndAddL mem incr));
9561 ins_cost(VOLATILE_REF_COST + 1);
9562 format %{ "get_and_addL_acq $newval, [$mem], $incr" %}
9563 ins_encode %{
9564 __ atomic_addal($newval$$Register, $incr$$constant, as_Register($mem$$base));
9565 %}
9566 ins_pipe(pipe_serial);
9567 %}
9568
9569 instruct get_and_addLi_no_resAcq(indirect mem, Universe dummy, immLAddSub incr) %{
9570 predicate(n->as_LoadStore()->result_not_used() && needs_acquiring_load_exclusive(n));
9571 match(Set dummy (GetAndAddL mem incr));
9572 ins_cost(VOLATILE_REF_COST);
9573 format %{ "get_and_addL_acq [$mem], $incr" %}
9574 ins_encode %{
9575 __ atomic_addal(noreg, $incr$$constant, as_Register($mem$$base));
9576 %}
9577 ins_pipe(pipe_serial);
9578 %}
9579
9580 instruct get_and_addIAcq(indirect mem, iRegINoSp newval, iRegIorL2I incr) %{
9581 predicate(needs_acquiring_load_exclusive(n));
9582 match(Set newval (GetAndAddI mem incr));
9583 ins_cost(VOLATILE_REF_COST + 1);
9584 format %{ "get_and_addI_acq $newval, [$mem], $incr" %}
9585 ins_encode %{
9586 __ atomic_addalw($newval$$Register, $incr$$Register, as_Register($mem$$base));
9587 %}
9588 ins_pipe(pipe_serial);
9589 %}
9590
9591 instruct get_and_addI_no_resAcq(indirect mem, Universe dummy, iRegIorL2I incr) %{
9592 predicate(n->as_LoadStore()->result_not_used() && needs_acquiring_load_exclusive(n));
9593 match(Set dummy (GetAndAddI mem incr));
9594 ins_cost(VOLATILE_REF_COST);
9595 format %{ "get_and_addI_acq [$mem], $incr" %}
9596 ins_encode %{
9597 __ atomic_addalw(noreg, $incr$$Register, as_Register($mem$$base));
9598 %}
9599 ins_pipe(pipe_serial);
9600 %}
9601
9602 instruct get_and_addIiAcq(indirect mem, iRegINoSp newval, immIAddSub incr) %{
9603 predicate(needs_acquiring_load_exclusive(n));
9604 match(Set newval (GetAndAddI mem incr));
9605 ins_cost(VOLATILE_REF_COST + 1);
9606 format %{ "get_and_addI_acq $newval, [$mem], $incr" %}
9607 ins_encode %{
9608 __ atomic_addalw($newval$$Register, $incr$$constant, as_Register($mem$$base));
9609 %}
9610 ins_pipe(pipe_serial);
9611 %}
9612
9613 instruct get_and_addIi_no_resAcq(indirect mem, Universe dummy, immIAddSub incr) %{
9614 predicate(n->as_LoadStore()->result_not_used() && needs_acquiring_load_exclusive(n));
9615 match(Set dummy (GetAndAddI mem incr));
9616 ins_cost(VOLATILE_REF_COST);
9617 format %{ "get_and_addI_acq [$mem], $incr" %}
9618 ins_encode %{
9619 __ atomic_addalw(noreg, $incr$$constant, as_Register($mem$$base));
9620 %}
9621 ins_pipe(pipe_serial);
9622 %}
9623
9624 // Manifest a CmpL result in an integer register.
9625 // (src1 < src2) ? -1 : ((src1 > src2) ? 1 : 0)
9626 instruct cmpL3_reg_reg(iRegINoSp dst, iRegL src1, iRegL src2, rFlagsReg flags)
9627 %{
9628 match(Set dst (CmpL3 src1 src2));
9629 effect(KILL flags);
9630
9631 ins_cost(INSN_COST * 6);
9632 format %{
9633 "cmp $src1, $src2"
9634 "csetw $dst, ne"
9635 "cnegw $dst, lt"
9636 %}
9637 // format %{ "CmpL3 $dst, $src1, $src2" %}
9638 ins_encode %{
9639 __ cmp($src1$$Register, $src2$$Register);
9640 __ csetw($dst$$Register, Assembler::NE);
9641 __ cnegw($dst$$Register, $dst$$Register, Assembler::LT);
9642 %}
9643
9644 ins_pipe(pipe_class_default);
9645 %}
9646
9647 instruct cmpL3_reg_imm(iRegINoSp dst, iRegL src1, immLAddSub src2, rFlagsReg flags)
9648 %{
9649 match(Set dst (CmpL3 src1 src2));
9650 effect(KILL flags);
9651
9652 ins_cost(INSN_COST * 6);
9653 format %{
9654 "cmp $src1, $src2"
9655 "csetw $dst, ne"
9656 "cnegw $dst, lt"
9657 %}
9658 ins_encode %{
9659 int32_t con = (int32_t)$src2$$constant;
9660 if (con < 0) {
9661 __ adds(zr, $src1$$Register, -con);
9662 } else {
9663 __ subs(zr, $src1$$Register, con);
9664 }
9665 __ csetw($dst$$Register, Assembler::NE);
9666 __ cnegw($dst$$Register, $dst$$Register, Assembler::LT);
9667 %}
9668
9669 ins_pipe(pipe_class_default);
9670 %}
9671
9672 // ============================================================================
9673 // Conditional Move Instructions
9674
9675 // n.b. we have identical rules for both a signed compare op (cmpOp)
9676 // and an unsigned compare op (cmpOpU). it would be nice if we could
9677 // define an op class which merged both inputs and use it to type the
9678 // argument to a single rule. unfortunatelyt his fails because the
9679 // opclass does not live up to the COND_INTER interface of its
9680 // component operands. When the generic code tries to negate the
9681 // operand it ends up running the generci Machoper::negate method
9682 // which throws a ShouldNotHappen. So, we have to provide two flavours
9683 // of each rule, one for a cmpOp and a second for a cmpOpU (sigh).
9684
9685 instruct cmovI_reg_reg(cmpOp cmp, rFlagsReg cr, iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
9686 match(Set dst (CMoveI (Binary cmp cr) (Binary src1 src2)));
9687
9688 ins_cost(INSN_COST * 2);
9689 format %{ "cselw $dst, $src2, $src1 $cmp\t# signed, int" %}
9690
9691 ins_encode %{
9692 __ cselw(as_Register($dst$$reg),
9693 as_Register($src2$$reg),
9694 as_Register($src1$$reg),
9695 (Assembler::Condition)$cmp$$cmpcode);
9696 %}
9697
9698 ins_pipe(icond_reg_reg);
9699 %}
9700
9701 instruct cmovUI_reg_reg(cmpOpU cmp, rFlagsRegU cr, iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
9702 match(Set dst (CMoveI (Binary cmp cr) (Binary src1 src2)));
9703
9704 ins_cost(INSN_COST * 2);
9705 format %{ "cselw $dst, $src2, $src1 $cmp\t# unsigned, int" %}
9706
9707 ins_encode %{
9708 __ cselw(as_Register($dst$$reg),
9709 as_Register($src2$$reg),
9710 as_Register($src1$$reg),
9711 (Assembler::Condition)$cmp$$cmpcode);
9712 %}
9713
9714 ins_pipe(icond_reg_reg);
9715 %}
9716
9717 // special cases where one arg is zero
9718
9719 // n.b. this is selected in preference to the rule above because it
9720 // avoids loading constant 0 into a source register
9721
9722 // TODO
9723 // we ought only to be able to cull one of these variants as the ideal
9724 // transforms ought always to order the zero consistently (to left/right?)
9725
9726 instruct cmovI_zero_reg(cmpOp cmp, rFlagsReg cr, iRegINoSp dst, immI0 zero, iRegIorL2I src) %{
9727 match(Set dst (CMoveI (Binary cmp cr) (Binary zero src)));
9728
9729 ins_cost(INSN_COST * 2);
9730 format %{ "cselw $dst, $src, zr $cmp\t# signed, int" %}
9731
9732 ins_encode %{
9733 __ cselw(as_Register($dst$$reg),
9734 as_Register($src$$reg),
9735 zr,
9736 (Assembler::Condition)$cmp$$cmpcode);
9737 %}
9738
9739 ins_pipe(icond_reg);
9740 %}
9741
9742 instruct cmovUI_zero_reg(cmpOpU cmp, rFlagsRegU cr, iRegINoSp dst, immI0 zero, iRegIorL2I src) %{
9743 match(Set dst (CMoveI (Binary cmp cr) (Binary zero src)));
9744
9745 ins_cost(INSN_COST * 2);
9746 format %{ "cselw $dst, $src, zr $cmp\t# unsigned, int" %}
9747
9748 ins_encode %{
9749 __ cselw(as_Register($dst$$reg),
9750 as_Register($src$$reg),
9751 zr,
9752 (Assembler::Condition)$cmp$$cmpcode);
9753 %}
9754
9755 ins_pipe(icond_reg);
9756 %}
9757
9758 instruct cmovI_reg_zero(cmpOp cmp, rFlagsReg cr, iRegINoSp dst, iRegIorL2I src, immI0 zero) %{
9759 match(Set dst (CMoveI (Binary cmp cr) (Binary src zero)));
9760
9761 ins_cost(INSN_COST * 2);
9762 format %{ "cselw $dst, zr, $src $cmp\t# signed, int" %}
9763
9764 ins_encode %{
9765 __ cselw(as_Register($dst$$reg),
9766 zr,
9767 as_Register($src$$reg),
9768 (Assembler::Condition)$cmp$$cmpcode);
9769 %}
9770
9771 ins_pipe(icond_reg);
9772 %}
9773
9774 instruct cmovUI_reg_zero(cmpOpU cmp, rFlagsRegU cr, iRegINoSp dst, iRegIorL2I src, immI0 zero) %{
9775 match(Set dst (CMoveI (Binary cmp cr) (Binary src zero)));
9776
9777 ins_cost(INSN_COST * 2);
9778 format %{ "cselw $dst, zr, $src $cmp\t# unsigned, int" %}
9779
9780 ins_encode %{
9781 __ cselw(as_Register($dst$$reg),
9782 zr,
9783 as_Register($src$$reg),
9784 (Assembler::Condition)$cmp$$cmpcode);
9785 %}
9786
9787 ins_pipe(icond_reg);
9788 %}
9789
9790 // special case for creating a boolean 0 or 1
9791
9792 // n.b. this is selected in preference to the rule above because it
9793 // avoids loading constants 0 and 1 into a source register
9794
9795 instruct cmovI_reg_zero_one(cmpOp cmp, rFlagsReg cr, iRegINoSp dst, immI0 zero, immI_1 one) %{
9796 match(Set dst (CMoveI (Binary cmp cr) (Binary one zero)));
9797
9798 ins_cost(INSN_COST * 2);
9799 format %{ "csincw $dst, zr, zr $cmp\t# signed, int" %}
9800
9801 ins_encode %{
9802 // equivalently
9803 // cset(as_Register($dst$$reg),
9804 // negate_condition((Assembler::Condition)$cmp$$cmpcode));
9805 __ csincw(as_Register($dst$$reg),
9806 zr,
9807 zr,
9808 (Assembler::Condition)$cmp$$cmpcode);
9809 %}
9810
9811 ins_pipe(icond_none);
9812 %}
9813
9814 instruct cmovUI_reg_zero_one(cmpOpU cmp, rFlagsRegU cr, iRegINoSp dst, immI0 zero, immI_1 one) %{
9815 match(Set dst (CMoveI (Binary cmp cr) (Binary one zero)));
9816
9817 ins_cost(INSN_COST * 2);
9818 format %{ "csincw $dst, zr, zr $cmp\t# unsigned, int" %}
9819
9820 ins_encode %{
9821 // equivalently
9822 // cset(as_Register($dst$$reg),
9823 // negate_condition((Assembler::Condition)$cmp$$cmpcode));
9824 __ csincw(as_Register($dst$$reg),
9825 zr,
9826 zr,
9827 (Assembler::Condition)$cmp$$cmpcode);
9828 %}
9829
9830 ins_pipe(icond_none);
9831 %}
9832
9833 instruct cmovL_reg_reg(cmpOp cmp, rFlagsReg cr, iRegLNoSp dst, iRegL src1, iRegL src2) %{
9834 match(Set dst (CMoveL (Binary cmp cr) (Binary src1 src2)));
9835
9836 ins_cost(INSN_COST * 2);
9837 format %{ "csel $dst, $src2, $src1 $cmp\t# signed, long" %}
9838
9839 ins_encode %{
9840 __ csel(as_Register($dst$$reg),
9841 as_Register($src2$$reg),
9842 as_Register($src1$$reg),
9843 (Assembler::Condition)$cmp$$cmpcode);
9844 %}
9845
9846 ins_pipe(icond_reg_reg);
9847 %}
9848
9849 instruct cmovUL_reg_reg(cmpOpU cmp, rFlagsRegU cr, iRegLNoSp dst, iRegL src1, iRegL src2) %{
9850 match(Set dst (CMoveL (Binary cmp cr) (Binary src1 src2)));
9851
9852 ins_cost(INSN_COST * 2);
9853 format %{ "csel $dst, $src2, $src1 $cmp\t# unsigned, long" %}
9854
9855 ins_encode %{
9856 __ csel(as_Register($dst$$reg),
9857 as_Register($src2$$reg),
9858 as_Register($src1$$reg),
9859 (Assembler::Condition)$cmp$$cmpcode);
9860 %}
9861
9862 ins_pipe(icond_reg_reg);
9863 %}
9864
9865 // special cases where one arg is zero
9866
9867 instruct cmovL_reg_zero(cmpOp cmp, rFlagsReg cr, iRegLNoSp dst, iRegL src, immL0 zero) %{
9868 match(Set dst (CMoveL (Binary cmp cr) (Binary src zero)));
9869
9870 ins_cost(INSN_COST * 2);
9871 format %{ "csel $dst, zr, $src $cmp\t# signed, long" %}
9872
9873 ins_encode %{
9874 __ csel(as_Register($dst$$reg),
9875 zr,
9876 as_Register($src$$reg),
9877 (Assembler::Condition)$cmp$$cmpcode);
9878 %}
9879
9880 ins_pipe(icond_reg);
9881 %}
9882
9883 instruct cmovUL_reg_zero(cmpOpU cmp, rFlagsRegU cr, iRegLNoSp dst, iRegL src, immL0 zero) %{
9884 match(Set dst (CMoveL (Binary cmp cr) (Binary src zero)));
9885
9886 ins_cost(INSN_COST * 2);
9887 format %{ "csel $dst, zr, $src $cmp\t# unsigned, long" %}
9888
9889 ins_encode %{
9890 __ csel(as_Register($dst$$reg),
9891 zr,
9892 as_Register($src$$reg),
9893 (Assembler::Condition)$cmp$$cmpcode);
9894 %}
9895
9896 ins_pipe(icond_reg);
9897 %}
9898
9899 instruct cmovL_zero_reg(cmpOp cmp, rFlagsReg cr, iRegLNoSp dst, immL0 zero, iRegL src) %{
9900 match(Set dst (CMoveL (Binary cmp cr) (Binary zero src)));
9901
9902 ins_cost(INSN_COST * 2);
9903 format %{ "csel $dst, $src, zr $cmp\t# signed, long" %}
9904
9905 ins_encode %{
9906 __ csel(as_Register($dst$$reg),
9907 as_Register($src$$reg),
9908 zr,
9909 (Assembler::Condition)$cmp$$cmpcode);
9910 %}
9911
9912 ins_pipe(icond_reg);
9913 %}
9914
9915 instruct cmovUL_zero_reg(cmpOpU cmp, rFlagsRegU cr, iRegLNoSp dst, immL0 zero, iRegL src) %{
9916 match(Set dst (CMoveL (Binary cmp cr) (Binary zero src)));
9917
9918 ins_cost(INSN_COST * 2);
9919 format %{ "csel $dst, $src, zr $cmp\t# unsigned, long" %}
9920
9921 ins_encode %{
9922 __ csel(as_Register($dst$$reg),
9923 as_Register($src$$reg),
9924 zr,
9925 (Assembler::Condition)$cmp$$cmpcode);
9926 %}
9927
9928 ins_pipe(icond_reg);
9929 %}
9930
9931 instruct cmovP_reg_reg(cmpOp cmp, rFlagsReg cr, iRegPNoSp dst, iRegP src1, iRegP src2) %{
9932 match(Set dst (CMoveP (Binary cmp cr) (Binary src1 src2)));
9933
9934 ins_cost(INSN_COST * 2);
9935 format %{ "csel $dst, $src2, $src1 $cmp\t# signed, ptr" %}
9936
9937 ins_encode %{
9938 __ csel(as_Register($dst$$reg),
9939 as_Register($src2$$reg),
9940 as_Register($src1$$reg),
9941 (Assembler::Condition)$cmp$$cmpcode);
9942 %}
9943
9944 ins_pipe(icond_reg_reg);
9945 %}
9946
9947 instruct cmovUP_reg_reg(cmpOpU cmp, rFlagsRegU cr, iRegPNoSp dst, iRegP src1, iRegP src2) %{
9948 match(Set dst (CMoveP (Binary cmp cr) (Binary src1 src2)));
9949
9950 ins_cost(INSN_COST * 2);
9951 format %{ "csel $dst, $src2, $src1 $cmp\t# unsigned, ptr" %}
9952
9953 ins_encode %{
9954 __ csel(as_Register($dst$$reg),
9955 as_Register($src2$$reg),
9956 as_Register($src1$$reg),
9957 (Assembler::Condition)$cmp$$cmpcode);
9958 %}
9959
9960 ins_pipe(icond_reg_reg);
9961 %}
9962
9963 // special cases where one arg is zero
9964
9965 instruct cmovP_reg_zero(cmpOp cmp, rFlagsReg cr, iRegPNoSp dst, iRegP src, immP0 zero) %{
9966 match(Set dst (CMoveP (Binary cmp cr) (Binary src zero)));
9967
9968 ins_cost(INSN_COST * 2);
9969 format %{ "csel $dst, zr, $src $cmp\t# signed, ptr" %}
9970
9971 ins_encode %{
9972 __ csel(as_Register($dst$$reg),
9973 zr,
9974 as_Register($src$$reg),
9975 (Assembler::Condition)$cmp$$cmpcode);
9976 %}
9977
9978 ins_pipe(icond_reg);
9979 %}
9980
9981 instruct cmovUP_reg_zero(cmpOpU cmp, rFlagsRegU cr, iRegPNoSp dst, iRegP src, immP0 zero) %{
9982 match(Set dst (CMoveP (Binary cmp cr) (Binary src zero)));
9983
9984 ins_cost(INSN_COST * 2);
9985 format %{ "csel $dst, zr, $src $cmp\t# unsigned, ptr" %}
9986
9987 ins_encode %{
9988 __ csel(as_Register($dst$$reg),
9989 zr,
9990 as_Register($src$$reg),
9991 (Assembler::Condition)$cmp$$cmpcode);
9992 %}
9993
9994 ins_pipe(icond_reg);
9995 %}
9996
9997 instruct cmovP_zero_reg(cmpOp cmp, rFlagsReg cr, iRegPNoSp dst, immP0 zero, iRegP src) %{
9998 match(Set dst (CMoveP (Binary cmp cr) (Binary zero src)));
9999
10000 ins_cost(INSN_COST * 2);
10001 format %{ "csel $dst, $src, zr $cmp\t# signed, ptr" %}
10002
10003 ins_encode %{
10004 __ csel(as_Register($dst$$reg),
10005 as_Register($src$$reg),
10006 zr,
10007 (Assembler::Condition)$cmp$$cmpcode);
10008 %}
10009
10010 ins_pipe(icond_reg);
10011 %}
10012
10013 instruct cmovUP_zero_reg(cmpOpU cmp, rFlagsRegU cr, iRegPNoSp dst, immP0 zero, iRegP src) %{
10014 match(Set dst (CMoveP (Binary cmp cr) (Binary zero src)));
10015
10016 ins_cost(INSN_COST * 2);
10017 format %{ "csel $dst, $src, zr $cmp\t# unsigned, ptr" %}
10018
10019 ins_encode %{
10020 __ csel(as_Register($dst$$reg),
10021 as_Register($src$$reg),
10022 zr,
10023 (Assembler::Condition)$cmp$$cmpcode);
10024 %}
10025
10026 ins_pipe(icond_reg);
10027 %}
10028
10029 instruct cmovN_reg_reg(cmpOp cmp, rFlagsReg cr, iRegNNoSp dst, iRegN src1, iRegN src2) %{
10030 match(Set dst (CMoveN (Binary cmp cr) (Binary src1 src2)));
10031
10032 ins_cost(INSN_COST * 2);
10033 format %{ "cselw $dst, $src2, $src1 $cmp\t# signed, compressed ptr" %}
10034
10035 ins_encode %{
10036 __ cselw(as_Register($dst$$reg),
10037 as_Register($src2$$reg),
10038 as_Register($src1$$reg),
10039 (Assembler::Condition)$cmp$$cmpcode);
10040 %}
10041
10042 ins_pipe(icond_reg_reg);
10043 %}
10044
10045 instruct cmovUN_reg_reg(cmpOpU cmp, rFlagsRegU cr, iRegNNoSp dst, iRegN src1, iRegN src2) %{
10046 match(Set dst (CMoveN (Binary cmp cr) (Binary src1 src2)));
10047
10048 ins_cost(INSN_COST * 2);
10049 format %{ "cselw $dst, $src2, $src1 $cmp\t# signed, compressed ptr" %}
10050
10051 ins_encode %{
10052 __ cselw(as_Register($dst$$reg),
10053 as_Register($src2$$reg),
10054 as_Register($src1$$reg),
10055 (Assembler::Condition)$cmp$$cmpcode);
10056 %}
10057
10058 ins_pipe(icond_reg_reg);
10059 %}
10060
10061 // special cases where one arg is zero
10062
10063 instruct cmovN_reg_zero(cmpOp cmp, rFlagsReg cr, iRegNNoSp dst, iRegN src, immN0 zero) %{
10064 match(Set dst (CMoveN (Binary cmp cr) (Binary src zero)));
10065
10066 ins_cost(INSN_COST * 2);
10067 format %{ "cselw $dst, zr, $src $cmp\t# signed, compressed ptr" %}
10068
10069 ins_encode %{
10070 __ cselw(as_Register($dst$$reg),
10071 zr,
10072 as_Register($src$$reg),
10073 (Assembler::Condition)$cmp$$cmpcode);
10074 %}
10075
10076 ins_pipe(icond_reg);
10077 %}
10078
10079 instruct cmovUN_reg_zero(cmpOpU cmp, rFlagsRegU cr, iRegNNoSp dst, iRegN src, immN0 zero) %{
10080 match(Set dst (CMoveN (Binary cmp cr) (Binary src zero)));
10081
10082 ins_cost(INSN_COST * 2);
10083 format %{ "cselw $dst, zr, $src $cmp\t# unsigned, compressed ptr" %}
10084
10085 ins_encode %{
10086 __ cselw(as_Register($dst$$reg),
10087 zr,
10088 as_Register($src$$reg),
10089 (Assembler::Condition)$cmp$$cmpcode);
10090 %}
10091
10092 ins_pipe(icond_reg);
10093 %}
10094
10095 instruct cmovN_zero_reg(cmpOp cmp, rFlagsReg cr, iRegNNoSp dst, immN0 zero, iRegN src) %{
10096 match(Set dst (CMoveN (Binary cmp cr) (Binary zero src)));
10097
10098 ins_cost(INSN_COST * 2);
10099 format %{ "cselw $dst, $src, zr $cmp\t# signed, compressed ptr" %}
10100
10101 ins_encode %{
10102 __ cselw(as_Register($dst$$reg),
10103 as_Register($src$$reg),
10104 zr,
10105 (Assembler::Condition)$cmp$$cmpcode);
10106 %}
10107
10108 ins_pipe(icond_reg);
10109 %}
10110
10111 instruct cmovUN_zero_reg(cmpOpU cmp, rFlagsRegU cr, iRegNNoSp dst, immN0 zero, iRegN src) %{
10112 match(Set dst (CMoveN (Binary cmp cr) (Binary zero src)));
10113
10114 ins_cost(INSN_COST * 2);
10115 format %{ "cselw $dst, $src, zr $cmp\t# unsigned, compressed ptr" %}
10116
10117 ins_encode %{
10118 __ cselw(as_Register($dst$$reg),
10119 as_Register($src$$reg),
10120 zr,
10121 (Assembler::Condition)$cmp$$cmpcode);
10122 %}
10123
10124 ins_pipe(icond_reg);
10125 %}
10126
10127 instruct cmovF_reg(cmpOp cmp, rFlagsReg cr, vRegF dst, vRegF src1, vRegF src2)
10128 %{
10129 match(Set dst (CMoveF (Binary cmp cr) (Binary src1 src2)));
10130
10131 ins_cost(INSN_COST * 3);
10132
10133 format %{ "fcsels $dst, $src1, $src2, $cmp\t# signed cmove float\n\t" %}
10134 ins_encode %{
10135 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
10136 __ fcsels(as_FloatRegister($dst$$reg),
10137 as_FloatRegister($src2$$reg),
10138 as_FloatRegister($src1$$reg),
10139 cond);
10140 %}
10141
10142 ins_pipe(fp_cond_reg_reg_s);
10143 %}
10144
10145 instruct cmovUF_reg(cmpOpU cmp, rFlagsRegU cr, vRegF dst, vRegF src1, vRegF src2)
10146 %{
10147 match(Set dst (CMoveF (Binary cmp cr) (Binary src1 src2)));
10148
10149 ins_cost(INSN_COST * 3);
10150
10151 format %{ "fcsels $dst, $src1, $src2, $cmp\t# unsigned cmove float\n\t" %}
10152 ins_encode %{
10153 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
10154 __ fcsels(as_FloatRegister($dst$$reg),
10155 as_FloatRegister($src2$$reg),
10156 as_FloatRegister($src1$$reg),
10157 cond);
10158 %}
10159
10160 ins_pipe(fp_cond_reg_reg_s);
10161 %}
10162
10163 instruct cmovD_reg(cmpOp cmp, rFlagsReg cr, vRegD dst, vRegD src1, vRegD src2)
10164 %{
10165 match(Set dst (CMoveD (Binary cmp cr) (Binary src1 src2)));
10166
10167 ins_cost(INSN_COST * 3);
10168
10169 format %{ "fcseld $dst, $src1, $src2, $cmp\t# signed cmove float\n\t" %}
10170 ins_encode %{
10171 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
10172 __ fcseld(as_FloatRegister($dst$$reg),
10173 as_FloatRegister($src2$$reg),
10174 as_FloatRegister($src1$$reg),
10175 cond);
10176 %}
10177
10178 ins_pipe(fp_cond_reg_reg_d);
10179 %}
10180
10181 instruct cmovUD_reg(cmpOpU cmp, rFlagsRegU cr, vRegD dst, vRegD src1, vRegD src2)
10182 %{
10183 match(Set dst (CMoveD (Binary cmp cr) (Binary src1 src2)));
10184
10185 ins_cost(INSN_COST * 3);
10186
10187 format %{ "fcseld $dst, $src1, $src2, $cmp\t# unsigned cmove float\n\t" %}
10188 ins_encode %{
10189 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
10190 __ fcseld(as_FloatRegister($dst$$reg),
10191 as_FloatRegister($src2$$reg),
10192 as_FloatRegister($src1$$reg),
10193 cond);
10194 %}
10195
10196 ins_pipe(fp_cond_reg_reg_d);
10197 %}
10198
10199 // ============================================================================
10200 // Arithmetic Instructions
10201 //
10202
10203 // Integer Addition
10204
10205 // TODO
10206 // these currently employ operations which do not set CR and hence are
10207 // not flagged as killing CR but we would like to isolate the cases
10208 // where we want to set flags from those where we don't. need to work
10209 // out how to do that.
10210
10211 instruct addI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10212 match(Set dst (AddI src1 src2));
10213
10214 ins_cost(INSN_COST);
10215 format %{ "addw $dst, $src1, $src2" %}
10216
10217 ins_encode %{
10218 __ addw(as_Register($dst$$reg),
10219 as_Register($src1$$reg),
10220 as_Register($src2$$reg));
10221 %}
10222
10223 ins_pipe(ialu_reg_reg);
10224 %}
10225
10226 instruct addI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immIAddSub src2) %{
10227 match(Set dst (AddI src1 src2));
10228
10229 ins_cost(INSN_COST);
10230 format %{ "addw $dst, $src1, $src2" %}
10231
10232 // use opcode to indicate that this is an add not a sub
10233 opcode(0x0);
10234
10235 ins_encode(aarch64_enc_addsubw_imm(dst, src1, src2));
10236
10237 ins_pipe(ialu_reg_imm);
10238 %}
10239
10240 instruct addI_reg_imm_i2l(iRegINoSp dst, iRegL src1, immIAddSub src2) %{
10241 match(Set dst (AddI (ConvL2I src1) src2));
10242
10243 ins_cost(INSN_COST);
10244 format %{ "addw $dst, $src1, $src2" %}
10245
10246 // use opcode to indicate that this is an add not a sub
10247 opcode(0x0);
10248
10249 ins_encode(aarch64_enc_addsubw_imm(dst, src1, src2));
10250
10251 ins_pipe(ialu_reg_imm);
10252 %}
10253
10254 // Pointer Addition
10255 instruct addP_reg_reg(iRegPNoSp dst, iRegP src1, iRegL src2) %{
10256 match(Set dst (AddP src1 src2));
10257
10258 ins_cost(INSN_COST);
10259 format %{ "add $dst, $src1, $src2\t# ptr" %}
10260
10261 ins_encode %{
10262 __ add(as_Register($dst$$reg),
10263 as_Register($src1$$reg),
10264 as_Register($src2$$reg));
10265 %}
10266
10267 ins_pipe(ialu_reg_reg);
10268 %}
10269
10270 instruct addP_reg_reg_ext(iRegPNoSp dst, iRegP src1, iRegIorL2I src2) %{
10271 match(Set dst (AddP src1 (ConvI2L src2)));
10272
10273 ins_cost(1.9 * INSN_COST);
10274 format %{ "add $dst, $src1, $src2, sxtw\t# ptr" %}
10275
10276 ins_encode %{
10277 __ add(as_Register($dst$$reg),
10278 as_Register($src1$$reg),
10279 as_Register($src2$$reg), ext::sxtw);
10280 %}
10281
10282 ins_pipe(ialu_reg_reg);
10283 %}
10284
10285 instruct addP_reg_reg_lsl(iRegPNoSp dst, iRegP src1, iRegL src2, immIScale scale) %{
10286 match(Set dst (AddP src1 (LShiftL src2 scale)));
10287
10288 ins_cost(1.9 * INSN_COST);
10289 format %{ "add $dst, $src1, $src2, LShiftL $scale\t# ptr" %}
10290
10291 ins_encode %{
10292 __ lea(as_Register($dst$$reg),
10293 Address(as_Register($src1$$reg), as_Register($src2$$reg),
10294 Address::lsl($scale$$constant)));
10295 %}
10296
10297 ins_pipe(ialu_reg_reg_shift);
10298 %}
10299
10300 instruct addP_reg_reg_ext_shift(iRegPNoSp dst, iRegP src1, iRegIorL2I src2, immIScale scale) %{
10301 match(Set dst (AddP src1 (LShiftL (ConvI2L src2) scale)));
10302
10303 ins_cost(1.9 * INSN_COST);
10304 format %{ "add $dst, $src1, $src2, I2L $scale\t# ptr" %}
10305
10306 ins_encode %{
10307 __ lea(as_Register($dst$$reg),
10308 Address(as_Register($src1$$reg), as_Register($src2$$reg),
10309 Address::sxtw($scale$$constant)));
10310 %}
10311
10312 ins_pipe(ialu_reg_reg_shift);
10313 %}
10314
10315 instruct lshift_ext(iRegLNoSp dst, iRegIorL2I src, immI scale, rFlagsReg cr) %{
10316 match(Set dst (LShiftL (ConvI2L src) scale));
10317
10318 ins_cost(INSN_COST);
10319 format %{ "sbfiz $dst, $src, $scale & 63, -$scale & 63\t" %}
10320
10321 ins_encode %{
10322 __ sbfiz(as_Register($dst$$reg),
10323 as_Register($src$$reg),
10324 $scale$$constant & 63, MIN2(32, (int)((-$scale$$constant) & 63)));
10325 %}
10326
10327 ins_pipe(ialu_reg_shift);
10328 %}
10329
10330 // Pointer Immediate Addition
10331 // n.b. this needs to be more expensive than using an indirect memory
10332 // operand
10333 instruct addP_reg_imm(iRegPNoSp dst, iRegP src1, immLAddSub src2) %{
10334 match(Set dst (AddP src1 src2));
10335
10336 ins_cost(INSN_COST);
10337 format %{ "add $dst, $src1, $src2\t# ptr" %}
10338
10339 // use opcode to indicate that this is an add not a sub
10340 opcode(0x0);
10341
10342 ins_encode( aarch64_enc_addsub_imm(dst, src1, src2) );
10343
10344 ins_pipe(ialu_reg_imm);
10345 %}
10346
10347 // Long Addition
10348 instruct addL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
10349
10350 match(Set dst (AddL src1 src2));
10351
10352 ins_cost(INSN_COST);
10353 format %{ "add $dst, $src1, $src2" %}
10354
10355 ins_encode %{
10356 __ add(as_Register($dst$$reg),
10357 as_Register($src1$$reg),
10358 as_Register($src2$$reg));
10359 %}
10360
10361 ins_pipe(ialu_reg_reg);
10362 %}
10363
10364 // No constant pool entries requiredLong Immediate Addition.
10365 instruct addL_reg_imm(iRegLNoSp dst, iRegL src1, immLAddSub src2) %{
10366 match(Set dst (AddL src1 src2));
10367
10368 ins_cost(INSN_COST);
10369 format %{ "add $dst, $src1, $src2" %}
10370
10371 // use opcode to indicate that this is an add not a sub
10372 opcode(0x0);
10373
10374 ins_encode( aarch64_enc_addsub_imm(dst, src1, src2) );
10375
10376 ins_pipe(ialu_reg_imm);
10377 %}
10378
10379 // Integer Subtraction
10380 instruct subI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10381 match(Set dst (SubI src1 src2));
10382
10383 ins_cost(INSN_COST);
10384 format %{ "subw $dst, $src1, $src2" %}
10385
10386 ins_encode %{
10387 __ subw(as_Register($dst$$reg),
10388 as_Register($src1$$reg),
10389 as_Register($src2$$reg));
10390 %}
10391
10392 ins_pipe(ialu_reg_reg);
10393 %}
10394
10395 // Immediate Subtraction
10396 instruct subI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immIAddSub src2) %{
10397 match(Set dst (SubI src1 src2));
10398
10399 ins_cost(INSN_COST);
10400 format %{ "subw $dst, $src1, $src2" %}
10401
10402 // use opcode to indicate that this is a sub not an add
10403 opcode(0x1);
10404
10405 ins_encode(aarch64_enc_addsubw_imm(dst, src1, src2));
10406
10407 ins_pipe(ialu_reg_imm);
10408 %}
10409
10410 // Long Subtraction
10411 instruct subL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
10412
10413 match(Set dst (SubL src1 src2));
10414
10415 ins_cost(INSN_COST);
10416 format %{ "sub $dst, $src1, $src2" %}
10417
10418 ins_encode %{
10419 __ sub(as_Register($dst$$reg),
10420 as_Register($src1$$reg),
10421 as_Register($src2$$reg));
10422 %}
10423
10424 ins_pipe(ialu_reg_reg);
10425 %}
10426
10427 // No constant pool entries requiredLong Immediate Subtraction.
10428 instruct subL_reg_imm(iRegLNoSp dst, iRegL src1, immLAddSub src2) %{
10429 match(Set dst (SubL src1 src2));
10430
10431 ins_cost(INSN_COST);
10432 format %{ "sub$dst, $src1, $src2" %}
10433
10434 // use opcode to indicate that this is a sub not an add
10435 opcode(0x1);
10436
10437 ins_encode( aarch64_enc_addsub_imm(dst, src1, src2) );
10438
10439 ins_pipe(ialu_reg_imm);
10440 %}
10441
10442 // Integer Negation (special case for sub)
10443
10444 instruct negI_reg(iRegINoSp dst, iRegIorL2I src, immI0 zero, rFlagsReg cr) %{
10445 match(Set dst (SubI zero src));
10446
10447 ins_cost(INSN_COST);
10448 format %{ "negw $dst, $src\t# int" %}
10449
10450 ins_encode %{
10451 __ negw(as_Register($dst$$reg),
10452 as_Register($src$$reg));
10453 %}
10454
10455 ins_pipe(ialu_reg);
10456 %}
10457
10458 // Long Negation
10459
10460 instruct negL_reg(iRegLNoSp dst, iRegL src, immL0 zero, rFlagsReg cr) %{
10461 match(Set dst (SubL zero src));
10462
10463 ins_cost(INSN_COST);
10464 format %{ "neg $dst, $src\t# long" %}
10465
10466 ins_encode %{
10467 __ neg(as_Register($dst$$reg),
10468 as_Register($src$$reg));
10469 %}
10470
10471 ins_pipe(ialu_reg);
10472 %}
10473
10474 // Integer Multiply
10475
10476 instruct mulI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10477 match(Set dst (MulI src1 src2));
10478
10479 ins_cost(INSN_COST * 3);
10480 format %{ "mulw $dst, $src1, $src2" %}
10481
10482 ins_encode %{
10483 __ mulw(as_Register($dst$$reg),
10484 as_Register($src1$$reg),
10485 as_Register($src2$$reg));
10486 %}
10487
10488 ins_pipe(imul_reg_reg);
10489 %}
10490
10491 instruct smulI(iRegLNoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10492 match(Set dst (MulL (ConvI2L src1) (ConvI2L src2)));
10493
10494 ins_cost(INSN_COST * 3);
10495 format %{ "smull $dst, $src1, $src2" %}
10496
10497 ins_encode %{
10498 __ smull(as_Register($dst$$reg),
10499 as_Register($src1$$reg),
10500 as_Register($src2$$reg));
10501 %}
10502
10503 ins_pipe(imul_reg_reg);
10504 %}
10505
10506 // Long Multiply
10507
10508 instruct mulL(iRegLNoSp dst, iRegL src1, iRegL src2) %{
10509 match(Set dst (MulL src1 src2));
10510
10511 ins_cost(INSN_COST * 5);
10512 format %{ "mul $dst, $src1, $src2" %}
10513
10514 ins_encode %{
10515 __ mul(as_Register($dst$$reg),
10516 as_Register($src1$$reg),
10517 as_Register($src2$$reg));
10518 %}
10519
10520 ins_pipe(lmul_reg_reg);
10521 %}
10522
10523 instruct mulHiL_rReg(iRegLNoSp dst, iRegL src1, iRegL src2, rFlagsReg cr)
10524 %{
10525 match(Set dst (MulHiL src1 src2));
10526
10527 ins_cost(INSN_COST * 7);
10528 format %{ "smulh $dst, $src1, $src2\t# mulhi" %}
10529
10530 ins_encode %{
10531 __ smulh(as_Register($dst$$reg),
10532 as_Register($src1$$reg),
10533 as_Register($src2$$reg));
10534 %}
10535
10536 ins_pipe(lmul_reg_reg);
10537 %}
10538
10539 instruct umulHiL_rReg(iRegLNoSp dst, iRegL src1, iRegL src2, rFlagsReg cr)
10540 %{
10541 match(Set dst (UMulHiL src1 src2));
10542
10543 ins_cost(INSN_COST * 7);
10544 format %{ "umulh $dst, $src1, $src2\t# umulhi" %}
10545
10546 ins_encode %{
10547 __ umulh(as_Register($dst$$reg),
10548 as_Register($src1$$reg),
10549 as_Register($src2$$reg));
10550 %}
10551
10552 ins_pipe(lmul_reg_reg);
10553 %}
10554
10555 // Combined Integer Multiply & Add/Sub
10556
10557 instruct maddI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegIorL2I src3) %{
10558 match(Set dst (AddI src3 (MulI src1 src2)));
10559
10560 ins_cost(INSN_COST * 3);
10561 format %{ "madd $dst, $src1, $src2, $src3" %}
10562
10563 ins_encode %{
10564 __ maddw(as_Register($dst$$reg),
10565 as_Register($src1$$reg),
10566 as_Register($src2$$reg),
10567 as_Register($src3$$reg));
10568 %}
10569
10570 ins_pipe(imac_reg_reg);
10571 %}
10572
10573 instruct msubI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegIorL2I src3) %{
10574 match(Set dst (SubI src3 (MulI src1 src2)));
10575
10576 ins_cost(INSN_COST * 3);
10577 format %{ "msub $dst, $src1, $src2, $src3" %}
10578
10579 ins_encode %{
10580 __ msubw(as_Register($dst$$reg),
10581 as_Register($src1$$reg),
10582 as_Register($src2$$reg),
10583 as_Register($src3$$reg));
10584 %}
10585
10586 ins_pipe(imac_reg_reg);
10587 %}
10588
10589 // Combined Integer Multiply & Neg
10590
10591 instruct mnegI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI0 zero) %{
10592 match(Set dst (MulI (SubI zero src1) src2));
10593
10594 ins_cost(INSN_COST * 3);
10595 format %{ "mneg $dst, $src1, $src2" %}
10596
10597 ins_encode %{
10598 __ mnegw(as_Register($dst$$reg),
10599 as_Register($src1$$reg),
10600 as_Register($src2$$reg));
10601 %}
10602
10603 ins_pipe(imac_reg_reg);
10604 %}
10605
10606 // Combined Long Multiply & Add/Sub
10607
10608 instruct maddL(iRegLNoSp dst, iRegL src1, iRegL src2, iRegL src3) %{
10609 match(Set dst (AddL src3 (MulL src1 src2)));
10610
10611 ins_cost(INSN_COST * 5);
10612 format %{ "madd $dst, $src1, $src2, $src3" %}
10613
10614 ins_encode %{
10615 __ madd(as_Register($dst$$reg),
10616 as_Register($src1$$reg),
10617 as_Register($src2$$reg),
10618 as_Register($src3$$reg));
10619 %}
10620
10621 ins_pipe(lmac_reg_reg);
10622 %}
10623
10624 instruct msubL(iRegLNoSp dst, iRegL src1, iRegL src2, iRegL src3) %{
10625 match(Set dst (SubL src3 (MulL src1 src2)));
10626
10627 ins_cost(INSN_COST * 5);
10628 format %{ "msub $dst, $src1, $src2, $src3" %}
10629
10630 ins_encode %{
10631 __ msub(as_Register($dst$$reg),
10632 as_Register($src1$$reg),
10633 as_Register($src2$$reg),
10634 as_Register($src3$$reg));
10635 %}
10636
10637 ins_pipe(lmac_reg_reg);
10638 %}
10639
10640 // Combined Long Multiply & Neg
10641
10642 instruct mnegL(iRegLNoSp dst, iRegL src1, iRegL src2, immL0 zero) %{
10643 match(Set dst (MulL (SubL zero src1) src2));
10644
10645 ins_cost(INSN_COST * 5);
10646 format %{ "mneg $dst, $src1, $src2" %}
10647
10648 ins_encode %{
10649 __ mneg(as_Register($dst$$reg),
10650 as_Register($src1$$reg),
10651 as_Register($src2$$reg));
10652 %}
10653
10654 ins_pipe(lmac_reg_reg);
10655 %}
10656
10657 // Combine Integer Signed Multiply & Add/Sub/Neg Long
10658
10659 instruct smaddL(iRegLNoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegLNoSp src3) %{
10660 match(Set dst (AddL src3 (MulL (ConvI2L src1) (ConvI2L src2))));
10661
10662 ins_cost(INSN_COST * 3);
10663 format %{ "smaddl $dst, $src1, $src2, $src3" %}
10664
10665 ins_encode %{
10666 __ smaddl(as_Register($dst$$reg),
10667 as_Register($src1$$reg),
10668 as_Register($src2$$reg),
10669 as_Register($src3$$reg));
10670 %}
10671
10672 ins_pipe(imac_reg_reg);
10673 %}
10674
10675 instruct smsubL(iRegLNoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegLNoSp src3) %{
10676 match(Set dst (SubL src3 (MulL (ConvI2L src1) (ConvI2L src2))));
10677
10678 ins_cost(INSN_COST * 3);
10679 format %{ "smsubl $dst, $src1, $src2, $src3" %}
10680
10681 ins_encode %{
10682 __ smsubl(as_Register($dst$$reg),
10683 as_Register($src1$$reg),
10684 as_Register($src2$$reg),
10685 as_Register($src3$$reg));
10686 %}
10687
10688 ins_pipe(imac_reg_reg);
10689 %}
10690
10691 instruct smnegL(iRegLNoSp dst, iRegIorL2I src1, iRegIorL2I src2, immL0 zero) %{
10692 match(Set dst (MulL (SubL zero (ConvI2L src1)) (ConvI2L src2)));
10693
10694 ins_cost(INSN_COST * 3);
10695 format %{ "smnegl $dst, $src1, $src2" %}
10696
10697 ins_encode %{
10698 __ smnegl(as_Register($dst$$reg),
10699 as_Register($src1$$reg),
10700 as_Register($src2$$reg));
10701 %}
10702
10703 ins_pipe(imac_reg_reg);
10704 %}
10705
10706 // Combined Multiply-Add Shorts into Integer (dst = src1 * src2 + src3 * src4)
10707
10708 instruct muladdS2I(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegIorL2I src3, iRegIorL2I src4) %{
10709 match(Set dst (MulAddS2I (Binary src1 src2) (Binary src3 src4)));
10710
10711 ins_cost(INSN_COST * 5);
10712 format %{ "mulw rscratch1, $src1, $src2\n\t"
10713 "maddw $dst, $src3, $src4, rscratch1" %}
10714
10715 ins_encode %{
10716 __ mulw(rscratch1, as_Register($src1$$reg), as_Register($src2$$reg));
10717 __ maddw(as_Register($dst$$reg), as_Register($src3$$reg), as_Register($src4$$reg), rscratch1); %}
10718
10719 ins_pipe(imac_reg_reg);
10720 %}
10721
10722 // Integer Divide
10723
10724 instruct divI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10725 match(Set dst (DivI src1 src2));
10726
10727 ins_cost(INSN_COST * 19);
10728 format %{ "sdivw $dst, $src1, $src2" %}
10729
10730 ins_encode(aarch64_enc_divw(dst, src1, src2));
10731 ins_pipe(idiv_reg_reg);
10732 %}
10733
10734 // Long Divide
10735
10736 instruct divL(iRegLNoSp dst, iRegL src1, iRegL src2) %{
10737 match(Set dst (DivL src1 src2));
10738
10739 ins_cost(INSN_COST * 35);
10740 format %{ "sdiv $dst, $src1, $src2" %}
10741
10742 ins_encode(aarch64_enc_div(dst, src1, src2));
10743 ins_pipe(ldiv_reg_reg);
10744 %}
10745
10746 // Integer Remainder
10747
10748 instruct modI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10749 match(Set dst (ModI src1 src2));
10750
10751 ins_cost(INSN_COST * 22);
10752 format %{ "sdivw rscratch1, $src1, $src2\n\t"
10753 "msubw $dst, rscratch1, $src2, $src1" %}
10754
10755 ins_encode(aarch64_enc_modw(dst, src1, src2));
10756 ins_pipe(idiv_reg_reg);
10757 %}
10758
10759 // Long Remainder
10760
10761 instruct modL(iRegLNoSp dst, iRegL src1, iRegL src2) %{
10762 match(Set dst (ModL src1 src2));
10763
10764 ins_cost(INSN_COST * 38);
10765 format %{ "sdiv rscratch1, $src1, $src2\n"
10766 "msub $dst, rscratch1, $src2, $src1" %}
10767
10768 ins_encode(aarch64_enc_mod(dst, src1, src2));
10769 ins_pipe(ldiv_reg_reg);
10770 %}
10771
10772 // Unsigned Integer Divide
10773
10774 instruct UdivI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10775 match(Set dst (UDivI src1 src2));
10776
10777 ins_cost(INSN_COST * 19);
10778 format %{ "udivw $dst, $src1, $src2" %}
10779
10780 ins_encode %{
10781 __ udivw($dst$$Register, $src1$$Register, $src2$$Register);
10782 %}
10783
10784 ins_pipe(idiv_reg_reg);
10785 %}
10786
10787 // Unsigned Long Divide
10788
10789 instruct UdivL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
10790 match(Set dst (UDivL src1 src2));
10791
10792 ins_cost(INSN_COST * 35);
10793 format %{ "udiv $dst, $src1, $src2" %}
10794
10795 ins_encode %{
10796 __ udiv($dst$$Register, $src1$$Register, $src2$$Register);
10797 %}
10798
10799 ins_pipe(ldiv_reg_reg);
10800 %}
10801
10802 // Unsigned Integer Remainder
10803
10804 instruct UmodI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10805 match(Set dst (UModI src1 src2));
10806
10807 ins_cost(INSN_COST * 22);
10808 format %{ "udivw rscratch1, $src1, $src2\n\t"
10809 "msubw $dst, rscratch1, $src2, $src1" %}
10810
10811 ins_encode %{
10812 __ udivw(rscratch1, $src1$$Register, $src2$$Register);
10813 __ msubw($dst$$Register, rscratch1, $src2$$Register, $src1$$Register);
10814 %}
10815
10816 ins_pipe(idiv_reg_reg);
10817 %}
10818
10819 // Unsigned Long Remainder
10820
10821 instruct UModL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
10822 match(Set dst (UModL src1 src2));
10823
10824 ins_cost(INSN_COST * 38);
10825 format %{ "udiv rscratch1, $src1, $src2\n"
10826 "msub $dst, rscratch1, $src2, $src1" %}
10827
10828 ins_encode %{
10829 __ udiv(rscratch1, $src1$$Register, $src2$$Register);
10830 __ msub($dst$$Register, rscratch1, $src2$$Register, $src1$$Register);
10831 %}
10832
10833 ins_pipe(ldiv_reg_reg);
10834 %}
10835
10836 // Integer Shifts
10837
10838 // Shift Left Register
10839 instruct lShiftI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10840 match(Set dst (LShiftI src1 src2));
10841
10842 ins_cost(INSN_COST * 2);
10843 format %{ "lslvw $dst, $src1, $src2" %}
10844
10845 ins_encode %{
10846 __ lslvw(as_Register($dst$$reg),
10847 as_Register($src1$$reg),
10848 as_Register($src2$$reg));
10849 %}
10850
10851 ins_pipe(ialu_reg_reg_vshift);
10852 %}
10853
10854 // Shift Left Immediate
10855 instruct lShiftI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immI src2) %{
10856 match(Set dst (LShiftI src1 src2));
10857
10858 ins_cost(INSN_COST);
10859 format %{ "lslw $dst, $src1, ($src2 & 0x1f)" %}
10860
10861 ins_encode %{
10862 __ lslw(as_Register($dst$$reg),
10863 as_Register($src1$$reg),
10864 $src2$$constant & 0x1f);
10865 %}
10866
10867 ins_pipe(ialu_reg_shift);
10868 %}
10869
10870 // Shift Right Logical Register
10871 instruct urShiftI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10872 match(Set dst (URShiftI src1 src2));
10873
10874 ins_cost(INSN_COST * 2);
10875 format %{ "lsrvw $dst, $src1, $src2" %}
10876
10877 ins_encode %{
10878 __ lsrvw(as_Register($dst$$reg),
10879 as_Register($src1$$reg),
10880 as_Register($src2$$reg));
10881 %}
10882
10883 ins_pipe(ialu_reg_reg_vshift);
10884 %}
10885
10886 // Shift Right Logical Immediate
10887 instruct urShiftI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immI src2) %{
10888 match(Set dst (URShiftI src1 src2));
10889
10890 ins_cost(INSN_COST);
10891 format %{ "lsrw $dst, $src1, ($src2 & 0x1f)" %}
10892
10893 ins_encode %{
10894 __ lsrw(as_Register($dst$$reg),
10895 as_Register($src1$$reg),
10896 $src2$$constant & 0x1f);
10897 %}
10898
10899 ins_pipe(ialu_reg_shift);
10900 %}
10901
10902 // Shift Right Arithmetic Register
10903 instruct rShiftI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10904 match(Set dst (RShiftI src1 src2));
10905
10906 ins_cost(INSN_COST * 2);
10907 format %{ "asrvw $dst, $src1, $src2" %}
10908
10909 ins_encode %{
10910 __ asrvw(as_Register($dst$$reg),
10911 as_Register($src1$$reg),
10912 as_Register($src2$$reg));
10913 %}
10914
10915 ins_pipe(ialu_reg_reg_vshift);
10916 %}
10917
10918 // Shift Right Arithmetic Immediate
10919 instruct rShiftI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immI src2) %{
10920 match(Set dst (RShiftI src1 src2));
10921
10922 ins_cost(INSN_COST);
10923 format %{ "asrw $dst, $src1, ($src2 & 0x1f)" %}
10924
10925 ins_encode %{
10926 __ asrw(as_Register($dst$$reg),
10927 as_Register($src1$$reg),
10928 $src2$$constant & 0x1f);
10929 %}
10930
10931 ins_pipe(ialu_reg_shift);
10932 %}
10933
10934 // Combined Int Mask and Right Shift (using UBFM)
10935 // TODO
10936
10937 // Long Shifts
10938
10939 // Shift Left Register
10940 instruct lShiftL_reg_reg(iRegLNoSp dst, iRegL src1, iRegIorL2I src2) %{
10941 match(Set dst (LShiftL src1 src2));
10942
10943 ins_cost(INSN_COST * 2);
10944 format %{ "lslv $dst, $src1, $src2" %}
10945
10946 ins_encode %{
10947 __ lslv(as_Register($dst$$reg),
10948 as_Register($src1$$reg),
10949 as_Register($src2$$reg));
10950 %}
10951
10952 ins_pipe(ialu_reg_reg_vshift);
10953 %}
10954
10955 // Shift Left Immediate
10956 instruct lShiftL_reg_imm(iRegLNoSp dst, iRegL src1, immI src2) %{
10957 match(Set dst (LShiftL src1 src2));
10958
10959 ins_cost(INSN_COST);
10960 format %{ "lsl $dst, $src1, ($src2 & 0x3f)" %}
10961
10962 ins_encode %{
10963 __ lsl(as_Register($dst$$reg),
10964 as_Register($src1$$reg),
10965 $src2$$constant & 0x3f);
10966 %}
10967
10968 ins_pipe(ialu_reg_shift);
10969 %}
10970
10971 // Shift Right Logical Register
10972 instruct urShiftL_reg_reg(iRegLNoSp dst, iRegL src1, iRegIorL2I src2) %{
10973 match(Set dst (URShiftL src1 src2));
10974
10975 ins_cost(INSN_COST * 2);
10976 format %{ "lsrv $dst, $src1, $src2" %}
10977
10978 ins_encode %{
10979 __ lsrv(as_Register($dst$$reg),
10980 as_Register($src1$$reg),
10981 as_Register($src2$$reg));
10982 %}
10983
10984 ins_pipe(ialu_reg_reg_vshift);
10985 %}
10986
10987 // Shift Right Logical Immediate
10988 instruct urShiftL_reg_imm(iRegLNoSp dst, iRegL src1, immI src2) %{
10989 match(Set dst (URShiftL src1 src2));
10990
10991 ins_cost(INSN_COST);
10992 format %{ "lsr $dst, $src1, ($src2 & 0x3f)" %}
10993
10994 ins_encode %{
10995 __ lsr(as_Register($dst$$reg),
10996 as_Register($src1$$reg),
10997 $src2$$constant & 0x3f);
10998 %}
10999
11000 ins_pipe(ialu_reg_shift);
11001 %}
11002
11003 // A special-case pattern for card table stores.
11004 instruct urShiftP_reg_imm(iRegLNoSp dst, iRegP src1, immI src2) %{
11005 match(Set dst (URShiftL (CastP2X src1) src2));
11006
11007 ins_cost(INSN_COST);
11008 format %{ "lsr $dst, p2x($src1), ($src2 & 0x3f)" %}
11009
11010 ins_encode %{
11011 __ lsr(as_Register($dst$$reg),
11012 as_Register($src1$$reg),
11013 $src2$$constant & 0x3f);
11014 %}
11015
11016 ins_pipe(ialu_reg_shift);
11017 %}
11018
11019 // Shift Right Arithmetic Register
11020 instruct rShiftL_reg_reg(iRegLNoSp dst, iRegL src1, iRegIorL2I src2) %{
11021 match(Set dst (RShiftL src1 src2));
11022
11023 ins_cost(INSN_COST * 2);
11024 format %{ "asrv $dst, $src1, $src2" %}
11025
11026 ins_encode %{
11027 __ asrv(as_Register($dst$$reg),
11028 as_Register($src1$$reg),
11029 as_Register($src2$$reg));
11030 %}
11031
11032 ins_pipe(ialu_reg_reg_vshift);
11033 %}
11034
11035 // Shift Right Arithmetic Immediate
11036 instruct rShiftL_reg_imm(iRegLNoSp dst, iRegL src1, immI src2) %{
11037 match(Set dst (RShiftL src1 src2));
11038
11039 ins_cost(INSN_COST);
11040 format %{ "asr $dst, $src1, ($src2 & 0x3f)" %}
11041
11042 ins_encode %{
11043 __ asr(as_Register($dst$$reg),
11044 as_Register($src1$$reg),
11045 $src2$$constant & 0x3f);
11046 %}
11047
11048 ins_pipe(ialu_reg_shift);
11049 %}
11050
11051 // BEGIN This section of the file is automatically generated. Do not edit --------------
11052 // This section is generated from aarch64_ad.m4
11053
11054
11055 // This pattern is automatically generated from aarch64_ad.m4
11056 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11057 instruct regL_not_reg(iRegLNoSp dst,
11058 iRegL src1, immL_M1 m1,
11059 rFlagsReg cr) %{
11060 match(Set dst (XorL src1 m1));
11061 ins_cost(INSN_COST);
11062 format %{ "eon $dst, $src1, zr" %}
11063
11064 ins_encode %{
11065 __ eon(as_Register($dst$$reg),
11066 as_Register($src1$$reg),
11067 zr,
11068 Assembler::LSL, 0);
11069 %}
11070
11071 ins_pipe(ialu_reg);
11072 %}
11073
11074 // This pattern is automatically generated from aarch64_ad.m4
11075 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11076 instruct regI_not_reg(iRegINoSp dst,
11077 iRegIorL2I src1, immI_M1 m1,
11078 rFlagsReg cr) %{
11079 match(Set dst (XorI src1 m1));
11080 ins_cost(INSN_COST);
11081 format %{ "eonw $dst, $src1, zr" %}
11082
11083 ins_encode %{
11084 __ eonw(as_Register($dst$$reg),
11085 as_Register($src1$$reg),
11086 zr,
11087 Assembler::LSL, 0);
11088 %}
11089
11090 ins_pipe(ialu_reg);
11091 %}
11092
11093 // This pattern is automatically generated from aarch64_ad.m4
11094 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11095 instruct NegI_reg_URShift_reg(iRegINoSp dst,
11096 immI0 zero, iRegIorL2I src1, immI src2) %{
11097 match(Set dst (SubI zero (URShiftI src1 src2)));
11098
11099 ins_cost(1.9 * INSN_COST);
11100 format %{ "negw $dst, $src1, LSR $src2" %}
11101
11102 ins_encode %{
11103 __ negw(as_Register($dst$$reg), as_Register($src1$$reg),
11104 Assembler::LSR, $src2$$constant & 0x1f);
11105 %}
11106
11107 ins_pipe(ialu_reg_shift);
11108 %}
11109
11110 // This pattern is automatically generated from aarch64_ad.m4
11111 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11112 instruct NegI_reg_RShift_reg(iRegINoSp dst,
11113 immI0 zero, iRegIorL2I src1, immI src2) %{
11114 match(Set dst (SubI zero (RShiftI src1 src2)));
11115
11116 ins_cost(1.9 * INSN_COST);
11117 format %{ "negw $dst, $src1, ASR $src2" %}
11118
11119 ins_encode %{
11120 __ negw(as_Register($dst$$reg), as_Register($src1$$reg),
11121 Assembler::ASR, $src2$$constant & 0x1f);
11122 %}
11123
11124 ins_pipe(ialu_reg_shift);
11125 %}
11126
11127 // This pattern is automatically generated from aarch64_ad.m4
11128 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11129 instruct NegI_reg_LShift_reg(iRegINoSp dst,
11130 immI0 zero, iRegIorL2I src1, immI src2) %{
11131 match(Set dst (SubI zero (LShiftI src1 src2)));
11132
11133 ins_cost(1.9 * INSN_COST);
11134 format %{ "negw $dst, $src1, LSL $src2" %}
11135
11136 ins_encode %{
11137 __ negw(as_Register($dst$$reg), as_Register($src1$$reg),
11138 Assembler::LSL, $src2$$constant & 0x1f);
11139 %}
11140
11141 ins_pipe(ialu_reg_shift);
11142 %}
11143
11144 // This pattern is automatically generated from aarch64_ad.m4
11145 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11146 instruct NegL_reg_URShift_reg(iRegLNoSp dst,
11147 immL0 zero, iRegL src1, immI src2) %{
11148 match(Set dst (SubL zero (URShiftL src1 src2)));
11149
11150 ins_cost(1.9 * INSN_COST);
11151 format %{ "neg $dst, $src1, LSR $src2" %}
11152
11153 ins_encode %{
11154 __ neg(as_Register($dst$$reg), as_Register($src1$$reg),
11155 Assembler::LSR, $src2$$constant & 0x3f);
11156 %}
11157
11158 ins_pipe(ialu_reg_shift);
11159 %}
11160
11161 // This pattern is automatically generated from aarch64_ad.m4
11162 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11163 instruct NegL_reg_RShift_reg(iRegLNoSp dst,
11164 immL0 zero, iRegL src1, immI src2) %{
11165 match(Set dst (SubL zero (RShiftL src1 src2)));
11166
11167 ins_cost(1.9 * INSN_COST);
11168 format %{ "neg $dst, $src1, ASR $src2" %}
11169
11170 ins_encode %{
11171 __ neg(as_Register($dst$$reg), as_Register($src1$$reg),
11172 Assembler::ASR, $src2$$constant & 0x3f);
11173 %}
11174
11175 ins_pipe(ialu_reg_shift);
11176 %}
11177
11178 // This pattern is automatically generated from aarch64_ad.m4
11179 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11180 instruct NegL_reg_LShift_reg(iRegLNoSp dst,
11181 immL0 zero, iRegL src1, immI src2) %{
11182 match(Set dst (SubL zero (LShiftL src1 src2)));
11183
11184 ins_cost(1.9 * INSN_COST);
11185 format %{ "neg $dst, $src1, LSL $src2" %}
11186
11187 ins_encode %{
11188 __ neg(as_Register($dst$$reg), as_Register($src1$$reg),
11189 Assembler::LSL, $src2$$constant & 0x3f);
11190 %}
11191
11192 ins_pipe(ialu_reg_shift);
11193 %}
11194
11195 // This pattern is automatically generated from aarch64_ad.m4
11196 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11197 instruct AndI_reg_not_reg(iRegINoSp dst,
11198 iRegIorL2I src1, iRegIorL2I src2, immI_M1 m1) %{
11199 match(Set dst (AndI src1 (XorI src2 m1)));
11200 ins_cost(INSN_COST);
11201 format %{ "bicw $dst, $src1, $src2" %}
11202
11203 ins_encode %{
11204 __ bicw(as_Register($dst$$reg),
11205 as_Register($src1$$reg),
11206 as_Register($src2$$reg),
11207 Assembler::LSL, 0);
11208 %}
11209
11210 ins_pipe(ialu_reg_reg);
11211 %}
11212
11213 // This pattern is automatically generated from aarch64_ad.m4
11214 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11215 instruct AndL_reg_not_reg(iRegLNoSp dst,
11216 iRegL src1, iRegL src2, immL_M1 m1) %{
11217 match(Set dst (AndL src1 (XorL src2 m1)));
11218 ins_cost(INSN_COST);
11219 format %{ "bic $dst, $src1, $src2" %}
11220
11221 ins_encode %{
11222 __ bic(as_Register($dst$$reg),
11223 as_Register($src1$$reg),
11224 as_Register($src2$$reg),
11225 Assembler::LSL, 0);
11226 %}
11227
11228 ins_pipe(ialu_reg_reg);
11229 %}
11230
11231 // This pattern is automatically generated from aarch64_ad.m4
11232 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11233 instruct OrI_reg_not_reg(iRegINoSp dst,
11234 iRegIorL2I src1, iRegIorL2I src2, immI_M1 m1) %{
11235 match(Set dst (OrI src1 (XorI src2 m1)));
11236 ins_cost(INSN_COST);
11237 format %{ "ornw $dst, $src1, $src2" %}
11238
11239 ins_encode %{
11240 __ ornw(as_Register($dst$$reg),
11241 as_Register($src1$$reg),
11242 as_Register($src2$$reg),
11243 Assembler::LSL, 0);
11244 %}
11245
11246 ins_pipe(ialu_reg_reg);
11247 %}
11248
11249 // This pattern is automatically generated from aarch64_ad.m4
11250 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11251 instruct OrL_reg_not_reg(iRegLNoSp dst,
11252 iRegL src1, iRegL src2, immL_M1 m1) %{
11253 match(Set dst (OrL src1 (XorL src2 m1)));
11254 ins_cost(INSN_COST);
11255 format %{ "orn $dst, $src1, $src2" %}
11256
11257 ins_encode %{
11258 __ orn(as_Register($dst$$reg),
11259 as_Register($src1$$reg),
11260 as_Register($src2$$reg),
11261 Assembler::LSL, 0);
11262 %}
11263
11264 ins_pipe(ialu_reg_reg);
11265 %}
11266
11267 // This pattern is automatically generated from aarch64_ad.m4
11268 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11269 instruct XorI_reg_not_reg(iRegINoSp dst,
11270 iRegIorL2I src1, iRegIorL2I src2, immI_M1 m1) %{
11271 match(Set dst (XorI m1 (XorI src2 src1)));
11272 ins_cost(INSN_COST);
11273 format %{ "eonw $dst, $src1, $src2" %}
11274
11275 ins_encode %{
11276 __ eonw(as_Register($dst$$reg),
11277 as_Register($src1$$reg),
11278 as_Register($src2$$reg),
11279 Assembler::LSL, 0);
11280 %}
11281
11282 ins_pipe(ialu_reg_reg);
11283 %}
11284
11285 // This pattern is automatically generated from aarch64_ad.m4
11286 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11287 instruct XorL_reg_not_reg(iRegLNoSp dst,
11288 iRegL src1, iRegL src2, immL_M1 m1) %{
11289 match(Set dst (XorL m1 (XorL src2 src1)));
11290 ins_cost(INSN_COST);
11291 format %{ "eon $dst, $src1, $src2" %}
11292
11293 ins_encode %{
11294 __ eon(as_Register($dst$$reg),
11295 as_Register($src1$$reg),
11296 as_Register($src2$$reg),
11297 Assembler::LSL, 0);
11298 %}
11299
11300 ins_pipe(ialu_reg_reg);
11301 %}
11302
11303 // This pattern is automatically generated from aarch64_ad.m4
11304 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11305 // val & (-1 ^ (val >>> shift)) ==> bicw
11306 instruct AndI_reg_URShift_not_reg(iRegINoSp dst,
11307 iRegIorL2I src1, iRegIorL2I src2,
11308 immI src3, immI_M1 src4) %{
11309 match(Set dst (AndI src1 (XorI(URShiftI src2 src3) src4)));
11310 ins_cost(1.9 * INSN_COST);
11311 format %{ "bicw $dst, $src1, $src2, LSR $src3" %}
11312
11313 ins_encode %{
11314 __ bicw(as_Register($dst$$reg),
11315 as_Register($src1$$reg),
11316 as_Register($src2$$reg),
11317 Assembler::LSR,
11318 $src3$$constant & 0x1f);
11319 %}
11320
11321 ins_pipe(ialu_reg_reg_shift);
11322 %}
11323
11324 // This pattern is automatically generated from aarch64_ad.m4
11325 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11326 // val & (-1 ^ (val >>> shift)) ==> bic
11327 instruct AndL_reg_URShift_not_reg(iRegLNoSp dst,
11328 iRegL src1, iRegL src2,
11329 immI src3, immL_M1 src4) %{
11330 match(Set dst (AndL src1 (XorL(URShiftL src2 src3) src4)));
11331 ins_cost(1.9 * INSN_COST);
11332 format %{ "bic $dst, $src1, $src2, LSR $src3" %}
11333
11334 ins_encode %{
11335 __ bic(as_Register($dst$$reg),
11336 as_Register($src1$$reg),
11337 as_Register($src2$$reg),
11338 Assembler::LSR,
11339 $src3$$constant & 0x3f);
11340 %}
11341
11342 ins_pipe(ialu_reg_reg_shift);
11343 %}
11344
11345 // This pattern is automatically generated from aarch64_ad.m4
11346 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11347 // val & (-1 ^ (val >> shift)) ==> bicw
11348 instruct AndI_reg_RShift_not_reg(iRegINoSp dst,
11349 iRegIorL2I src1, iRegIorL2I src2,
11350 immI src3, immI_M1 src4) %{
11351 match(Set dst (AndI src1 (XorI(RShiftI src2 src3) src4)));
11352 ins_cost(1.9 * INSN_COST);
11353 format %{ "bicw $dst, $src1, $src2, ASR $src3" %}
11354
11355 ins_encode %{
11356 __ bicw(as_Register($dst$$reg),
11357 as_Register($src1$$reg),
11358 as_Register($src2$$reg),
11359 Assembler::ASR,
11360 $src3$$constant & 0x1f);
11361 %}
11362
11363 ins_pipe(ialu_reg_reg_shift);
11364 %}
11365
11366 // This pattern is automatically generated from aarch64_ad.m4
11367 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11368 // val & (-1 ^ (val >> shift)) ==> bic
11369 instruct AndL_reg_RShift_not_reg(iRegLNoSp dst,
11370 iRegL src1, iRegL src2,
11371 immI src3, immL_M1 src4) %{
11372 match(Set dst (AndL src1 (XorL(RShiftL src2 src3) src4)));
11373 ins_cost(1.9 * INSN_COST);
11374 format %{ "bic $dst, $src1, $src2, ASR $src3" %}
11375
11376 ins_encode %{
11377 __ bic(as_Register($dst$$reg),
11378 as_Register($src1$$reg),
11379 as_Register($src2$$reg),
11380 Assembler::ASR,
11381 $src3$$constant & 0x3f);
11382 %}
11383
11384 ins_pipe(ialu_reg_reg_shift);
11385 %}
11386
11387 // This pattern is automatically generated from aarch64_ad.m4
11388 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11389 // val & (-1 ^ (val ror shift)) ==> bicw
11390 instruct AndI_reg_RotateRight_not_reg(iRegINoSp dst,
11391 iRegIorL2I src1, iRegIorL2I src2,
11392 immI src3, immI_M1 src4) %{
11393 match(Set dst (AndI src1 (XorI(RotateRight src2 src3) src4)));
11394 ins_cost(1.9 * INSN_COST);
11395 format %{ "bicw $dst, $src1, $src2, ROR $src3" %}
11396
11397 ins_encode %{
11398 __ bicw(as_Register($dst$$reg),
11399 as_Register($src1$$reg),
11400 as_Register($src2$$reg),
11401 Assembler::ROR,
11402 $src3$$constant & 0x1f);
11403 %}
11404
11405 ins_pipe(ialu_reg_reg_shift);
11406 %}
11407
11408 // This pattern is automatically generated from aarch64_ad.m4
11409 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11410 // val & (-1 ^ (val ror shift)) ==> bic
11411 instruct AndL_reg_RotateRight_not_reg(iRegLNoSp dst,
11412 iRegL src1, iRegL src2,
11413 immI src3, immL_M1 src4) %{
11414 match(Set dst (AndL src1 (XorL(RotateRight src2 src3) src4)));
11415 ins_cost(1.9 * INSN_COST);
11416 format %{ "bic $dst, $src1, $src2, ROR $src3" %}
11417
11418 ins_encode %{
11419 __ bic(as_Register($dst$$reg),
11420 as_Register($src1$$reg),
11421 as_Register($src2$$reg),
11422 Assembler::ROR,
11423 $src3$$constant & 0x3f);
11424 %}
11425
11426 ins_pipe(ialu_reg_reg_shift);
11427 %}
11428
11429 // This pattern is automatically generated from aarch64_ad.m4
11430 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11431 // val & (-1 ^ (val << shift)) ==> bicw
11432 instruct AndI_reg_LShift_not_reg(iRegINoSp dst,
11433 iRegIorL2I src1, iRegIorL2I src2,
11434 immI src3, immI_M1 src4) %{
11435 match(Set dst (AndI src1 (XorI(LShiftI src2 src3) src4)));
11436 ins_cost(1.9 * INSN_COST);
11437 format %{ "bicw $dst, $src1, $src2, LSL $src3" %}
11438
11439 ins_encode %{
11440 __ bicw(as_Register($dst$$reg),
11441 as_Register($src1$$reg),
11442 as_Register($src2$$reg),
11443 Assembler::LSL,
11444 $src3$$constant & 0x1f);
11445 %}
11446
11447 ins_pipe(ialu_reg_reg_shift);
11448 %}
11449
11450 // This pattern is automatically generated from aarch64_ad.m4
11451 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11452 // val & (-1 ^ (val << shift)) ==> bic
11453 instruct AndL_reg_LShift_not_reg(iRegLNoSp dst,
11454 iRegL src1, iRegL src2,
11455 immI src3, immL_M1 src4) %{
11456 match(Set dst (AndL src1 (XorL(LShiftL src2 src3) src4)));
11457 ins_cost(1.9 * INSN_COST);
11458 format %{ "bic $dst, $src1, $src2, LSL $src3" %}
11459
11460 ins_encode %{
11461 __ bic(as_Register($dst$$reg),
11462 as_Register($src1$$reg),
11463 as_Register($src2$$reg),
11464 Assembler::LSL,
11465 $src3$$constant & 0x3f);
11466 %}
11467
11468 ins_pipe(ialu_reg_reg_shift);
11469 %}
11470
11471 // This pattern is automatically generated from aarch64_ad.m4
11472 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11473 // val ^ (-1 ^ (val >>> shift)) ==> eonw
11474 instruct XorI_reg_URShift_not_reg(iRegINoSp dst,
11475 iRegIorL2I src1, iRegIorL2I src2,
11476 immI src3, immI_M1 src4) %{
11477 match(Set dst (XorI src4 (XorI(URShiftI src2 src3) src1)));
11478 ins_cost(1.9 * INSN_COST);
11479 format %{ "eonw $dst, $src1, $src2, LSR $src3" %}
11480
11481 ins_encode %{
11482 __ eonw(as_Register($dst$$reg),
11483 as_Register($src1$$reg),
11484 as_Register($src2$$reg),
11485 Assembler::LSR,
11486 $src3$$constant & 0x1f);
11487 %}
11488
11489 ins_pipe(ialu_reg_reg_shift);
11490 %}
11491
11492 // This pattern is automatically generated from aarch64_ad.m4
11493 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11494 // val ^ (-1 ^ (val >>> shift)) ==> eon
11495 instruct XorL_reg_URShift_not_reg(iRegLNoSp dst,
11496 iRegL src1, iRegL src2,
11497 immI src3, immL_M1 src4) %{
11498 match(Set dst (XorL src4 (XorL(URShiftL src2 src3) src1)));
11499 ins_cost(1.9 * INSN_COST);
11500 format %{ "eon $dst, $src1, $src2, LSR $src3" %}
11501
11502 ins_encode %{
11503 __ eon(as_Register($dst$$reg),
11504 as_Register($src1$$reg),
11505 as_Register($src2$$reg),
11506 Assembler::LSR,
11507 $src3$$constant & 0x3f);
11508 %}
11509
11510 ins_pipe(ialu_reg_reg_shift);
11511 %}
11512
11513 // This pattern is automatically generated from aarch64_ad.m4
11514 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11515 // val ^ (-1 ^ (val >> shift)) ==> eonw
11516 instruct XorI_reg_RShift_not_reg(iRegINoSp dst,
11517 iRegIorL2I src1, iRegIorL2I src2,
11518 immI src3, immI_M1 src4) %{
11519 match(Set dst (XorI src4 (XorI(RShiftI src2 src3) src1)));
11520 ins_cost(1.9 * INSN_COST);
11521 format %{ "eonw $dst, $src1, $src2, ASR $src3" %}
11522
11523 ins_encode %{
11524 __ eonw(as_Register($dst$$reg),
11525 as_Register($src1$$reg),
11526 as_Register($src2$$reg),
11527 Assembler::ASR,
11528 $src3$$constant & 0x1f);
11529 %}
11530
11531 ins_pipe(ialu_reg_reg_shift);
11532 %}
11533
11534 // This pattern is automatically generated from aarch64_ad.m4
11535 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11536 // val ^ (-1 ^ (val >> shift)) ==> eon
11537 instruct XorL_reg_RShift_not_reg(iRegLNoSp dst,
11538 iRegL src1, iRegL src2,
11539 immI src3, immL_M1 src4) %{
11540 match(Set dst (XorL src4 (XorL(RShiftL src2 src3) src1)));
11541 ins_cost(1.9 * INSN_COST);
11542 format %{ "eon $dst, $src1, $src2, ASR $src3" %}
11543
11544 ins_encode %{
11545 __ eon(as_Register($dst$$reg),
11546 as_Register($src1$$reg),
11547 as_Register($src2$$reg),
11548 Assembler::ASR,
11549 $src3$$constant & 0x3f);
11550 %}
11551
11552 ins_pipe(ialu_reg_reg_shift);
11553 %}
11554
11555 // This pattern is automatically generated from aarch64_ad.m4
11556 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11557 // val ^ (-1 ^ (val ror shift)) ==> eonw
11558 instruct XorI_reg_RotateRight_not_reg(iRegINoSp dst,
11559 iRegIorL2I src1, iRegIorL2I src2,
11560 immI src3, immI_M1 src4) %{
11561 match(Set dst (XorI src4 (XorI(RotateRight src2 src3) src1)));
11562 ins_cost(1.9 * INSN_COST);
11563 format %{ "eonw $dst, $src1, $src2, ROR $src3" %}
11564
11565 ins_encode %{
11566 __ eonw(as_Register($dst$$reg),
11567 as_Register($src1$$reg),
11568 as_Register($src2$$reg),
11569 Assembler::ROR,
11570 $src3$$constant & 0x1f);
11571 %}
11572
11573 ins_pipe(ialu_reg_reg_shift);
11574 %}
11575
11576 // This pattern is automatically generated from aarch64_ad.m4
11577 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11578 // val ^ (-1 ^ (val ror shift)) ==> eon
11579 instruct XorL_reg_RotateRight_not_reg(iRegLNoSp dst,
11580 iRegL src1, iRegL src2,
11581 immI src3, immL_M1 src4) %{
11582 match(Set dst (XorL src4 (XorL(RotateRight src2 src3) src1)));
11583 ins_cost(1.9 * INSN_COST);
11584 format %{ "eon $dst, $src1, $src2, ROR $src3" %}
11585
11586 ins_encode %{
11587 __ eon(as_Register($dst$$reg),
11588 as_Register($src1$$reg),
11589 as_Register($src2$$reg),
11590 Assembler::ROR,
11591 $src3$$constant & 0x3f);
11592 %}
11593
11594 ins_pipe(ialu_reg_reg_shift);
11595 %}
11596
11597 // This pattern is automatically generated from aarch64_ad.m4
11598 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11599 // val ^ (-1 ^ (val << shift)) ==> eonw
11600 instruct XorI_reg_LShift_not_reg(iRegINoSp dst,
11601 iRegIorL2I src1, iRegIorL2I src2,
11602 immI src3, immI_M1 src4) %{
11603 match(Set dst (XorI src4 (XorI(LShiftI src2 src3) src1)));
11604 ins_cost(1.9 * INSN_COST);
11605 format %{ "eonw $dst, $src1, $src2, LSL $src3" %}
11606
11607 ins_encode %{
11608 __ eonw(as_Register($dst$$reg),
11609 as_Register($src1$$reg),
11610 as_Register($src2$$reg),
11611 Assembler::LSL,
11612 $src3$$constant & 0x1f);
11613 %}
11614
11615 ins_pipe(ialu_reg_reg_shift);
11616 %}
11617
11618 // This pattern is automatically generated from aarch64_ad.m4
11619 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11620 // val ^ (-1 ^ (val << shift)) ==> eon
11621 instruct XorL_reg_LShift_not_reg(iRegLNoSp dst,
11622 iRegL src1, iRegL src2,
11623 immI src3, immL_M1 src4) %{
11624 match(Set dst (XorL src4 (XorL(LShiftL src2 src3) src1)));
11625 ins_cost(1.9 * INSN_COST);
11626 format %{ "eon $dst, $src1, $src2, LSL $src3" %}
11627
11628 ins_encode %{
11629 __ eon(as_Register($dst$$reg),
11630 as_Register($src1$$reg),
11631 as_Register($src2$$reg),
11632 Assembler::LSL,
11633 $src3$$constant & 0x3f);
11634 %}
11635
11636 ins_pipe(ialu_reg_reg_shift);
11637 %}
11638
11639 // This pattern is automatically generated from aarch64_ad.m4
11640 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11641 // val | (-1 ^ (val >>> shift)) ==> ornw
11642 instruct OrI_reg_URShift_not_reg(iRegINoSp dst,
11643 iRegIorL2I src1, iRegIorL2I src2,
11644 immI src3, immI_M1 src4) %{
11645 match(Set dst (OrI src1 (XorI(URShiftI src2 src3) src4)));
11646 ins_cost(1.9 * INSN_COST);
11647 format %{ "ornw $dst, $src1, $src2, LSR $src3" %}
11648
11649 ins_encode %{
11650 __ ornw(as_Register($dst$$reg),
11651 as_Register($src1$$reg),
11652 as_Register($src2$$reg),
11653 Assembler::LSR,
11654 $src3$$constant & 0x1f);
11655 %}
11656
11657 ins_pipe(ialu_reg_reg_shift);
11658 %}
11659
11660 // This pattern is automatically generated from aarch64_ad.m4
11661 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11662 // val | (-1 ^ (val >>> shift)) ==> orn
11663 instruct OrL_reg_URShift_not_reg(iRegLNoSp dst,
11664 iRegL src1, iRegL src2,
11665 immI src3, immL_M1 src4) %{
11666 match(Set dst (OrL src1 (XorL(URShiftL src2 src3) src4)));
11667 ins_cost(1.9 * INSN_COST);
11668 format %{ "orn $dst, $src1, $src2, LSR $src3" %}
11669
11670 ins_encode %{
11671 __ orn(as_Register($dst$$reg),
11672 as_Register($src1$$reg),
11673 as_Register($src2$$reg),
11674 Assembler::LSR,
11675 $src3$$constant & 0x3f);
11676 %}
11677
11678 ins_pipe(ialu_reg_reg_shift);
11679 %}
11680
11681 // This pattern is automatically generated from aarch64_ad.m4
11682 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11683 // val | (-1 ^ (val >> shift)) ==> ornw
11684 instruct OrI_reg_RShift_not_reg(iRegINoSp dst,
11685 iRegIorL2I src1, iRegIorL2I src2,
11686 immI src3, immI_M1 src4) %{
11687 match(Set dst (OrI src1 (XorI(RShiftI src2 src3) src4)));
11688 ins_cost(1.9 * INSN_COST);
11689 format %{ "ornw $dst, $src1, $src2, ASR $src3" %}
11690
11691 ins_encode %{
11692 __ ornw(as_Register($dst$$reg),
11693 as_Register($src1$$reg),
11694 as_Register($src2$$reg),
11695 Assembler::ASR,
11696 $src3$$constant & 0x1f);
11697 %}
11698
11699 ins_pipe(ialu_reg_reg_shift);
11700 %}
11701
11702 // This pattern is automatically generated from aarch64_ad.m4
11703 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11704 // val | (-1 ^ (val >> shift)) ==> orn
11705 instruct OrL_reg_RShift_not_reg(iRegLNoSp dst,
11706 iRegL src1, iRegL src2,
11707 immI src3, immL_M1 src4) %{
11708 match(Set dst (OrL src1 (XorL(RShiftL src2 src3) src4)));
11709 ins_cost(1.9 * INSN_COST);
11710 format %{ "orn $dst, $src1, $src2, ASR $src3" %}
11711
11712 ins_encode %{
11713 __ orn(as_Register($dst$$reg),
11714 as_Register($src1$$reg),
11715 as_Register($src2$$reg),
11716 Assembler::ASR,
11717 $src3$$constant & 0x3f);
11718 %}
11719
11720 ins_pipe(ialu_reg_reg_shift);
11721 %}
11722
11723 // This pattern is automatically generated from aarch64_ad.m4
11724 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11725 // val | (-1 ^ (val ror shift)) ==> ornw
11726 instruct OrI_reg_RotateRight_not_reg(iRegINoSp dst,
11727 iRegIorL2I src1, iRegIorL2I src2,
11728 immI src3, immI_M1 src4) %{
11729 match(Set dst (OrI src1 (XorI(RotateRight src2 src3) src4)));
11730 ins_cost(1.9 * INSN_COST);
11731 format %{ "ornw $dst, $src1, $src2, ROR $src3" %}
11732
11733 ins_encode %{
11734 __ ornw(as_Register($dst$$reg),
11735 as_Register($src1$$reg),
11736 as_Register($src2$$reg),
11737 Assembler::ROR,
11738 $src3$$constant & 0x1f);
11739 %}
11740
11741 ins_pipe(ialu_reg_reg_shift);
11742 %}
11743
11744 // This pattern is automatically generated from aarch64_ad.m4
11745 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11746 // val | (-1 ^ (val ror shift)) ==> orn
11747 instruct OrL_reg_RotateRight_not_reg(iRegLNoSp dst,
11748 iRegL src1, iRegL src2,
11749 immI src3, immL_M1 src4) %{
11750 match(Set dst (OrL src1 (XorL(RotateRight src2 src3) src4)));
11751 ins_cost(1.9 * INSN_COST);
11752 format %{ "orn $dst, $src1, $src2, ROR $src3" %}
11753
11754 ins_encode %{
11755 __ orn(as_Register($dst$$reg),
11756 as_Register($src1$$reg),
11757 as_Register($src2$$reg),
11758 Assembler::ROR,
11759 $src3$$constant & 0x3f);
11760 %}
11761
11762 ins_pipe(ialu_reg_reg_shift);
11763 %}
11764
11765 // This pattern is automatically generated from aarch64_ad.m4
11766 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11767 // val | (-1 ^ (val << shift)) ==> ornw
11768 instruct OrI_reg_LShift_not_reg(iRegINoSp dst,
11769 iRegIorL2I src1, iRegIorL2I src2,
11770 immI src3, immI_M1 src4) %{
11771 match(Set dst (OrI src1 (XorI(LShiftI src2 src3) src4)));
11772 ins_cost(1.9 * INSN_COST);
11773 format %{ "ornw $dst, $src1, $src2, LSL $src3" %}
11774
11775 ins_encode %{
11776 __ ornw(as_Register($dst$$reg),
11777 as_Register($src1$$reg),
11778 as_Register($src2$$reg),
11779 Assembler::LSL,
11780 $src3$$constant & 0x1f);
11781 %}
11782
11783 ins_pipe(ialu_reg_reg_shift);
11784 %}
11785
11786 // This pattern is automatically generated from aarch64_ad.m4
11787 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11788 // val | (-1 ^ (val << shift)) ==> orn
11789 instruct OrL_reg_LShift_not_reg(iRegLNoSp dst,
11790 iRegL src1, iRegL src2,
11791 immI src3, immL_M1 src4) %{
11792 match(Set dst (OrL src1 (XorL(LShiftL src2 src3) src4)));
11793 ins_cost(1.9 * INSN_COST);
11794 format %{ "orn $dst, $src1, $src2, LSL $src3" %}
11795
11796 ins_encode %{
11797 __ orn(as_Register($dst$$reg),
11798 as_Register($src1$$reg),
11799 as_Register($src2$$reg),
11800 Assembler::LSL,
11801 $src3$$constant & 0x3f);
11802 %}
11803
11804 ins_pipe(ialu_reg_reg_shift);
11805 %}
11806
11807 // This pattern is automatically generated from aarch64_ad.m4
11808 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11809 instruct AndI_reg_URShift_reg(iRegINoSp dst,
11810 iRegIorL2I src1, iRegIorL2I src2,
11811 immI src3) %{
11812 match(Set dst (AndI src1 (URShiftI src2 src3)));
11813
11814 ins_cost(1.9 * INSN_COST);
11815 format %{ "andw $dst, $src1, $src2, LSR $src3" %}
11816
11817 ins_encode %{
11818 __ andw(as_Register($dst$$reg),
11819 as_Register($src1$$reg),
11820 as_Register($src2$$reg),
11821 Assembler::LSR,
11822 $src3$$constant & 0x1f);
11823 %}
11824
11825 ins_pipe(ialu_reg_reg_shift);
11826 %}
11827
11828 // This pattern is automatically generated from aarch64_ad.m4
11829 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11830 instruct AndL_reg_URShift_reg(iRegLNoSp dst,
11831 iRegL src1, iRegL src2,
11832 immI src3) %{
11833 match(Set dst (AndL src1 (URShiftL src2 src3)));
11834
11835 ins_cost(1.9 * INSN_COST);
11836 format %{ "andr $dst, $src1, $src2, LSR $src3" %}
11837
11838 ins_encode %{
11839 __ andr(as_Register($dst$$reg),
11840 as_Register($src1$$reg),
11841 as_Register($src2$$reg),
11842 Assembler::LSR,
11843 $src3$$constant & 0x3f);
11844 %}
11845
11846 ins_pipe(ialu_reg_reg_shift);
11847 %}
11848
11849 // This pattern is automatically generated from aarch64_ad.m4
11850 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11851 instruct AndI_reg_RShift_reg(iRegINoSp dst,
11852 iRegIorL2I src1, iRegIorL2I src2,
11853 immI src3) %{
11854 match(Set dst (AndI src1 (RShiftI src2 src3)));
11855
11856 ins_cost(1.9 * INSN_COST);
11857 format %{ "andw $dst, $src1, $src2, ASR $src3" %}
11858
11859 ins_encode %{
11860 __ andw(as_Register($dst$$reg),
11861 as_Register($src1$$reg),
11862 as_Register($src2$$reg),
11863 Assembler::ASR,
11864 $src3$$constant & 0x1f);
11865 %}
11866
11867 ins_pipe(ialu_reg_reg_shift);
11868 %}
11869
11870 // This pattern is automatically generated from aarch64_ad.m4
11871 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11872 instruct AndL_reg_RShift_reg(iRegLNoSp dst,
11873 iRegL src1, iRegL src2,
11874 immI src3) %{
11875 match(Set dst (AndL src1 (RShiftL src2 src3)));
11876
11877 ins_cost(1.9 * INSN_COST);
11878 format %{ "andr $dst, $src1, $src2, ASR $src3" %}
11879
11880 ins_encode %{
11881 __ andr(as_Register($dst$$reg),
11882 as_Register($src1$$reg),
11883 as_Register($src2$$reg),
11884 Assembler::ASR,
11885 $src3$$constant & 0x3f);
11886 %}
11887
11888 ins_pipe(ialu_reg_reg_shift);
11889 %}
11890
11891 // This pattern is automatically generated from aarch64_ad.m4
11892 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11893 instruct AndI_reg_LShift_reg(iRegINoSp dst,
11894 iRegIorL2I src1, iRegIorL2I src2,
11895 immI src3) %{
11896 match(Set dst (AndI src1 (LShiftI src2 src3)));
11897
11898 ins_cost(1.9 * INSN_COST);
11899 format %{ "andw $dst, $src1, $src2, LSL $src3" %}
11900
11901 ins_encode %{
11902 __ andw(as_Register($dst$$reg),
11903 as_Register($src1$$reg),
11904 as_Register($src2$$reg),
11905 Assembler::LSL,
11906 $src3$$constant & 0x1f);
11907 %}
11908
11909 ins_pipe(ialu_reg_reg_shift);
11910 %}
11911
11912 // This pattern is automatically generated from aarch64_ad.m4
11913 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11914 instruct AndL_reg_LShift_reg(iRegLNoSp dst,
11915 iRegL src1, iRegL src2,
11916 immI src3) %{
11917 match(Set dst (AndL src1 (LShiftL src2 src3)));
11918
11919 ins_cost(1.9 * INSN_COST);
11920 format %{ "andr $dst, $src1, $src2, LSL $src3" %}
11921
11922 ins_encode %{
11923 __ andr(as_Register($dst$$reg),
11924 as_Register($src1$$reg),
11925 as_Register($src2$$reg),
11926 Assembler::LSL,
11927 $src3$$constant & 0x3f);
11928 %}
11929
11930 ins_pipe(ialu_reg_reg_shift);
11931 %}
11932
11933 // This pattern is automatically generated from aarch64_ad.m4
11934 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11935 instruct AndI_reg_RotateRight_reg(iRegINoSp dst,
11936 iRegIorL2I src1, iRegIorL2I src2,
11937 immI src3) %{
11938 match(Set dst (AndI src1 (RotateRight src2 src3)));
11939
11940 ins_cost(1.9 * INSN_COST);
11941 format %{ "andw $dst, $src1, $src2, ROR $src3" %}
11942
11943 ins_encode %{
11944 __ andw(as_Register($dst$$reg),
11945 as_Register($src1$$reg),
11946 as_Register($src2$$reg),
11947 Assembler::ROR,
11948 $src3$$constant & 0x1f);
11949 %}
11950
11951 ins_pipe(ialu_reg_reg_shift);
11952 %}
11953
11954 // This pattern is automatically generated from aarch64_ad.m4
11955 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11956 instruct AndL_reg_RotateRight_reg(iRegLNoSp dst,
11957 iRegL src1, iRegL src2,
11958 immI src3) %{
11959 match(Set dst (AndL src1 (RotateRight src2 src3)));
11960
11961 ins_cost(1.9 * INSN_COST);
11962 format %{ "andr $dst, $src1, $src2, ROR $src3" %}
11963
11964 ins_encode %{
11965 __ andr(as_Register($dst$$reg),
11966 as_Register($src1$$reg),
11967 as_Register($src2$$reg),
11968 Assembler::ROR,
11969 $src3$$constant & 0x3f);
11970 %}
11971
11972 ins_pipe(ialu_reg_reg_shift);
11973 %}
11974
11975 // This pattern is automatically generated from aarch64_ad.m4
11976 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11977 instruct XorI_reg_URShift_reg(iRegINoSp dst,
11978 iRegIorL2I src1, iRegIorL2I src2,
11979 immI src3) %{
11980 match(Set dst (XorI src1 (URShiftI src2 src3)));
11981
11982 ins_cost(1.9 * INSN_COST);
11983 format %{ "eorw $dst, $src1, $src2, LSR $src3" %}
11984
11985 ins_encode %{
11986 __ eorw(as_Register($dst$$reg),
11987 as_Register($src1$$reg),
11988 as_Register($src2$$reg),
11989 Assembler::LSR,
11990 $src3$$constant & 0x1f);
11991 %}
11992
11993 ins_pipe(ialu_reg_reg_shift);
11994 %}
11995
11996 // This pattern is automatically generated from aarch64_ad.m4
11997 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11998 instruct XorL_reg_URShift_reg(iRegLNoSp dst,
11999 iRegL src1, iRegL src2,
12000 immI src3) %{
12001 match(Set dst (XorL src1 (URShiftL src2 src3)));
12002
12003 ins_cost(1.9 * INSN_COST);
12004 format %{ "eor $dst, $src1, $src2, LSR $src3" %}
12005
12006 ins_encode %{
12007 __ eor(as_Register($dst$$reg),
12008 as_Register($src1$$reg),
12009 as_Register($src2$$reg),
12010 Assembler::LSR,
12011 $src3$$constant & 0x3f);
12012 %}
12013
12014 ins_pipe(ialu_reg_reg_shift);
12015 %}
12016
12017 // This pattern is automatically generated from aarch64_ad.m4
12018 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12019 instruct XorI_reg_RShift_reg(iRegINoSp dst,
12020 iRegIorL2I src1, iRegIorL2I src2,
12021 immI src3) %{
12022 match(Set dst (XorI src1 (RShiftI src2 src3)));
12023
12024 ins_cost(1.9 * INSN_COST);
12025 format %{ "eorw $dst, $src1, $src2, ASR $src3" %}
12026
12027 ins_encode %{
12028 __ eorw(as_Register($dst$$reg),
12029 as_Register($src1$$reg),
12030 as_Register($src2$$reg),
12031 Assembler::ASR,
12032 $src3$$constant & 0x1f);
12033 %}
12034
12035 ins_pipe(ialu_reg_reg_shift);
12036 %}
12037
12038 // This pattern is automatically generated from aarch64_ad.m4
12039 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12040 instruct XorL_reg_RShift_reg(iRegLNoSp dst,
12041 iRegL src1, iRegL src2,
12042 immI src3) %{
12043 match(Set dst (XorL src1 (RShiftL src2 src3)));
12044
12045 ins_cost(1.9 * INSN_COST);
12046 format %{ "eor $dst, $src1, $src2, ASR $src3" %}
12047
12048 ins_encode %{
12049 __ eor(as_Register($dst$$reg),
12050 as_Register($src1$$reg),
12051 as_Register($src2$$reg),
12052 Assembler::ASR,
12053 $src3$$constant & 0x3f);
12054 %}
12055
12056 ins_pipe(ialu_reg_reg_shift);
12057 %}
12058
12059 // This pattern is automatically generated from aarch64_ad.m4
12060 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12061 instruct XorI_reg_LShift_reg(iRegINoSp dst,
12062 iRegIorL2I src1, iRegIorL2I src2,
12063 immI src3) %{
12064 match(Set dst (XorI src1 (LShiftI src2 src3)));
12065
12066 ins_cost(1.9 * INSN_COST);
12067 format %{ "eorw $dst, $src1, $src2, LSL $src3" %}
12068
12069 ins_encode %{
12070 __ eorw(as_Register($dst$$reg),
12071 as_Register($src1$$reg),
12072 as_Register($src2$$reg),
12073 Assembler::LSL,
12074 $src3$$constant & 0x1f);
12075 %}
12076
12077 ins_pipe(ialu_reg_reg_shift);
12078 %}
12079
12080 // This pattern is automatically generated from aarch64_ad.m4
12081 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12082 instruct XorL_reg_LShift_reg(iRegLNoSp dst,
12083 iRegL src1, iRegL src2,
12084 immI src3) %{
12085 match(Set dst (XorL src1 (LShiftL src2 src3)));
12086
12087 ins_cost(1.9 * INSN_COST);
12088 format %{ "eor $dst, $src1, $src2, LSL $src3" %}
12089
12090 ins_encode %{
12091 __ eor(as_Register($dst$$reg),
12092 as_Register($src1$$reg),
12093 as_Register($src2$$reg),
12094 Assembler::LSL,
12095 $src3$$constant & 0x3f);
12096 %}
12097
12098 ins_pipe(ialu_reg_reg_shift);
12099 %}
12100
12101 // This pattern is automatically generated from aarch64_ad.m4
12102 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12103 instruct XorI_reg_RotateRight_reg(iRegINoSp dst,
12104 iRegIorL2I src1, iRegIorL2I src2,
12105 immI src3) %{
12106 match(Set dst (XorI src1 (RotateRight src2 src3)));
12107
12108 ins_cost(1.9 * INSN_COST);
12109 format %{ "eorw $dst, $src1, $src2, ROR $src3" %}
12110
12111 ins_encode %{
12112 __ eorw(as_Register($dst$$reg),
12113 as_Register($src1$$reg),
12114 as_Register($src2$$reg),
12115 Assembler::ROR,
12116 $src3$$constant & 0x1f);
12117 %}
12118
12119 ins_pipe(ialu_reg_reg_shift);
12120 %}
12121
12122 // This pattern is automatically generated from aarch64_ad.m4
12123 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12124 instruct XorL_reg_RotateRight_reg(iRegLNoSp dst,
12125 iRegL src1, iRegL src2,
12126 immI src3) %{
12127 match(Set dst (XorL src1 (RotateRight src2 src3)));
12128
12129 ins_cost(1.9 * INSN_COST);
12130 format %{ "eor $dst, $src1, $src2, ROR $src3" %}
12131
12132 ins_encode %{
12133 __ eor(as_Register($dst$$reg),
12134 as_Register($src1$$reg),
12135 as_Register($src2$$reg),
12136 Assembler::ROR,
12137 $src3$$constant & 0x3f);
12138 %}
12139
12140 ins_pipe(ialu_reg_reg_shift);
12141 %}
12142
12143 // This pattern is automatically generated from aarch64_ad.m4
12144 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12145 instruct OrI_reg_URShift_reg(iRegINoSp dst,
12146 iRegIorL2I src1, iRegIorL2I src2,
12147 immI src3) %{
12148 match(Set dst (OrI src1 (URShiftI src2 src3)));
12149
12150 ins_cost(1.9 * INSN_COST);
12151 format %{ "orrw $dst, $src1, $src2, LSR $src3" %}
12152
12153 ins_encode %{
12154 __ orrw(as_Register($dst$$reg),
12155 as_Register($src1$$reg),
12156 as_Register($src2$$reg),
12157 Assembler::LSR,
12158 $src3$$constant & 0x1f);
12159 %}
12160
12161 ins_pipe(ialu_reg_reg_shift);
12162 %}
12163
12164 // This pattern is automatically generated from aarch64_ad.m4
12165 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12166 instruct OrL_reg_URShift_reg(iRegLNoSp dst,
12167 iRegL src1, iRegL src2,
12168 immI src3) %{
12169 match(Set dst (OrL src1 (URShiftL src2 src3)));
12170
12171 ins_cost(1.9 * INSN_COST);
12172 format %{ "orr $dst, $src1, $src2, LSR $src3" %}
12173
12174 ins_encode %{
12175 __ orr(as_Register($dst$$reg),
12176 as_Register($src1$$reg),
12177 as_Register($src2$$reg),
12178 Assembler::LSR,
12179 $src3$$constant & 0x3f);
12180 %}
12181
12182 ins_pipe(ialu_reg_reg_shift);
12183 %}
12184
12185 // This pattern is automatically generated from aarch64_ad.m4
12186 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12187 instruct OrI_reg_RShift_reg(iRegINoSp dst,
12188 iRegIorL2I src1, iRegIorL2I src2,
12189 immI src3) %{
12190 match(Set dst (OrI src1 (RShiftI src2 src3)));
12191
12192 ins_cost(1.9 * INSN_COST);
12193 format %{ "orrw $dst, $src1, $src2, ASR $src3" %}
12194
12195 ins_encode %{
12196 __ orrw(as_Register($dst$$reg),
12197 as_Register($src1$$reg),
12198 as_Register($src2$$reg),
12199 Assembler::ASR,
12200 $src3$$constant & 0x1f);
12201 %}
12202
12203 ins_pipe(ialu_reg_reg_shift);
12204 %}
12205
12206 // This pattern is automatically generated from aarch64_ad.m4
12207 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12208 instruct OrL_reg_RShift_reg(iRegLNoSp dst,
12209 iRegL src1, iRegL src2,
12210 immI src3) %{
12211 match(Set dst (OrL src1 (RShiftL src2 src3)));
12212
12213 ins_cost(1.9 * INSN_COST);
12214 format %{ "orr $dst, $src1, $src2, ASR $src3" %}
12215
12216 ins_encode %{
12217 __ orr(as_Register($dst$$reg),
12218 as_Register($src1$$reg),
12219 as_Register($src2$$reg),
12220 Assembler::ASR,
12221 $src3$$constant & 0x3f);
12222 %}
12223
12224 ins_pipe(ialu_reg_reg_shift);
12225 %}
12226
12227 // This pattern is automatically generated from aarch64_ad.m4
12228 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12229 instruct OrI_reg_LShift_reg(iRegINoSp dst,
12230 iRegIorL2I src1, iRegIorL2I src2,
12231 immI src3) %{
12232 match(Set dst (OrI src1 (LShiftI src2 src3)));
12233
12234 ins_cost(1.9 * INSN_COST);
12235 format %{ "orrw $dst, $src1, $src2, LSL $src3" %}
12236
12237 ins_encode %{
12238 __ orrw(as_Register($dst$$reg),
12239 as_Register($src1$$reg),
12240 as_Register($src2$$reg),
12241 Assembler::LSL,
12242 $src3$$constant & 0x1f);
12243 %}
12244
12245 ins_pipe(ialu_reg_reg_shift);
12246 %}
12247
12248 // This pattern is automatically generated from aarch64_ad.m4
12249 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12250 instruct OrL_reg_LShift_reg(iRegLNoSp dst,
12251 iRegL src1, iRegL src2,
12252 immI src3) %{
12253 match(Set dst (OrL src1 (LShiftL src2 src3)));
12254
12255 ins_cost(1.9 * INSN_COST);
12256 format %{ "orr $dst, $src1, $src2, LSL $src3" %}
12257
12258 ins_encode %{
12259 __ orr(as_Register($dst$$reg),
12260 as_Register($src1$$reg),
12261 as_Register($src2$$reg),
12262 Assembler::LSL,
12263 $src3$$constant & 0x3f);
12264 %}
12265
12266 ins_pipe(ialu_reg_reg_shift);
12267 %}
12268
12269 // This pattern is automatically generated from aarch64_ad.m4
12270 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12271 instruct OrI_reg_RotateRight_reg(iRegINoSp dst,
12272 iRegIorL2I src1, iRegIorL2I src2,
12273 immI src3) %{
12274 match(Set dst (OrI src1 (RotateRight src2 src3)));
12275
12276 ins_cost(1.9 * INSN_COST);
12277 format %{ "orrw $dst, $src1, $src2, ROR $src3" %}
12278
12279 ins_encode %{
12280 __ orrw(as_Register($dst$$reg),
12281 as_Register($src1$$reg),
12282 as_Register($src2$$reg),
12283 Assembler::ROR,
12284 $src3$$constant & 0x1f);
12285 %}
12286
12287 ins_pipe(ialu_reg_reg_shift);
12288 %}
12289
12290 // This pattern is automatically generated from aarch64_ad.m4
12291 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12292 instruct OrL_reg_RotateRight_reg(iRegLNoSp dst,
12293 iRegL src1, iRegL src2,
12294 immI src3) %{
12295 match(Set dst (OrL src1 (RotateRight src2 src3)));
12296
12297 ins_cost(1.9 * INSN_COST);
12298 format %{ "orr $dst, $src1, $src2, ROR $src3" %}
12299
12300 ins_encode %{
12301 __ orr(as_Register($dst$$reg),
12302 as_Register($src1$$reg),
12303 as_Register($src2$$reg),
12304 Assembler::ROR,
12305 $src3$$constant & 0x3f);
12306 %}
12307
12308 ins_pipe(ialu_reg_reg_shift);
12309 %}
12310
12311 // This pattern is automatically generated from aarch64_ad.m4
12312 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12313 instruct AddI_reg_URShift_reg(iRegINoSp dst,
12314 iRegIorL2I src1, iRegIorL2I src2,
12315 immI src3) %{
12316 match(Set dst (AddI src1 (URShiftI src2 src3)));
12317
12318 ins_cost(1.9 * INSN_COST);
12319 format %{ "addw $dst, $src1, $src2, LSR $src3" %}
12320
12321 ins_encode %{
12322 __ addw(as_Register($dst$$reg),
12323 as_Register($src1$$reg),
12324 as_Register($src2$$reg),
12325 Assembler::LSR,
12326 $src3$$constant & 0x1f);
12327 %}
12328
12329 ins_pipe(ialu_reg_reg_shift);
12330 %}
12331
12332 // This pattern is automatically generated from aarch64_ad.m4
12333 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12334 instruct AddL_reg_URShift_reg(iRegLNoSp dst,
12335 iRegL src1, iRegL src2,
12336 immI src3) %{
12337 match(Set dst (AddL src1 (URShiftL src2 src3)));
12338
12339 ins_cost(1.9 * INSN_COST);
12340 format %{ "add $dst, $src1, $src2, LSR $src3" %}
12341
12342 ins_encode %{
12343 __ add(as_Register($dst$$reg),
12344 as_Register($src1$$reg),
12345 as_Register($src2$$reg),
12346 Assembler::LSR,
12347 $src3$$constant & 0x3f);
12348 %}
12349
12350 ins_pipe(ialu_reg_reg_shift);
12351 %}
12352
12353 // This pattern is automatically generated from aarch64_ad.m4
12354 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12355 instruct AddI_reg_RShift_reg(iRegINoSp dst,
12356 iRegIorL2I src1, iRegIorL2I src2,
12357 immI src3) %{
12358 match(Set dst (AddI src1 (RShiftI src2 src3)));
12359
12360 ins_cost(1.9 * INSN_COST);
12361 format %{ "addw $dst, $src1, $src2, ASR $src3" %}
12362
12363 ins_encode %{
12364 __ addw(as_Register($dst$$reg),
12365 as_Register($src1$$reg),
12366 as_Register($src2$$reg),
12367 Assembler::ASR,
12368 $src3$$constant & 0x1f);
12369 %}
12370
12371 ins_pipe(ialu_reg_reg_shift);
12372 %}
12373
12374 // This pattern is automatically generated from aarch64_ad.m4
12375 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12376 instruct AddL_reg_RShift_reg(iRegLNoSp dst,
12377 iRegL src1, iRegL src2,
12378 immI src3) %{
12379 match(Set dst (AddL src1 (RShiftL src2 src3)));
12380
12381 ins_cost(1.9 * INSN_COST);
12382 format %{ "add $dst, $src1, $src2, ASR $src3" %}
12383
12384 ins_encode %{
12385 __ add(as_Register($dst$$reg),
12386 as_Register($src1$$reg),
12387 as_Register($src2$$reg),
12388 Assembler::ASR,
12389 $src3$$constant & 0x3f);
12390 %}
12391
12392 ins_pipe(ialu_reg_reg_shift);
12393 %}
12394
12395 // This pattern is automatically generated from aarch64_ad.m4
12396 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12397 instruct AddI_reg_LShift_reg(iRegINoSp dst,
12398 iRegIorL2I src1, iRegIorL2I src2,
12399 immI src3) %{
12400 match(Set dst (AddI src1 (LShiftI src2 src3)));
12401
12402 ins_cost(1.9 * INSN_COST);
12403 format %{ "addw $dst, $src1, $src2, LSL $src3" %}
12404
12405 ins_encode %{
12406 __ addw(as_Register($dst$$reg),
12407 as_Register($src1$$reg),
12408 as_Register($src2$$reg),
12409 Assembler::LSL,
12410 $src3$$constant & 0x1f);
12411 %}
12412
12413 ins_pipe(ialu_reg_reg_shift);
12414 %}
12415
12416 // This pattern is automatically generated from aarch64_ad.m4
12417 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12418 instruct AddL_reg_LShift_reg(iRegLNoSp dst,
12419 iRegL src1, iRegL src2,
12420 immI src3) %{
12421 match(Set dst (AddL src1 (LShiftL src2 src3)));
12422
12423 ins_cost(1.9 * INSN_COST);
12424 format %{ "add $dst, $src1, $src2, LSL $src3" %}
12425
12426 ins_encode %{
12427 __ add(as_Register($dst$$reg),
12428 as_Register($src1$$reg),
12429 as_Register($src2$$reg),
12430 Assembler::LSL,
12431 $src3$$constant & 0x3f);
12432 %}
12433
12434 ins_pipe(ialu_reg_reg_shift);
12435 %}
12436
12437 // This pattern is automatically generated from aarch64_ad.m4
12438 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12439 instruct SubI_reg_URShift_reg(iRegINoSp dst,
12440 iRegIorL2I src1, iRegIorL2I src2,
12441 immI src3) %{
12442 match(Set dst (SubI src1 (URShiftI src2 src3)));
12443
12444 ins_cost(1.9 * INSN_COST);
12445 format %{ "subw $dst, $src1, $src2, LSR $src3" %}
12446
12447 ins_encode %{
12448 __ subw(as_Register($dst$$reg),
12449 as_Register($src1$$reg),
12450 as_Register($src2$$reg),
12451 Assembler::LSR,
12452 $src3$$constant & 0x1f);
12453 %}
12454
12455 ins_pipe(ialu_reg_reg_shift);
12456 %}
12457
12458 // This pattern is automatically generated from aarch64_ad.m4
12459 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12460 instruct SubL_reg_URShift_reg(iRegLNoSp dst,
12461 iRegL src1, iRegL src2,
12462 immI src3) %{
12463 match(Set dst (SubL src1 (URShiftL src2 src3)));
12464
12465 ins_cost(1.9 * INSN_COST);
12466 format %{ "sub $dst, $src1, $src2, LSR $src3" %}
12467
12468 ins_encode %{
12469 __ sub(as_Register($dst$$reg),
12470 as_Register($src1$$reg),
12471 as_Register($src2$$reg),
12472 Assembler::LSR,
12473 $src3$$constant & 0x3f);
12474 %}
12475
12476 ins_pipe(ialu_reg_reg_shift);
12477 %}
12478
12479 // This pattern is automatically generated from aarch64_ad.m4
12480 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12481 instruct SubI_reg_RShift_reg(iRegINoSp dst,
12482 iRegIorL2I src1, iRegIorL2I src2,
12483 immI src3) %{
12484 match(Set dst (SubI src1 (RShiftI src2 src3)));
12485
12486 ins_cost(1.9 * INSN_COST);
12487 format %{ "subw $dst, $src1, $src2, ASR $src3" %}
12488
12489 ins_encode %{
12490 __ subw(as_Register($dst$$reg),
12491 as_Register($src1$$reg),
12492 as_Register($src2$$reg),
12493 Assembler::ASR,
12494 $src3$$constant & 0x1f);
12495 %}
12496
12497 ins_pipe(ialu_reg_reg_shift);
12498 %}
12499
12500 // This pattern is automatically generated from aarch64_ad.m4
12501 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12502 instruct SubL_reg_RShift_reg(iRegLNoSp dst,
12503 iRegL src1, iRegL src2,
12504 immI src3) %{
12505 match(Set dst (SubL src1 (RShiftL src2 src3)));
12506
12507 ins_cost(1.9 * INSN_COST);
12508 format %{ "sub $dst, $src1, $src2, ASR $src3" %}
12509
12510 ins_encode %{
12511 __ sub(as_Register($dst$$reg),
12512 as_Register($src1$$reg),
12513 as_Register($src2$$reg),
12514 Assembler::ASR,
12515 $src3$$constant & 0x3f);
12516 %}
12517
12518 ins_pipe(ialu_reg_reg_shift);
12519 %}
12520
12521 // This pattern is automatically generated from aarch64_ad.m4
12522 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12523 instruct SubI_reg_LShift_reg(iRegINoSp dst,
12524 iRegIorL2I src1, iRegIorL2I src2,
12525 immI src3) %{
12526 match(Set dst (SubI src1 (LShiftI src2 src3)));
12527
12528 ins_cost(1.9 * INSN_COST);
12529 format %{ "subw $dst, $src1, $src2, LSL $src3" %}
12530
12531 ins_encode %{
12532 __ subw(as_Register($dst$$reg),
12533 as_Register($src1$$reg),
12534 as_Register($src2$$reg),
12535 Assembler::LSL,
12536 $src3$$constant & 0x1f);
12537 %}
12538
12539 ins_pipe(ialu_reg_reg_shift);
12540 %}
12541
12542 // This pattern is automatically generated from aarch64_ad.m4
12543 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12544 instruct SubL_reg_LShift_reg(iRegLNoSp dst,
12545 iRegL src1, iRegL src2,
12546 immI src3) %{
12547 match(Set dst (SubL src1 (LShiftL src2 src3)));
12548
12549 ins_cost(1.9 * INSN_COST);
12550 format %{ "sub $dst, $src1, $src2, LSL $src3" %}
12551
12552 ins_encode %{
12553 __ sub(as_Register($dst$$reg),
12554 as_Register($src1$$reg),
12555 as_Register($src2$$reg),
12556 Assembler::LSL,
12557 $src3$$constant & 0x3f);
12558 %}
12559
12560 ins_pipe(ialu_reg_reg_shift);
12561 %}
12562
12563 // This pattern is automatically generated from aarch64_ad.m4
12564 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12565
12566 // Shift Left followed by Shift Right.
12567 // This idiom is used by the compiler for the i2b bytecode etc.
12568 instruct sbfmL(iRegLNoSp dst, iRegL src, immI lshift_count, immI rshift_count)
12569 %{
12570 match(Set dst (RShiftL (LShiftL src lshift_count) rshift_count));
12571 ins_cost(INSN_COST * 2);
12572 format %{ "sbfm $dst, $src, $rshift_count - $lshift_count, #63 - $lshift_count" %}
12573 ins_encode %{
12574 int lshift = $lshift_count$$constant & 63;
12575 int rshift = $rshift_count$$constant & 63;
12576 int s = 63 - lshift;
12577 int r = (rshift - lshift) & 63;
12578 __ sbfm(as_Register($dst$$reg),
12579 as_Register($src$$reg),
12580 r, s);
12581 %}
12582
12583 ins_pipe(ialu_reg_shift);
12584 %}
12585
12586 // This pattern is automatically generated from aarch64_ad.m4
12587 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12588
12589 // Shift Left followed by Shift Right.
12590 // This idiom is used by the compiler for the i2b bytecode etc.
12591 instruct sbfmwI(iRegINoSp dst, iRegIorL2I src, immI lshift_count, immI rshift_count)
12592 %{
12593 match(Set dst (RShiftI (LShiftI src lshift_count) rshift_count));
12594 ins_cost(INSN_COST * 2);
12595 format %{ "sbfmw $dst, $src, $rshift_count - $lshift_count, #31 - $lshift_count" %}
12596 ins_encode %{
12597 int lshift = $lshift_count$$constant & 31;
12598 int rshift = $rshift_count$$constant & 31;
12599 int s = 31 - lshift;
12600 int r = (rshift - lshift) & 31;
12601 __ sbfmw(as_Register($dst$$reg),
12602 as_Register($src$$reg),
12603 r, s);
12604 %}
12605
12606 ins_pipe(ialu_reg_shift);
12607 %}
12608
12609 // This pattern is automatically generated from aarch64_ad.m4
12610 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12611
12612 // Shift Left followed by Shift Right.
12613 // This idiom is used by the compiler for the i2b bytecode etc.
12614 instruct ubfmL(iRegLNoSp dst, iRegL src, immI lshift_count, immI rshift_count)
12615 %{
12616 match(Set dst (URShiftL (LShiftL src lshift_count) rshift_count));
12617 ins_cost(INSN_COST * 2);
12618 format %{ "ubfm $dst, $src, $rshift_count - $lshift_count, #63 - $lshift_count" %}
12619 ins_encode %{
12620 int lshift = $lshift_count$$constant & 63;
12621 int rshift = $rshift_count$$constant & 63;
12622 int s = 63 - lshift;
12623 int r = (rshift - lshift) & 63;
12624 __ ubfm(as_Register($dst$$reg),
12625 as_Register($src$$reg),
12626 r, s);
12627 %}
12628
12629 ins_pipe(ialu_reg_shift);
12630 %}
12631
12632 // This pattern is automatically generated from aarch64_ad.m4
12633 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12634
12635 // Shift Left followed by Shift Right.
12636 // This idiom is used by the compiler for the i2b bytecode etc.
12637 instruct ubfmwI(iRegINoSp dst, iRegIorL2I src, immI lshift_count, immI rshift_count)
12638 %{
12639 match(Set dst (URShiftI (LShiftI src lshift_count) rshift_count));
12640 ins_cost(INSN_COST * 2);
12641 format %{ "ubfmw $dst, $src, $rshift_count - $lshift_count, #31 - $lshift_count" %}
12642 ins_encode %{
12643 int lshift = $lshift_count$$constant & 31;
12644 int rshift = $rshift_count$$constant & 31;
12645 int s = 31 - lshift;
12646 int r = (rshift - lshift) & 31;
12647 __ ubfmw(as_Register($dst$$reg),
12648 as_Register($src$$reg),
12649 r, s);
12650 %}
12651
12652 ins_pipe(ialu_reg_shift);
12653 %}
12654
12655 // Bitfield extract with shift & mask
12656
12657 // This pattern is automatically generated from aarch64_ad.m4
12658 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12659 instruct ubfxwI(iRegINoSp dst, iRegIorL2I src, immI rshift, immI_bitmask mask)
12660 %{
12661 match(Set dst (AndI (URShiftI src rshift) mask));
12662 // Make sure we are not going to exceed what ubfxw can do.
12663 predicate((exact_log2(n->in(2)->get_int() + 1) + (n->in(1)->in(2)->get_int() & 31)) <= (31 + 1));
12664
12665 ins_cost(INSN_COST);
12666 format %{ "ubfxw $dst, $src, $rshift, $mask" %}
12667 ins_encode %{
12668 int rshift = $rshift$$constant & 31;
12669 intptr_t mask = $mask$$constant;
12670 int width = exact_log2(mask+1);
12671 __ ubfxw(as_Register($dst$$reg),
12672 as_Register($src$$reg), rshift, width);
12673 %}
12674 ins_pipe(ialu_reg_shift);
12675 %}
12676
12677 // This pattern is automatically generated from aarch64_ad.m4
12678 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12679 instruct ubfxL(iRegLNoSp dst, iRegL src, immI rshift, immL_bitmask mask)
12680 %{
12681 match(Set dst (AndL (URShiftL src rshift) mask));
12682 // Make sure we are not going to exceed what ubfx can do.
12683 predicate((exact_log2_long(n->in(2)->get_long() + 1) + (n->in(1)->in(2)->get_int() & 63)) <= (63 + 1));
12684
12685 ins_cost(INSN_COST);
12686 format %{ "ubfx $dst, $src, $rshift, $mask" %}
12687 ins_encode %{
12688 int rshift = $rshift$$constant & 63;
12689 intptr_t mask = $mask$$constant;
12690 int width = exact_log2_long(mask+1);
12691 __ ubfx(as_Register($dst$$reg),
12692 as_Register($src$$reg), rshift, width);
12693 %}
12694 ins_pipe(ialu_reg_shift);
12695 %}
12696
12697
12698 // This pattern is automatically generated from aarch64_ad.m4
12699 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12700
12701 // We can use ubfx when extending an And with a mask when we know mask
12702 // is positive. We know that because immI_bitmask guarantees it.
12703 instruct ubfxIConvI2L(iRegLNoSp dst, iRegIorL2I src, immI rshift, immI_bitmask mask)
12704 %{
12705 match(Set dst (ConvI2L (AndI (URShiftI src rshift) mask)));
12706 // Make sure we are not going to exceed what ubfxw can do.
12707 predicate((exact_log2(n->in(1)->in(2)->get_int() + 1) + (n->in(1)->in(1)->in(2)->get_int() & 31)) <= (31 + 1));
12708
12709 ins_cost(INSN_COST * 2);
12710 format %{ "ubfx $dst, $src, $rshift, $mask" %}
12711 ins_encode %{
12712 int rshift = $rshift$$constant & 31;
12713 intptr_t mask = $mask$$constant;
12714 int width = exact_log2(mask+1);
12715 __ ubfx(as_Register($dst$$reg),
12716 as_Register($src$$reg), rshift, width);
12717 %}
12718 ins_pipe(ialu_reg_shift);
12719 %}
12720
12721
12722 // This pattern is automatically generated from aarch64_ad.m4
12723 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12724
12725 // We can use ubfiz when masking by a positive number and then left shifting the result.
12726 // We know that the mask is positive because immI_bitmask guarantees it.
12727 instruct ubfizwI(iRegINoSp dst, iRegIorL2I src, immI lshift, immI_bitmask mask)
12728 %{
12729 match(Set dst (LShiftI (AndI src mask) lshift));
12730 predicate((exact_log2(n->in(1)->in(2)->get_int() + 1) + (n->in(2)->get_int() & 31)) <= (31 + 1));
12731
12732 ins_cost(INSN_COST);
12733 format %{ "ubfizw $dst, $src, $lshift, $mask" %}
12734 ins_encode %{
12735 int lshift = $lshift$$constant & 31;
12736 intptr_t mask = $mask$$constant;
12737 int width = exact_log2(mask+1);
12738 __ ubfizw(as_Register($dst$$reg),
12739 as_Register($src$$reg), lshift, width);
12740 %}
12741 ins_pipe(ialu_reg_shift);
12742 %}
12743
12744 // This pattern is automatically generated from aarch64_ad.m4
12745 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12746
12747 // We can use ubfiz when masking by a positive number and then left shifting the result.
12748 // We know that the mask is positive because immL_bitmask guarantees it.
12749 instruct ubfizL(iRegLNoSp dst, iRegL src, immI lshift, immL_bitmask mask)
12750 %{
12751 match(Set dst (LShiftL (AndL src mask) lshift));
12752 predicate((exact_log2_long(n->in(1)->in(2)->get_long() + 1) + (n->in(2)->get_int() & 63)) <= (63 + 1));
12753
12754 ins_cost(INSN_COST);
12755 format %{ "ubfiz $dst, $src, $lshift, $mask" %}
12756 ins_encode %{
12757 int lshift = $lshift$$constant & 63;
12758 intptr_t mask = $mask$$constant;
12759 int width = exact_log2_long(mask+1);
12760 __ ubfiz(as_Register($dst$$reg),
12761 as_Register($src$$reg), lshift, width);
12762 %}
12763 ins_pipe(ialu_reg_shift);
12764 %}
12765
12766 // This pattern is automatically generated from aarch64_ad.m4
12767 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12768
12769 // We can use ubfiz when masking by a positive number and then left shifting the result.
12770 // We know that the mask is positive because immI_bitmask guarantees it.
12771 instruct ubfizwIConvI2L(iRegLNoSp dst, iRegIorL2I src, immI lshift, immI_bitmask mask)
12772 %{
12773 match(Set dst (ConvI2L (LShiftI (AndI src mask) lshift)));
12774 predicate((exact_log2(n->in(1)->in(1)->in(2)->get_int() + 1) + (n->in(1)->in(2)->get_int() & 31)) <= 31);
12775
12776 ins_cost(INSN_COST);
12777 format %{ "ubfizw $dst, $src, $lshift, $mask" %}
12778 ins_encode %{
12779 int lshift = $lshift$$constant & 31;
12780 intptr_t mask = $mask$$constant;
12781 int width = exact_log2(mask+1);
12782 __ ubfizw(as_Register($dst$$reg),
12783 as_Register($src$$reg), lshift, width);
12784 %}
12785 ins_pipe(ialu_reg_shift);
12786 %}
12787
12788 // This pattern is automatically generated from aarch64_ad.m4
12789 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12790
12791 // We can use ubfiz when masking by a positive number and then left shifting the result.
12792 // We know that the mask is positive because immL_bitmask guarantees it.
12793 instruct ubfizLConvL2I(iRegINoSp dst, iRegL src, immI lshift, immL_positive_bitmaskI mask)
12794 %{
12795 match(Set dst (ConvL2I (LShiftL (AndL src mask) lshift)));
12796 predicate((exact_log2_long(n->in(1)->in(1)->in(2)->get_long() + 1) + (n->in(1)->in(2)->get_int() & 63)) <= 31);
12797
12798 ins_cost(INSN_COST);
12799 format %{ "ubfiz $dst, $src, $lshift, $mask" %}
12800 ins_encode %{
12801 int lshift = $lshift$$constant & 63;
12802 intptr_t mask = $mask$$constant;
12803 int width = exact_log2_long(mask+1);
12804 __ ubfiz(as_Register($dst$$reg),
12805 as_Register($src$$reg), lshift, width);
12806 %}
12807 ins_pipe(ialu_reg_shift);
12808 %}
12809
12810
12811 // This pattern is automatically generated from aarch64_ad.m4
12812 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12813
12814 // If there is a convert I to L block between and AndI and a LShiftL, we can also match ubfiz
12815 instruct ubfizIConvI2L(iRegLNoSp dst, iRegIorL2I src, immI lshift, immI_bitmask mask)
12816 %{
12817 match(Set dst (LShiftL (ConvI2L (AndI src mask)) lshift));
12818 predicate((exact_log2(n->in(1)->in(1)->in(2)->get_int() + 1) + (n->in(2)->get_int() & 63)) <= (63 + 1));
12819
12820 ins_cost(INSN_COST);
12821 format %{ "ubfiz $dst, $src, $lshift, $mask" %}
12822 ins_encode %{
12823 int lshift = $lshift$$constant & 63;
12824 intptr_t mask = $mask$$constant;
12825 int width = exact_log2(mask+1);
12826 __ ubfiz(as_Register($dst$$reg),
12827 as_Register($src$$reg), lshift, width);
12828 %}
12829 ins_pipe(ialu_reg_shift);
12830 %}
12831
12832 // This pattern is automatically generated from aarch64_ad.m4
12833 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12834
12835 // If there is a convert L to I block between and AndL and a LShiftI, we can also match ubfiz
12836 instruct ubfizLConvL2Ix(iRegINoSp dst, iRegL src, immI lshift, immL_positive_bitmaskI mask)
12837 %{
12838 match(Set dst (LShiftI (ConvL2I (AndL src mask)) lshift));
12839 predicate((exact_log2_long(n->in(1)->in(1)->in(2)->get_long() + 1) + (n->in(2)->get_int() & 31)) <= 31);
12840
12841 ins_cost(INSN_COST);
12842 format %{ "ubfiz $dst, $src, $lshift, $mask" %}
12843 ins_encode %{
12844 int lshift = $lshift$$constant & 31;
12845 intptr_t mask = $mask$$constant;
12846 int width = exact_log2(mask+1);
12847 __ ubfiz(as_Register($dst$$reg),
12848 as_Register($src$$reg), lshift, width);
12849 %}
12850 ins_pipe(ialu_reg_shift);
12851 %}
12852
12853 // This pattern is automatically generated from aarch64_ad.m4
12854 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12855
12856 // Can skip int2long conversions after AND with small bitmask
12857 instruct ubfizIConvI2LAndI(iRegLNoSp dst, iRegI src, immI_bitmask msk)
12858 %{
12859 match(Set dst (ConvI2L (AndI src msk)));
12860 ins_cost(INSN_COST);
12861 format %{ "ubfiz $dst, $src, 0, exact_log2($msk + 1) " %}
12862 ins_encode %{
12863 __ ubfiz(as_Register($dst$$reg), as_Register($src$$reg), 0, exact_log2($msk$$constant + 1));
12864 %}
12865 ins_pipe(ialu_reg_shift);
12866 %}
12867
12868
12869 // Rotations
12870 // This pattern is automatically generated from aarch64_ad.m4
12871 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12872 instruct extrOrL(iRegLNoSp dst, iRegL src1, iRegL src2, immI lshift, immI rshift, rFlagsReg cr)
12873 %{
12874 match(Set dst (OrL (LShiftL src1 lshift) (URShiftL src2 rshift)));
12875 predicate(0 == (((n->in(1)->in(2)->get_int() & 63) + (n->in(2)->in(2)->get_int() & 63)) & 63));
12876
12877 ins_cost(INSN_COST);
12878 format %{ "extr $dst, $src1, $src2, #$rshift" %}
12879
12880 ins_encode %{
12881 __ extr(as_Register($dst$$reg), as_Register($src1$$reg), as_Register($src2$$reg),
12882 $rshift$$constant & 63);
12883 %}
12884 ins_pipe(ialu_reg_reg_extr);
12885 %}
12886
12887
12888 // This pattern is automatically generated from aarch64_ad.m4
12889 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12890 instruct extrOrI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI lshift, immI rshift, rFlagsReg cr)
12891 %{
12892 match(Set dst (OrI (LShiftI src1 lshift) (URShiftI src2 rshift)));
12893 predicate(0 == (((n->in(1)->in(2)->get_int() & 31) + (n->in(2)->in(2)->get_int() & 31)) & 31));
12894
12895 ins_cost(INSN_COST);
12896 format %{ "extr $dst, $src1, $src2, #$rshift" %}
12897
12898 ins_encode %{
12899 __ extrw(as_Register($dst$$reg), as_Register($src1$$reg), as_Register($src2$$reg),
12900 $rshift$$constant & 31);
12901 %}
12902 ins_pipe(ialu_reg_reg_extr);
12903 %}
12904
12905
12906 // This pattern is automatically generated from aarch64_ad.m4
12907 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12908 instruct extrAddL(iRegLNoSp dst, iRegL src1, iRegL src2, immI lshift, immI rshift, rFlagsReg cr)
12909 %{
12910 match(Set dst (AddL (LShiftL src1 lshift) (URShiftL src2 rshift)));
12911 predicate(0 == (((n->in(1)->in(2)->get_int() & 63) + (n->in(2)->in(2)->get_int() & 63)) & 63));
12912
12913 ins_cost(INSN_COST);
12914 format %{ "extr $dst, $src1, $src2, #$rshift" %}
12915
12916 ins_encode %{
12917 __ extr(as_Register($dst$$reg), as_Register($src1$$reg), as_Register($src2$$reg),
12918 $rshift$$constant & 63);
12919 %}
12920 ins_pipe(ialu_reg_reg_extr);
12921 %}
12922
12923
12924 // This pattern is automatically generated from aarch64_ad.m4
12925 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12926 instruct extrAddI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI lshift, immI rshift, rFlagsReg cr)
12927 %{
12928 match(Set dst (AddI (LShiftI src1 lshift) (URShiftI src2 rshift)));
12929 predicate(0 == (((n->in(1)->in(2)->get_int() & 31) + (n->in(2)->in(2)->get_int() & 31)) & 31));
12930
12931 ins_cost(INSN_COST);
12932 format %{ "extr $dst, $src1, $src2, #$rshift" %}
12933
12934 ins_encode %{
12935 __ extrw(as_Register($dst$$reg), as_Register($src1$$reg), as_Register($src2$$reg),
12936 $rshift$$constant & 31);
12937 %}
12938 ins_pipe(ialu_reg_reg_extr);
12939 %}
12940
12941
12942 // This pattern is automatically generated from aarch64_ad.m4
12943 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12944 instruct rorI_imm(iRegINoSp dst, iRegI src, immI shift)
12945 %{
12946 match(Set dst (RotateRight src shift));
12947
12948 ins_cost(INSN_COST);
12949 format %{ "ror $dst, $src, $shift" %}
12950
12951 ins_encode %{
12952 __ extrw(as_Register($dst$$reg), as_Register($src$$reg), as_Register($src$$reg),
12953 $shift$$constant & 0x1f);
12954 %}
12955 ins_pipe(ialu_reg_reg_vshift);
12956 %}
12957
12958 // This pattern is automatically generated from aarch64_ad.m4
12959 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12960 instruct rorL_imm(iRegLNoSp dst, iRegL src, immI shift)
12961 %{
12962 match(Set dst (RotateRight src shift));
12963
12964 ins_cost(INSN_COST);
12965 format %{ "ror $dst, $src, $shift" %}
12966
12967 ins_encode %{
12968 __ extr(as_Register($dst$$reg), as_Register($src$$reg), as_Register($src$$reg),
12969 $shift$$constant & 0x3f);
12970 %}
12971 ins_pipe(ialu_reg_reg_vshift);
12972 %}
12973
12974 // This pattern is automatically generated from aarch64_ad.m4
12975 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12976 instruct rorI_reg(iRegINoSp dst, iRegI src, iRegI shift)
12977 %{
12978 match(Set dst (RotateRight src shift));
12979
12980 ins_cost(INSN_COST);
12981 format %{ "ror $dst, $src, $shift" %}
12982
12983 ins_encode %{
12984 __ rorvw(as_Register($dst$$reg), as_Register($src$$reg), as_Register($shift$$reg));
12985 %}
12986 ins_pipe(ialu_reg_reg_vshift);
12987 %}
12988
12989 // This pattern is automatically generated from aarch64_ad.m4
12990 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12991 instruct rorL_reg(iRegLNoSp dst, iRegL src, iRegI shift)
12992 %{
12993 match(Set dst (RotateRight src shift));
12994
12995 ins_cost(INSN_COST);
12996 format %{ "ror $dst, $src, $shift" %}
12997
12998 ins_encode %{
12999 __ rorv(as_Register($dst$$reg), as_Register($src$$reg), as_Register($shift$$reg));
13000 %}
13001 ins_pipe(ialu_reg_reg_vshift);
13002 %}
13003
13004 // This pattern is automatically generated from aarch64_ad.m4
13005 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13006 instruct rolI_reg(iRegINoSp dst, iRegI src, iRegI shift)
13007 %{
13008 match(Set dst (RotateLeft src shift));
13009
13010 ins_cost(INSN_COST);
13011 format %{ "rol $dst, $src, $shift" %}
13012
13013 ins_encode %{
13014 __ subw(rscratch1, zr, as_Register($shift$$reg));
13015 __ rorvw(as_Register($dst$$reg), as_Register($src$$reg), rscratch1);
13016 %}
13017 ins_pipe(ialu_reg_reg_vshift);
13018 %}
13019
13020 // This pattern is automatically generated from aarch64_ad.m4
13021 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13022 instruct rolL_reg(iRegLNoSp dst, iRegL src, iRegI shift)
13023 %{
13024 match(Set dst (RotateLeft src shift));
13025
13026 ins_cost(INSN_COST);
13027 format %{ "rol $dst, $src, $shift" %}
13028
13029 ins_encode %{
13030 __ subw(rscratch1, zr, as_Register($shift$$reg));
13031 __ rorv(as_Register($dst$$reg), as_Register($src$$reg), rscratch1);
13032 %}
13033 ins_pipe(ialu_reg_reg_vshift);
13034 %}
13035
13036
13037 // Add/subtract (extended)
13038
13039 // This pattern is automatically generated from aarch64_ad.m4
13040 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13041 instruct AddExtI(iRegLNoSp dst, iRegL src1, iRegIorL2I src2, rFlagsReg cr)
13042 %{
13043 match(Set dst (AddL src1 (ConvI2L src2)));
13044 ins_cost(INSN_COST);
13045 format %{ "add $dst, $src1, $src2, sxtw" %}
13046
13047 ins_encode %{
13048 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13049 as_Register($src2$$reg), ext::sxtw);
13050 %}
13051 ins_pipe(ialu_reg_reg);
13052 %}
13053
13054 // This pattern is automatically generated from aarch64_ad.m4
13055 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13056 instruct SubExtI(iRegLNoSp dst, iRegL src1, iRegIorL2I src2, rFlagsReg cr)
13057 %{
13058 match(Set dst (SubL src1 (ConvI2L src2)));
13059 ins_cost(INSN_COST);
13060 format %{ "sub $dst, $src1, $src2, sxtw" %}
13061
13062 ins_encode %{
13063 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13064 as_Register($src2$$reg), ext::sxtw);
13065 %}
13066 ins_pipe(ialu_reg_reg);
13067 %}
13068
13069 // This pattern is automatically generated from aarch64_ad.m4
13070 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13071 instruct AddExtI_sxth(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_16 lshift, immI_16 rshift, rFlagsReg cr)
13072 %{
13073 match(Set dst (AddI src1 (RShiftI (LShiftI src2 lshift) rshift)));
13074 ins_cost(INSN_COST);
13075 format %{ "add $dst, $src1, $src2, sxth" %}
13076
13077 ins_encode %{
13078 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13079 as_Register($src2$$reg), ext::sxth);
13080 %}
13081 ins_pipe(ialu_reg_reg);
13082 %}
13083
13084 // This pattern is automatically generated from aarch64_ad.m4
13085 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13086 instruct AddExtI_sxtb(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_24 lshift, immI_24 rshift, rFlagsReg cr)
13087 %{
13088 match(Set dst (AddI src1 (RShiftI (LShiftI src2 lshift) rshift)));
13089 ins_cost(INSN_COST);
13090 format %{ "add $dst, $src1, $src2, sxtb" %}
13091
13092 ins_encode %{
13093 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13094 as_Register($src2$$reg), ext::sxtb);
13095 %}
13096 ins_pipe(ialu_reg_reg);
13097 %}
13098
13099 // This pattern is automatically generated from aarch64_ad.m4
13100 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13101 instruct AddExtI_uxtb(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_24 lshift, immI_24 rshift, rFlagsReg cr)
13102 %{
13103 match(Set dst (AddI src1 (URShiftI (LShiftI src2 lshift) rshift)));
13104 ins_cost(INSN_COST);
13105 format %{ "add $dst, $src1, $src2, uxtb" %}
13106
13107 ins_encode %{
13108 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13109 as_Register($src2$$reg), ext::uxtb);
13110 %}
13111 ins_pipe(ialu_reg_reg);
13112 %}
13113
13114 // This pattern is automatically generated from aarch64_ad.m4
13115 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13116 instruct AddExtL_sxth(iRegLNoSp dst, iRegL src1, iRegL src2, immI_48 lshift, immI_48 rshift, rFlagsReg cr)
13117 %{
13118 match(Set dst (AddL src1 (RShiftL (LShiftL src2 lshift) rshift)));
13119 ins_cost(INSN_COST);
13120 format %{ "add $dst, $src1, $src2, sxth" %}
13121
13122 ins_encode %{
13123 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13124 as_Register($src2$$reg), ext::sxth);
13125 %}
13126 ins_pipe(ialu_reg_reg);
13127 %}
13128
13129 // This pattern is automatically generated from aarch64_ad.m4
13130 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13131 instruct AddExtL_sxtw(iRegLNoSp dst, iRegL src1, iRegL src2, immI_32 lshift, immI_32 rshift, rFlagsReg cr)
13132 %{
13133 match(Set dst (AddL src1 (RShiftL (LShiftL src2 lshift) rshift)));
13134 ins_cost(INSN_COST);
13135 format %{ "add $dst, $src1, $src2, sxtw" %}
13136
13137 ins_encode %{
13138 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13139 as_Register($src2$$reg), ext::sxtw);
13140 %}
13141 ins_pipe(ialu_reg_reg);
13142 %}
13143
13144 // This pattern is automatically generated from aarch64_ad.m4
13145 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13146 instruct AddExtL_sxtb(iRegLNoSp dst, iRegL src1, iRegL src2, immI_56 lshift, immI_56 rshift, rFlagsReg cr)
13147 %{
13148 match(Set dst (AddL src1 (RShiftL (LShiftL src2 lshift) rshift)));
13149 ins_cost(INSN_COST);
13150 format %{ "add $dst, $src1, $src2, sxtb" %}
13151
13152 ins_encode %{
13153 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13154 as_Register($src2$$reg), ext::sxtb);
13155 %}
13156 ins_pipe(ialu_reg_reg);
13157 %}
13158
13159 // This pattern is automatically generated from aarch64_ad.m4
13160 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13161 instruct AddExtL_uxtb(iRegLNoSp dst, iRegL src1, iRegL src2, immI_56 lshift, immI_56 rshift, rFlagsReg cr)
13162 %{
13163 match(Set dst (AddL src1 (URShiftL (LShiftL src2 lshift) rshift)));
13164 ins_cost(INSN_COST);
13165 format %{ "add $dst, $src1, $src2, uxtb" %}
13166
13167 ins_encode %{
13168 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13169 as_Register($src2$$reg), ext::uxtb);
13170 %}
13171 ins_pipe(ialu_reg_reg);
13172 %}
13173
13174 // This pattern is automatically generated from aarch64_ad.m4
13175 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13176 instruct AddExtI_uxtb_and(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_255 mask, rFlagsReg cr)
13177 %{
13178 match(Set dst (AddI src1 (AndI src2 mask)));
13179 ins_cost(INSN_COST);
13180 format %{ "addw $dst, $src1, $src2, uxtb" %}
13181
13182 ins_encode %{
13183 __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
13184 as_Register($src2$$reg), ext::uxtb);
13185 %}
13186 ins_pipe(ialu_reg_reg);
13187 %}
13188
13189 // This pattern is automatically generated from aarch64_ad.m4
13190 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13191 instruct AddExtI_uxth_and(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_65535 mask, rFlagsReg cr)
13192 %{
13193 match(Set dst (AddI src1 (AndI src2 mask)));
13194 ins_cost(INSN_COST);
13195 format %{ "addw $dst, $src1, $src2, uxth" %}
13196
13197 ins_encode %{
13198 __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
13199 as_Register($src2$$reg), ext::uxth);
13200 %}
13201 ins_pipe(ialu_reg_reg);
13202 %}
13203
13204 // This pattern is automatically generated from aarch64_ad.m4
13205 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13206 instruct AddExtL_uxtb_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_255 mask, rFlagsReg cr)
13207 %{
13208 match(Set dst (AddL src1 (AndL src2 mask)));
13209 ins_cost(INSN_COST);
13210 format %{ "add $dst, $src1, $src2, uxtb" %}
13211
13212 ins_encode %{
13213 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13214 as_Register($src2$$reg), ext::uxtb);
13215 %}
13216 ins_pipe(ialu_reg_reg);
13217 %}
13218
13219 // This pattern is automatically generated from aarch64_ad.m4
13220 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13221 instruct AddExtL_uxth_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_65535 mask, rFlagsReg cr)
13222 %{
13223 match(Set dst (AddL src1 (AndL src2 mask)));
13224 ins_cost(INSN_COST);
13225 format %{ "add $dst, $src1, $src2, uxth" %}
13226
13227 ins_encode %{
13228 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13229 as_Register($src2$$reg), ext::uxth);
13230 %}
13231 ins_pipe(ialu_reg_reg);
13232 %}
13233
13234 // This pattern is automatically generated from aarch64_ad.m4
13235 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13236 instruct AddExtL_uxtw_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_4294967295 mask, rFlagsReg cr)
13237 %{
13238 match(Set dst (AddL src1 (AndL src2 mask)));
13239 ins_cost(INSN_COST);
13240 format %{ "add $dst, $src1, $src2, uxtw" %}
13241
13242 ins_encode %{
13243 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13244 as_Register($src2$$reg), ext::uxtw);
13245 %}
13246 ins_pipe(ialu_reg_reg);
13247 %}
13248
13249 // This pattern is automatically generated from aarch64_ad.m4
13250 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13251 instruct SubExtI_uxtb_and(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_255 mask, rFlagsReg cr)
13252 %{
13253 match(Set dst (SubI src1 (AndI src2 mask)));
13254 ins_cost(INSN_COST);
13255 format %{ "subw $dst, $src1, $src2, uxtb" %}
13256
13257 ins_encode %{
13258 __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
13259 as_Register($src2$$reg), ext::uxtb);
13260 %}
13261 ins_pipe(ialu_reg_reg);
13262 %}
13263
13264 // This pattern is automatically generated from aarch64_ad.m4
13265 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13266 instruct SubExtI_uxth_and(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_65535 mask, rFlagsReg cr)
13267 %{
13268 match(Set dst (SubI src1 (AndI src2 mask)));
13269 ins_cost(INSN_COST);
13270 format %{ "subw $dst, $src1, $src2, uxth" %}
13271
13272 ins_encode %{
13273 __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
13274 as_Register($src2$$reg), ext::uxth);
13275 %}
13276 ins_pipe(ialu_reg_reg);
13277 %}
13278
13279 // This pattern is automatically generated from aarch64_ad.m4
13280 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13281 instruct SubExtL_uxtb_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_255 mask, rFlagsReg cr)
13282 %{
13283 match(Set dst (SubL src1 (AndL src2 mask)));
13284 ins_cost(INSN_COST);
13285 format %{ "sub $dst, $src1, $src2, uxtb" %}
13286
13287 ins_encode %{
13288 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13289 as_Register($src2$$reg), ext::uxtb);
13290 %}
13291 ins_pipe(ialu_reg_reg);
13292 %}
13293
13294 // This pattern is automatically generated from aarch64_ad.m4
13295 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13296 instruct SubExtL_uxth_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_65535 mask, rFlagsReg cr)
13297 %{
13298 match(Set dst (SubL src1 (AndL src2 mask)));
13299 ins_cost(INSN_COST);
13300 format %{ "sub $dst, $src1, $src2, uxth" %}
13301
13302 ins_encode %{
13303 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13304 as_Register($src2$$reg), ext::uxth);
13305 %}
13306 ins_pipe(ialu_reg_reg);
13307 %}
13308
13309 // This pattern is automatically generated from aarch64_ad.m4
13310 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13311 instruct SubExtL_uxtw_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_4294967295 mask, rFlagsReg cr)
13312 %{
13313 match(Set dst (SubL src1 (AndL src2 mask)));
13314 ins_cost(INSN_COST);
13315 format %{ "sub $dst, $src1, $src2, uxtw" %}
13316
13317 ins_encode %{
13318 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13319 as_Register($src2$$reg), ext::uxtw);
13320 %}
13321 ins_pipe(ialu_reg_reg);
13322 %}
13323
13324
13325 // This pattern is automatically generated from aarch64_ad.m4
13326 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13327 instruct AddExtL_sxtb_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_56 lshift1, immI_56 rshift1, rFlagsReg cr)
13328 %{
13329 match(Set dst (AddL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
13330 ins_cost(1.9 * INSN_COST);
13331 format %{ "add $dst, $src1, $src2, sxtb #lshift2" %}
13332
13333 ins_encode %{
13334 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13335 as_Register($src2$$reg), ext::sxtb, ($lshift2$$constant));
13336 %}
13337 ins_pipe(ialu_reg_reg_shift);
13338 %}
13339
13340 // This pattern is automatically generated from aarch64_ad.m4
13341 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13342 instruct AddExtL_sxth_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_48 lshift1, immI_48 rshift1, rFlagsReg cr)
13343 %{
13344 match(Set dst (AddL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
13345 ins_cost(1.9 * INSN_COST);
13346 format %{ "add $dst, $src1, $src2, sxth #lshift2" %}
13347
13348 ins_encode %{
13349 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13350 as_Register($src2$$reg), ext::sxth, ($lshift2$$constant));
13351 %}
13352 ins_pipe(ialu_reg_reg_shift);
13353 %}
13354
13355 // This pattern is automatically generated from aarch64_ad.m4
13356 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13357 instruct AddExtL_sxtw_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_32 lshift1, immI_32 rshift1, rFlagsReg cr)
13358 %{
13359 match(Set dst (AddL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
13360 ins_cost(1.9 * INSN_COST);
13361 format %{ "add $dst, $src1, $src2, sxtw #lshift2" %}
13362
13363 ins_encode %{
13364 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13365 as_Register($src2$$reg), ext::sxtw, ($lshift2$$constant));
13366 %}
13367 ins_pipe(ialu_reg_reg_shift);
13368 %}
13369
13370 // This pattern is automatically generated from aarch64_ad.m4
13371 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13372 instruct SubExtL_sxtb_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_56 lshift1, immI_56 rshift1, rFlagsReg cr)
13373 %{
13374 match(Set dst (SubL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
13375 ins_cost(1.9 * INSN_COST);
13376 format %{ "sub $dst, $src1, $src2, sxtb #lshift2" %}
13377
13378 ins_encode %{
13379 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13380 as_Register($src2$$reg), ext::sxtb, ($lshift2$$constant));
13381 %}
13382 ins_pipe(ialu_reg_reg_shift);
13383 %}
13384
13385 // This pattern is automatically generated from aarch64_ad.m4
13386 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13387 instruct SubExtL_sxth_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_48 lshift1, immI_48 rshift1, rFlagsReg cr)
13388 %{
13389 match(Set dst (SubL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
13390 ins_cost(1.9 * INSN_COST);
13391 format %{ "sub $dst, $src1, $src2, sxth #lshift2" %}
13392
13393 ins_encode %{
13394 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13395 as_Register($src2$$reg), ext::sxth, ($lshift2$$constant));
13396 %}
13397 ins_pipe(ialu_reg_reg_shift);
13398 %}
13399
13400 // This pattern is automatically generated from aarch64_ad.m4
13401 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13402 instruct SubExtL_sxtw_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_32 lshift1, immI_32 rshift1, rFlagsReg cr)
13403 %{
13404 match(Set dst (SubL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
13405 ins_cost(1.9 * INSN_COST);
13406 format %{ "sub $dst, $src1, $src2, sxtw #lshift2" %}
13407
13408 ins_encode %{
13409 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13410 as_Register($src2$$reg), ext::sxtw, ($lshift2$$constant));
13411 %}
13412 ins_pipe(ialu_reg_reg_shift);
13413 %}
13414
13415 // This pattern is automatically generated from aarch64_ad.m4
13416 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13417 instruct AddExtI_sxtb_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immIExt lshift2, immI_24 lshift1, immI_24 rshift1, rFlagsReg cr)
13418 %{
13419 match(Set dst (AddI src1 (LShiftI (RShiftI (LShiftI src2 lshift1) rshift1) lshift2)));
13420 ins_cost(1.9 * INSN_COST);
13421 format %{ "addw $dst, $src1, $src2, sxtb #lshift2" %}
13422
13423 ins_encode %{
13424 __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
13425 as_Register($src2$$reg), ext::sxtb, ($lshift2$$constant));
13426 %}
13427 ins_pipe(ialu_reg_reg_shift);
13428 %}
13429
13430 // This pattern is automatically generated from aarch64_ad.m4
13431 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13432 instruct AddExtI_sxth_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immIExt lshift2, immI_16 lshift1, immI_16 rshift1, rFlagsReg cr)
13433 %{
13434 match(Set dst (AddI src1 (LShiftI (RShiftI (LShiftI src2 lshift1) rshift1) lshift2)));
13435 ins_cost(1.9 * INSN_COST);
13436 format %{ "addw $dst, $src1, $src2, sxth #lshift2" %}
13437
13438 ins_encode %{
13439 __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
13440 as_Register($src2$$reg), ext::sxth, ($lshift2$$constant));
13441 %}
13442 ins_pipe(ialu_reg_reg_shift);
13443 %}
13444
13445 // This pattern is automatically generated from aarch64_ad.m4
13446 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13447 instruct SubExtI_sxtb_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immIExt lshift2, immI_24 lshift1, immI_24 rshift1, rFlagsReg cr)
13448 %{
13449 match(Set dst (SubI src1 (LShiftI (RShiftI (LShiftI src2 lshift1) rshift1) lshift2)));
13450 ins_cost(1.9 * INSN_COST);
13451 format %{ "subw $dst, $src1, $src2, sxtb #lshift2" %}
13452
13453 ins_encode %{
13454 __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
13455 as_Register($src2$$reg), ext::sxtb, ($lshift2$$constant));
13456 %}
13457 ins_pipe(ialu_reg_reg_shift);
13458 %}
13459
13460 // This pattern is automatically generated from aarch64_ad.m4
13461 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13462 instruct SubExtI_sxth_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immIExt lshift2, immI_16 lshift1, immI_16 rshift1, rFlagsReg cr)
13463 %{
13464 match(Set dst (SubI src1 (LShiftI (RShiftI (LShiftI src2 lshift1) rshift1) lshift2)));
13465 ins_cost(1.9 * INSN_COST);
13466 format %{ "subw $dst, $src1, $src2, sxth #lshift2" %}
13467
13468 ins_encode %{
13469 __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
13470 as_Register($src2$$reg), ext::sxth, ($lshift2$$constant));
13471 %}
13472 ins_pipe(ialu_reg_reg_shift);
13473 %}
13474
13475 // This pattern is automatically generated from aarch64_ad.m4
13476 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13477 instruct AddExtI_shift(iRegLNoSp dst, iRegL src1, iRegIorL2I src2, immIExt lshift, rFlagsReg cr)
13478 %{
13479 match(Set dst (AddL src1 (LShiftL (ConvI2L src2) lshift)));
13480 ins_cost(1.9 * INSN_COST);
13481 format %{ "add $dst, $src1, $src2, sxtw #lshift" %}
13482
13483 ins_encode %{
13484 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13485 as_Register($src2$$reg), ext::sxtw, ($lshift$$constant));
13486 %}
13487 ins_pipe(ialu_reg_reg_shift);
13488 %}
13489
13490 // This pattern is automatically generated from aarch64_ad.m4
13491 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13492 instruct SubExtI_shift(iRegLNoSp dst, iRegL src1, iRegIorL2I src2, immIExt lshift, rFlagsReg cr)
13493 %{
13494 match(Set dst (SubL src1 (LShiftL (ConvI2L src2) lshift)));
13495 ins_cost(1.9 * INSN_COST);
13496 format %{ "sub $dst, $src1, $src2, sxtw #lshift" %}
13497
13498 ins_encode %{
13499 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13500 as_Register($src2$$reg), ext::sxtw, ($lshift$$constant));
13501 %}
13502 ins_pipe(ialu_reg_reg_shift);
13503 %}
13504
13505 // This pattern is automatically generated from aarch64_ad.m4
13506 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13507 instruct AddExtL_uxtb_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_255 mask, immIExt lshift, rFlagsReg cr)
13508 %{
13509 match(Set dst (AddL src1 (LShiftL (AndL src2 mask) lshift)));
13510 ins_cost(1.9 * INSN_COST);
13511 format %{ "add $dst, $src1, $src2, uxtb #lshift" %}
13512
13513 ins_encode %{
13514 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13515 as_Register($src2$$reg), ext::uxtb, ($lshift$$constant));
13516 %}
13517 ins_pipe(ialu_reg_reg_shift);
13518 %}
13519
13520 // This pattern is automatically generated from aarch64_ad.m4
13521 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13522 instruct AddExtL_uxth_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_65535 mask, immIExt lshift, rFlagsReg cr)
13523 %{
13524 match(Set dst (AddL src1 (LShiftL (AndL src2 mask) lshift)));
13525 ins_cost(1.9 * INSN_COST);
13526 format %{ "add $dst, $src1, $src2, uxth #lshift" %}
13527
13528 ins_encode %{
13529 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13530 as_Register($src2$$reg), ext::uxth, ($lshift$$constant));
13531 %}
13532 ins_pipe(ialu_reg_reg_shift);
13533 %}
13534
13535 // This pattern is automatically generated from aarch64_ad.m4
13536 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13537 instruct AddExtL_uxtw_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_4294967295 mask, immIExt lshift, rFlagsReg cr)
13538 %{
13539 match(Set dst (AddL src1 (LShiftL (AndL src2 mask) lshift)));
13540 ins_cost(1.9 * INSN_COST);
13541 format %{ "add $dst, $src1, $src2, uxtw #lshift" %}
13542
13543 ins_encode %{
13544 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13545 as_Register($src2$$reg), ext::uxtw, ($lshift$$constant));
13546 %}
13547 ins_pipe(ialu_reg_reg_shift);
13548 %}
13549
13550 // This pattern is automatically generated from aarch64_ad.m4
13551 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13552 instruct SubExtL_uxtb_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_255 mask, immIExt lshift, rFlagsReg cr)
13553 %{
13554 match(Set dst (SubL src1 (LShiftL (AndL src2 mask) lshift)));
13555 ins_cost(1.9 * INSN_COST);
13556 format %{ "sub $dst, $src1, $src2, uxtb #lshift" %}
13557
13558 ins_encode %{
13559 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13560 as_Register($src2$$reg), ext::uxtb, ($lshift$$constant));
13561 %}
13562 ins_pipe(ialu_reg_reg_shift);
13563 %}
13564
13565 // This pattern is automatically generated from aarch64_ad.m4
13566 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13567 instruct SubExtL_uxth_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_65535 mask, immIExt lshift, rFlagsReg cr)
13568 %{
13569 match(Set dst (SubL src1 (LShiftL (AndL src2 mask) lshift)));
13570 ins_cost(1.9 * INSN_COST);
13571 format %{ "sub $dst, $src1, $src2, uxth #lshift" %}
13572
13573 ins_encode %{
13574 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13575 as_Register($src2$$reg), ext::uxth, ($lshift$$constant));
13576 %}
13577 ins_pipe(ialu_reg_reg_shift);
13578 %}
13579
13580 // This pattern is automatically generated from aarch64_ad.m4
13581 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13582 instruct SubExtL_uxtw_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_4294967295 mask, immIExt lshift, rFlagsReg cr)
13583 %{
13584 match(Set dst (SubL src1 (LShiftL (AndL src2 mask) lshift)));
13585 ins_cost(1.9 * INSN_COST);
13586 format %{ "sub $dst, $src1, $src2, uxtw #lshift" %}
13587
13588 ins_encode %{
13589 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13590 as_Register($src2$$reg), ext::uxtw, ($lshift$$constant));
13591 %}
13592 ins_pipe(ialu_reg_reg_shift);
13593 %}
13594
13595 // This pattern is automatically generated from aarch64_ad.m4
13596 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13597 instruct AddExtI_uxtb_and_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_255 mask, immIExt lshift, rFlagsReg cr)
13598 %{
13599 match(Set dst (AddI src1 (LShiftI (AndI src2 mask) lshift)));
13600 ins_cost(1.9 * INSN_COST);
13601 format %{ "addw $dst, $src1, $src2, uxtb #lshift" %}
13602
13603 ins_encode %{
13604 __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
13605 as_Register($src2$$reg), ext::uxtb, ($lshift$$constant));
13606 %}
13607 ins_pipe(ialu_reg_reg_shift);
13608 %}
13609
13610 // This pattern is automatically generated from aarch64_ad.m4
13611 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13612 instruct AddExtI_uxth_and_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_65535 mask, immIExt lshift, rFlagsReg cr)
13613 %{
13614 match(Set dst (AddI src1 (LShiftI (AndI src2 mask) lshift)));
13615 ins_cost(1.9 * INSN_COST);
13616 format %{ "addw $dst, $src1, $src2, uxth #lshift" %}
13617
13618 ins_encode %{
13619 __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
13620 as_Register($src2$$reg), ext::uxth, ($lshift$$constant));
13621 %}
13622 ins_pipe(ialu_reg_reg_shift);
13623 %}
13624
13625 // This pattern is automatically generated from aarch64_ad.m4
13626 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13627 instruct SubExtI_uxtb_and_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_255 mask, immIExt lshift, rFlagsReg cr)
13628 %{
13629 match(Set dst (SubI src1 (LShiftI (AndI src2 mask) lshift)));
13630 ins_cost(1.9 * INSN_COST);
13631 format %{ "subw $dst, $src1, $src2, uxtb #lshift" %}
13632
13633 ins_encode %{
13634 __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
13635 as_Register($src2$$reg), ext::uxtb, ($lshift$$constant));
13636 %}
13637 ins_pipe(ialu_reg_reg_shift);
13638 %}
13639
13640 // This pattern is automatically generated from aarch64_ad.m4
13641 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13642 instruct SubExtI_uxth_and_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_65535 mask, immIExt lshift, rFlagsReg cr)
13643 %{
13644 match(Set dst (SubI src1 (LShiftI (AndI src2 mask) lshift)));
13645 ins_cost(1.9 * INSN_COST);
13646 format %{ "subw $dst, $src1, $src2, uxth #lshift" %}
13647
13648 ins_encode %{
13649 __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
13650 as_Register($src2$$reg), ext::uxth, ($lshift$$constant));
13651 %}
13652 ins_pipe(ialu_reg_reg_shift);
13653 %}
13654
13655
13656
13657 // END This section of the file is automatically generated. Do not edit --------------
13658
13659
13660 // ============================================================================
13661 // Floating Point Arithmetic Instructions
13662
13663 instruct addF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
13664 match(Set dst (AddF src1 src2));
13665
13666 ins_cost(INSN_COST * 5);
13667 format %{ "fadds $dst, $src1, $src2" %}
13668
13669 ins_encode %{
13670 __ fadds(as_FloatRegister($dst$$reg),
13671 as_FloatRegister($src1$$reg),
13672 as_FloatRegister($src2$$reg));
13673 %}
13674
13675 ins_pipe(fp_dop_reg_reg_s);
13676 %}
13677
13678 instruct addD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
13679 match(Set dst (AddD src1 src2));
13680
13681 ins_cost(INSN_COST * 5);
13682 format %{ "faddd $dst, $src1, $src2" %}
13683
13684 ins_encode %{
13685 __ faddd(as_FloatRegister($dst$$reg),
13686 as_FloatRegister($src1$$reg),
13687 as_FloatRegister($src2$$reg));
13688 %}
13689
13690 ins_pipe(fp_dop_reg_reg_d);
13691 %}
13692
13693 instruct subF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
13694 match(Set dst (SubF src1 src2));
13695
13696 ins_cost(INSN_COST * 5);
13697 format %{ "fsubs $dst, $src1, $src2" %}
13698
13699 ins_encode %{
13700 __ fsubs(as_FloatRegister($dst$$reg),
13701 as_FloatRegister($src1$$reg),
13702 as_FloatRegister($src2$$reg));
13703 %}
13704
13705 ins_pipe(fp_dop_reg_reg_s);
13706 %}
13707
13708 instruct subD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
13709 match(Set dst (SubD src1 src2));
13710
13711 ins_cost(INSN_COST * 5);
13712 format %{ "fsubd $dst, $src1, $src2" %}
13713
13714 ins_encode %{
13715 __ fsubd(as_FloatRegister($dst$$reg),
13716 as_FloatRegister($src1$$reg),
13717 as_FloatRegister($src2$$reg));
13718 %}
13719
13720 ins_pipe(fp_dop_reg_reg_d);
13721 %}
13722
13723 instruct mulF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
13724 match(Set dst (MulF src1 src2));
13725
13726 ins_cost(INSN_COST * 6);
13727 format %{ "fmuls $dst, $src1, $src2" %}
13728
13729 ins_encode %{
13730 __ fmuls(as_FloatRegister($dst$$reg),
13731 as_FloatRegister($src1$$reg),
13732 as_FloatRegister($src2$$reg));
13733 %}
13734
13735 ins_pipe(fp_dop_reg_reg_s);
13736 %}
13737
13738 instruct mulD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
13739 match(Set dst (MulD src1 src2));
13740
13741 ins_cost(INSN_COST * 6);
13742 format %{ "fmuld $dst, $src1, $src2" %}
13743
13744 ins_encode %{
13745 __ fmuld(as_FloatRegister($dst$$reg),
13746 as_FloatRegister($src1$$reg),
13747 as_FloatRegister($src2$$reg));
13748 %}
13749
13750 ins_pipe(fp_dop_reg_reg_d);
13751 %}
13752
13753 // src1 * src2 + src3
13754 instruct maddF_reg_reg(vRegF dst, vRegF src1, vRegF src2, vRegF src3) %{
13755 predicate(UseFMA);
13756 match(Set dst (FmaF src3 (Binary src1 src2)));
13757
13758 format %{ "fmadds $dst, $src1, $src2, $src3" %}
13759
13760 ins_encode %{
13761 __ fmadds(as_FloatRegister($dst$$reg),
13762 as_FloatRegister($src1$$reg),
13763 as_FloatRegister($src2$$reg),
13764 as_FloatRegister($src3$$reg));
13765 %}
13766
13767 ins_pipe(pipe_class_default);
13768 %}
13769
13770 // src1 * src2 + src3
13771 instruct maddD_reg_reg(vRegD dst, vRegD src1, vRegD src2, vRegD src3) %{
13772 predicate(UseFMA);
13773 match(Set dst (FmaD src3 (Binary src1 src2)));
13774
13775 format %{ "fmaddd $dst, $src1, $src2, $src3" %}
13776
13777 ins_encode %{
13778 __ fmaddd(as_FloatRegister($dst$$reg),
13779 as_FloatRegister($src1$$reg),
13780 as_FloatRegister($src2$$reg),
13781 as_FloatRegister($src3$$reg));
13782 %}
13783
13784 ins_pipe(pipe_class_default);
13785 %}
13786
13787 // -src1 * src2 + src3
13788 instruct msubF_reg_reg(vRegF dst, vRegF src1, vRegF src2, vRegF src3) %{
13789 predicate(UseFMA);
13790 match(Set dst (FmaF src3 (Binary (NegF src1) src2)));
13791 match(Set dst (FmaF src3 (Binary src1 (NegF src2))));
13792
13793 format %{ "fmsubs $dst, $src1, $src2, $src3" %}
13794
13795 ins_encode %{
13796 __ fmsubs(as_FloatRegister($dst$$reg),
13797 as_FloatRegister($src1$$reg),
13798 as_FloatRegister($src2$$reg),
13799 as_FloatRegister($src3$$reg));
13800 %}
13801
13802 ins_pipe(pipe_class_default);
13803 %}
13804
13805 // -src1 * src2 + src3
13806 instruct msubD_reg_reg(vRegD dst, vRegD src1, vRegD src2, vRegD src3) %{
13807 predicate(UseFMA);
13808 match(Set dst (FmaD src3 (Binary (NegD src1) src2)));
13809 match(Set dst (FmaD src3 (Binary src1 (NegD src2))));
13810
13811 format %{ "fmsubd $dst, $src1, $src2, $src3" %}
13812
13813 ins_encode %{
13814 __ fmsubd(as_FloatRegister($dst$$reg),
13815 as_FloatRegister($src1$$reg),
13816 as_FloatRegister($src2$$reg),
13817 as_FloatRegister($src3$$reg));
13818 %}
13819
13820 ins_pipe(pipe_class_default);
13821 %}
13822
13823 // -src1 * src2 - src3
13824 instruct mnaddF_reg_reg(vRegF dst, vRegF src1, vRegF src2, vRegF src3) %{
13825 predicate(UseFMA);
13826 match(Set dst (FmaF (NegF src3) (Binary (NegF src1) src2)));
13827 match(Set dst (FmaF (NegF src3) (Binary src1 (NegF src2))));
13828
13829 format %{ "fnmadds $dst, $src1, $src2, $src3" %}
13830
13831 ins_encode %{
13832 __ fnmadds(as_FloatRegister($dst$$reg),
13833 as_FloatRegister($src1$$reg),
13834 as_FloatRegister($src2$$reg),
13835 as_FloatRegister($src3$$reg));
13836 %}
13837
13838 ins_pipe(pipe_class_default);
13839 %}
13840
13841 // -src1 * src2 - src3
13842 instruct mnaddD_reg_reg(vRegD dst, vRegD src1, vRegD src2, vRegD src3) %{
13843 predicate(UseFMA);
13844 match(Set dst (FmaD (NegD src3) (Binary (NegD src1) src2)));
13845 match(Set dst (FmaD (NegD src3) (Binary src1 (NegD src2))));
13846
13847 format %{ "fnmaddd $dst, $src1, $src2, $src3" %}
13848
13849 ins_encode %{
13850 __ fnmaddd(as_FloatRegister($dst$$reg),
13851 as_FloatRegister($src1$$reg),
13852 as_FloatRegister($src2$$reg),
13853 as_FloatRegister($src3$$reg));
13854 %}
13855
13856 ins_pipe(pipe_class_default);
13857 %}
13858
13859 // src1 * src2 - src3
13860 instruct mnsubF_reg_reg(vRegF dst, vRegF src1, vRegF src2, vRegF src3, immF0 zero) %{
13861 predicate(UseFMA);
13862 match(Set dst (FmaF (NegF src3) (Binary src1 src2)));
13863
13864 format %{ "fnmsubs $dst, $src1, $src2, $src3" %}
13865
13866 ins_encode %{
13867 __ fnmsubs(as_FloatRegister($dst$$reg),
13868 as_FloatRegister($src1$$reg),
13869 as_FloatRegister($src2$$reg),
13870 as_FloatRegister($src3$$reg));
13871 %}
13872
13873 ins_pipe(pipe_class_default);
13874 %}
13875
13876 // src1 * src2 - src3
13877 instruct mnsubD_reg_reg(vRegD dst, vRegD src1, vRegD src2, vRegD src3, immD0 zero) %{
13878 predicate(UseFMA);
13879 match(Set dst (FmaD (NegD src3) (Binary src1 src2)));
13880
13881 format %{ "fnmsubd $dst, $src1, $src2, $src3" %}
13882
13883 ins_encode %{
13884 // n.b. insn name should be fnmsubd
13885 __ fnmsub(as_FloatRegister($dst$$reg),
13886 as_FloatRegister($src1$$reg),
13887 as_FloatRegister($src2$$reg),
13888 as_FloatRegister($src3$$reg));
13889 %}
13890
13891 ins_pipe(pipe_class_default);
13892 %}
13893
13894
13895 // Math.max(FF)F
13896 instruct maxF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
13897 match(Set dst (MaxF src1 src2));
13898
13899 format %{ "fmaxs $dst, $src1, $src2" %}
13900 ins_encode %{
13901 __ fmaxs(as_FloatRegister($dst$$reg),
13902 as_FloatRegister($src1$$reg),
13903 as_FloatRegister($src2$$reg));
13904 %}
13905
13906 ins_pipe(fp_dop_reg_reg_s);
13907 %}
13908
13909 // Math.min(FF)F
13910 instruct minF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
13911 match(Set dst (MinF src1 src2));
13912
13913 format %{ "fmins $dst, $src1, $src2" %}
13914 ins_encode %{
13915 __ fmins(as_FloatRegister($dst$$reg),
13916 as_FloatRegister($src1$$reg),
13917 as_FloatRegister($src2$$reg));
13918 %}
13919
13920 ins_pipe(fp_dop_reg_reg_s);
13921 %}
13922
13923 // Math.max(DD)D
13924 instruct maxD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
13925 match(Set dst (MaxD src1 src2));
13926
13927 format %{ "fmaxd $dst, $src1, $src2" %}
13928 ins_encode %{
13929 __ fmaxd(as_FloatRegister($dst$$reg),
13930 as_FloatRegister($src1$$reg),
13931 as_FloatRegister($src2$$reg));
13932 %}
13933
13934 ins_pipe(fp_dop_reg_reg_d);
13935 %}
13936
13937 // Math.min(DD)D
13938 instruct minD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
13939 match(Set dst (MinD src1 src2));
13940
13941 format %{ "fmind $dst, $src1, $src2" %}
13942 ins_encode %{
13943 __ fmind(as_FloatRegister($dst$$reg),
13944 as_FloatRegister($src1$$reg),
13945 as_FloatRegister($src2$$reg));
13946 %}
13947
13948 ins_pipe(fp_dop_reg_reg_d);
13949 %}
13950
13951
13952 instruct divF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
13953 match(Set dst (DivF src1 src2));
13954
13955 ins_cost(INSN_COST * 18);
13956 format %{ "fdivs $dst, $src1, $src2" %}
13957
13958 ins_encode %{
13959 __ fdivs(as_FloatRegister($dst$$reg),
13960 as_FloatRegister($src1$$reg),
13961 as_FloatRegister($src2$$reg));
13962 %}
13963
13964 ins_pipe(fp_div_s);
13965 %}
13966
13967 instruct divD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
13968 match(Set dst (DivD src1 src2));
13969
13970 ins_cost(INSN_COST * 32);
13971 format %{ "fdivd $dst, $src1, $src2" %}
13972
13973 ins_encode %{
13974 __ fdivd(as_FloatRegister($dst$$reg),
13975 as_FloatRegister($src1$$reg),
13976 as_FloatRegister($src2$$reg));
13977 %}
13978
13979 ins_pipe(fp_div_d);
13980 %}
13981
13982 instruct negF_reg_reg(vRegF dst, vRegF src) %{
13983 match(Set dst (NegF src));
13984
13985 ins_cost(INSN_COST * 3);
13986 format %{ "fneg $dst, $src" %}
13987
13988 ins_encode %{
13989 __ fnegs(as_FloatRegister($dst$$reg),
13990 as_FloatRegister($src$$reg));
13991 %}
13992
13993 ins_pipe(fp_uop_s);
13994 %}
13995
13996 instruct negD_reg_reg(vRegD dst, vRegD src) %{
13997 match(Set dst (NegD src));
13998
13999 ins_cost(INSN_COST * 3);
14000 format %{ "fnegd $dst, $src" %}
14001
14002 ins_encode %{
14003 __ fnegd(as_FloatRegister($dst$$reg),
14004 as_FloatRegister($src$$reg));
14005 %}
14006
14007 ins_pipe(fp_uop_d);
14008 %}
14009
14010 instruct absI_reg(iRegINoSp dst, iRegIorL2I src, rFlagsReg cr)
14011 %{
14012 match(Set dst (AbsI src));
14013
14014 effect(KILL cr);
14015 ins_cost(INSN_COST * 2);
14016 format %{ "cmpw $src, zr\n\t"
14017 "cnegw $dst, $src, Assembler::LT\t# int abs"
14018 %}
14019
14020 ins_encode %{
14021 __ cmpw(as_Register($src$$reg), zr);
14022 __ cnegw(as_Register($dst$$reg), as_Register($src$$reg), Assembler::LT);
14023 %}
14024 ins_pipe(pipe_class_default);
14025 %}
14026
14027 instruct absL_reg(iRegLNoSp dst, iRegL src, rFlagsReg cr)
14028 %{
14029 match(Set dst (AbsL src));
14030
14031 effect(KILL cr);
14032 ins_cost(INSN_COST * 2);
14033 format %{ "cmp $src, zr\n\t"
14034 "cneg $dst, $src, Assembler::LT\t# long abs"
14035 %}
14036
14037 ins_encode %{
14038 __ cmp(as_Register($src$$reg), zr);
14039 __ cneg(as_Register($dst$$reg), as_Register($src$$reg), Assembler::LT);
14040 %}
14041 ins_pipe(pipe_class_default);
14042 %}
14043
14044 instruct absF_reg(vRegF dst, vRegF src) %{
14045 match(Set dst (AbsF src));
14046
14047 ins_cost(INSN_COST * 3);
14048 format %{ "fabss $dst, $src" %}
14049 ins_encode %{
14050 __ fabss(as_FloatRegister($dst$$reg),
14051 as_FloatRegister($src$$reg));
14052 %}
14053
14054 ins_pipe(fp_uop_s);
14055 %}
14056
14057 instruct absD_reg(vRegD dst, vRegD src) %{
14058 match(Set dst (AbsD src));
14059
14060 ins_cost(INSN_COST * 3);
14061 format %{ "fabsd $dst, $src" %}
14062 ins_encode %{
14063 __ fabsd(as_FloatRegister($dst$$reg),
14064 as_FloatRegister($src$$reg));
14065 %}
14066
14067 ins_pipe(fp_uop_d);
14068 %}
14069
14070 instruct absdF_reg(vRegF dst, vRegF src1, vRegF src2) %{
14071 match(Set dst (AbsF (SubF src1 src2)));
14072
14073 ins_cost(INSN_COST * 3);
14074 format %{ "fabds $dst, $src1, $src2" %}
14075 ins_encode %{
14076 __ fabds(as_FloatRegister($dst$$reg),
14077 as_FloatRegister($src1$$reg),
14078 as_FloatRegister($src2$$reg));
14079 %}
14080
14081 ins_pipe(fp_uop_s);
14082 %}
14083
14084 instruct absdD_reg(vRegD dst, vRegD src1, vRegD src2) %{
14085 match(Set dst (AbsD (SubD src1 src2)));
14086
14087 ins_cost(INSN_COST * 3);
14088 format %{ "fabdd $dst, $src1, $src2" %}
14089 ins_encode %{
14090 __ fabdd(as_FloatRegister($dst$$reg),
14091 as_FloatRegister($src1$$reg),
14092 as_FloatRegister($src2$$reg));
14093 %}
14094
14095 ins_pipe(fp_uop_d);
14096 %}
14097
14098 instruct sqrtD_reg(vRegD dst, vRegD src) %{
14099 match(Set dst (SqrtD src));
14100
14101 ins_cost(INSN_COST * 50);
14102 format %{ "fsqrtd $dst, $src" %}
14103 ins_encode %{
14104 __ fsqrtd(as_FloatRegister($dst$$reg),
14105 as_FloatRegister($src$$reg));
14106 %}
14107
14108 ins_pipe(fp_div_s);
14109 %}
14110
14111 instruct sqrtF_reg(vRegF dst, vRegF src) %{
14112 match(Set dst (SqrtF src));
14113
14114 ins_cost(INSN_COST * 50);
14115 format %{ "fsqrts $dst, $src" %}
14116 ins_encode %{
14117 __ fsqrts(as_FloatRegister($dst$$reg),
14118 as_FloatRegister($src$$reg));
14119 %}
14120
14121 ins_pipe(fp_div_d);
14122 %}
14123
14124 // Math.rint, floor, ceil
14125 instruct roundD_reg(vRegD dst, vRegD src, immI rmode) %{
14126 match(Set dst (RoundDoubleMode src rmode));
14127 format %{ "frint $dst, $src, $rmode" %}
14128 ins_encode %{
14129 switch ($rmode$$constant) {
14130 case RoundDoubleModeNode::rmode_rint:
14131 __ frintnd(as_FloatRegister($dst$$reg),
14132 as_FloatRegister($src$$reg));
14133 break;
14134 case RoundDoubleModeNode::rmode_floor:
14135 __ frintmd(as_FloatRegister($dst$$reg),
14136 as_FloatRegister($src$$reg));
14137 break;
14138 case RoundDoubleModeNode::rmode_ceil:
14139 __ frintpd(as_FloatRegister($dst$$reg),
14140 as_FloatRegister($src$$reg));
14141 break;
14142 }
14143 %}
14144 ins_pipe(fp_uop_d);
14145 %}
14146
14147 instruct copySignD_reg(vRegD dst, vRegD src1, vRegD src2, vRegD zero) %{
14148 match(Set dst (CopySignD src1 (Binary src2 zero)));
14149 effect(TEMP_DEF dst, USE src1, USE src2, USE zero);
14150 format %{ "CopySignD $dst $src1 $src2" %}
14151 ins_encode %{
14152 FloatRegister dst = as_FloatRegister($dst$$reg),
14153 src1 = as_FloatRegister($src1$$reg),
14154 src2 = as_FloatRegister($src2$$reg),
14155 zero = as_FloatRegister($zero$$reg);
14156 __ fnegd(dst, zero);
14157 __ bsl(dst, __ T8B, src2, src1);
14158 %}
14159 ins_pipe(fp_uop_d);
14160 %}
14161
14162 instruct copySignF_reg(vRegF dst, vRegF src1, vRegF src2) %{
14163 match(Set dst (CopySignF src1 src2));
14164 effect(TEMP_DEF dst, USE src1, USE src2);
14165 format %{ "CopySignF $dst $src1 $src2" %}
14166 ins_encode %{
14167 FloatRegister dst = as_FloatRegister($dst$$reg),
14168 src1 = as_FloatRegister($src1$$reg),
14169 src2 = as_FloatRegister($src2$$reg);
14170 __ movi(dst, __ T2S, 0x80, 24);
14171 __ bsl(dst, __ T8B, src2, src1);
14172 %}
14173 ins_pipe(fp_uop_d);
14174 %}
14175
14176 instruct signumD_reg(vRegD dst, vRegD src, vRegD zero, vRegD one) %{
14177 match(Set dst (SignumD src (Binary zero one)));
14178 effect(TEMP_DEF dst, USE src, USE zero, USE one);
14179 format %{ "signumD $dst, $src" %}
14180 ins_encode %{
14181 FloatRegister src = as_FloatRegister($src$$reg),
14182 dst = as_FloatRegister($dst$$reg),
14183 zero = as_FloatRegister($zero$$reg),
14184 one = as_FloatRegister($one$$reg);
14185 __ facgtd(dst, src, zero); // dst=0 for +-0.0 and NaN. 0xFFF..F otherwise
14186 __ ushrd(dst, dst, 1); // dst=0 for +-0.0 and NaN. 0x7FF..F otherwise
14187 // Bit selection instruction gets bit from "one" for each enabled bit in
14188 // "dst", otherwise gets a bit from "src". For "src" that contains +-0.0 or
14189 // NaN the whole "src" will be copied because "dst" is zero. For all other
14190 // "src" values dst is 0x7FF..F, which means only the sign bit is copied
14191 // from "src", and all other bits are copied from 1.0.
14192 __ bsl(dst, __ T8B, one, src);
14193 %}
14194 ins_pipe(fp_uop_d);
14195 %}
14196
14197 instruct signumF_reg(vRegF dst, vRegF src, vRegF zero, vRegF one) %{
14198 match(Set dst (SignumF src (Binary zero one)));
14199 effect(TEMP_DEF dst, USE src, USE zero, USE one);
14200 format %{ "signumF $dst, $src" %}
14201 ins_encode %{
14202 FloatRegister src = as_FloatRegister($src$$reg),
14203 dst = as_FloatRegister($dst$$reg),
14204 zero = as_FloatRegister($zero$$reg),
14205 one = as_FloatRegister($one$$reg);
14206 __ facgts(dst, src, zero); // dst=0 for +-0.0 and NaN. 0xFFF..F otherwise
14207 __ ushr(dst, __ T2S, dst, 1); // dst=0 for +-0.0 and NaN. 0x7FF..F otherwise
14208 // Bit selection instruction gets bit from "one" for each enabled bit in
14209 // "dst", otherwise gets a bit from "src". For "src" that contains +-0.0 or
14210 // NaN the whole "src" will be copied because "dst" is zero. For all other
14211 // "src" values dst is 0x7FF..F, which means only the sign bit is copied
14212 // from "src", and all other bits are copied from 1.0.
14213 __ bsl(dst, __ T8B, one, src);
14214 %}
14215 ins_pipe(fp_uop_d);
14216 %}
14217
14218 instruct onspinwait() %{
14219 match(OnSpinWait);
14220 ins_cost(INSN_COST);
14221
14222 format %{ "onspinwait" %}
14223
14224 ins_encode %{
14225 __ spin_wait();
14226 %}
14227 ins_pipe(pipe_class_empty);
14228 %}
14229
14230 // ============================================================================
14231 // Logical Instructions
14232
14233 // Integer Logical Instructions
14234
14235 // And Instructions
14236
14237
14238 instruct andI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, rFlagsReg cr) %{
14239 match(Set dst (AndI src1 src2));
14240
14241 format %{ "andw $dst, $src1, $src2\t# int" %}
14242
14243 ins_cost(INSN_COST);
14244 ins_encode %{
14245 __ andw(as_Register($dst$$reg),
14246 as_Register($src1$$reg),
14247 as_Register($src2$$reg));
14248 %}
14249
14250 ins_pipe(ialu_reg_reg);
14251 %}
14252
14253 instruct andI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immILog src2, rFlagsReg cr) %{
14254 match(Set dst (AndI src1 src2));
14255
14256 format %{ "andsw $dst, $src1, $src2\t# int" %}
14257
14258 ins_cost(INSN_COST);
14259 ins_encode %{
14260 __ andw(as_Register($dst$$reg),
14261 as_Register($src1$$reg),
14262 (uint64_t)($src2$$constant));
14263 %}
14264
14265 ins_pipe(ialu_reg_imm);
14266 %}
14267
14268 // Or Instructions
14269
14270 instruct orI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
14271 match(Set dst (OrI src1 src2));
14272
14273 format %{ "orrw $dst, $src1, $src2\t# int" %}
14274
14275 ins_cost(INSN_COST);
14276 ins_encode %{
14277 __ orrw(as_Register($dst$$reg),
14278 as_Register($src1$$reg),
14279 as_Register($src2$$reg));
14280 %}
14281
14282 ins_pipe(ialu_reg_reg);
14283 %}
14284
14285 instruct orI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immILog src2) %{
14286 match(Set dst (OrI src1 src2));
14287
14288 format %{ "orrw $dst, $src1, $src2\t# int" %}
14289
14290 ins_cost(INSN_COST);
14291 ins_encode %{
14292 __ orrw(as_Register($dst$$reg),
14293 as_Register($src1$$reg),
14294 (uint64_t)($src2$$constant));
14295 %}
14296
14297 ins_pipe(ialu_reg_imm);
14298 %}
14299
14300 // Xor Instructions
14301
14302 instruct xorI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
14303 match(Set dst (XorI src1 src2));
14304
14305 format %{ "eorw $dst, $src1, $src2\t# int" %}
14306
14307 ins_cost(INSN_COST);
14308 ins_encode %{
14309 __ eorw(as_Register($dst$$reg),
14310 as_Register($src1$$reg),
14311 as_Register($src2$$reg));
14312 %}
14313
14314 ins_pipe(ialu_reg_reg);
14315 %}
14316
14317 instruct xorI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immILog src2) %{
14318 match(Set dst (XorI src1 src2));
14319
14320 format %{ "eorw $dst, $src1, $src2\t# int" %}
14321
14322 ins_cost(INSN_COST);
14323 ins_encode %{
14324 __ eorw(as_Register($dst$$reg),
14325 as_Register($src1$$reg),
14326 (uint64_t)($src2$$constant));
14327 %}
14328
14329 ins_pipe(ialu_reg_imm);
14330 %}
14331
14332 // Long Logical Instructions
14333 // TODO
14334
14335 instruct andL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2, rFlagsReg cr) %{
14336 match(Set dst (AndL src1 src2));
14337
14338 format %{ "and $dst, $src1, $src2\t# int" %}
14339
14340 ins_cost(INSN_COST);
14341 ins_encode %{
14342 __ andr(as_Register($dst$$reg),
14343 as_Register($src1$$reg),
14344 as_Register($src2$$reg));
14345 %}
14346
14347 ins_pipe(ialu_reg_reg);
14348 %}
14349
14350 instruct andL_reg_imm(iRegLNoSp dst, iRegL src1, immLLog src2, rFlagsReg cr) %{
14351 match(Set dst (AndL src1 src2));
14352
14353 format %{ "and $dst, $src1, $src2\t# int" %}
14354
14355 ins_cost(INSN_COST);
14356 ins_encode %{
14357 __ andr(as_Register($dst$$reg),
14358 as_Register($src1$$reg),
14359 (uint64_t)($src2$$constant));
14360 %}
14361
14362 ins_pipe(ialu_reg_imm);
14363 %}
14364
14365 // Or Instructions
14366
14367 instruct orL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
14368 match(Set dst (OrL src1 src2));
14369
14370 format %{ "orr $dst, $src1, $src2\t# int" %}
14371
14372 ins_cost(INSN_COST);
14373 ins_encode %{
14374 __ orr(as_Register($dst$$reg),
14375 as_Register($src1$$reg),
14376 as_Register($src2$$reg));
14377 %}
14378
14379 ins_pipe(ialu_reg_reg);
14380 %}
14381
14382 instruct orL_reg_imm(iRegLNoSp dst, iRegL src1, immLLog src2) %{
14383 match(Set dst (OrL src1 src2));
14384
14385 format %{ "orr $dst, $src1, $src2\t# int" %}
14386
14387 ins_cost(INSN_COST);
14388 ins_encode %{
14389 __ orr(as_Register($dst$$reg),
14390 as_Register($src1$$reg),
14391 (uint64_t)($src2$$constant));
14392 %}
14393
14394 ins_pipe(ialu_reg_imm);
14395 %}
14396
14397 // Xor Instructions
14398
14399 instruct xorL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
14400 match(Set dst (XorL src1 src2));
14401
14402 format %{ "eor $dst, $src1, $src2\t# int" %}
14403
14404 ins_cost(INSN_COST);
14405 ins_encode %{
14406 __ eor(as_Register($dst$$reg),
14407 as_Register($src1$$reg),
14408 as_Register($src2$$reg));
14409 %}
14410
14411 ins_pipe(ialu_reg_reg);
14412 %}
14413
14414 instruct xorL_reg_imm(iRegLNoSp dst, iRegL src1, immLLog src2) %{
14415 match(Set dst (XorL src1 src2));
14416
14417 ins_cost(INSN_COST);
14418 format %{ "eor $dst, $src1, $src2\t# int" %}
14419
14420 ins_encode %{
14421 __ eor(as_Register($dst$$reg),
14422 as_Register($src1$$reg),
14423 (uint64_t)($src2$$constant));
14424 %}
14425
14426 ins_pipe(ialu_reg_imm);
14427 %}
14428
14429 instruct convI2L_reg_reg(iRegLNoSp dst, iRegIorL2I src)
14430 %{
14431 match(Set dst (ConvI2L src));
14432
14433 ins_cost(INSN_COST);
14434 format %{ "sxtw $dst, $src\t# i2l" %}
14435 ins_encode %{
14436 __ sbfm($dst$$Register, $src$$Register, 0, 31);
14437 %}
14438 ins_pipe(ialu_reg_shift);
14439 %}
14440
14441 // this pattern occurs in bigmath arithmetic
14442 instruct convUI2L_reg_reg(iRegLNoSp dst, iRegIorL2I src, immL_32bits mask)
14443 %{
14444 match(Set dst (AndL (ConvI2L src) mask));
14445
14446 ins_cost(INSN_COST);
14447 format %{ "ubfm $dst, $src, 0, 31\t# ui2l" %}
14448 ins_encode %{
14449 __ ubfm($dst$$Register, $src$$Register, 0, 31);
14450 %}
14451
14452 ins_pipe(ialu_reg_shift);
14453 %}
14454
14455 instruct convL2I_reg(iRegINoSp dst, iRegL src) %{
14456 match(Set dst (ConvL2I src));
14457
14458 ins_cost(INSN_COST);
14459 format %{ "movw $dst, $src \t// l2i" %}
14460
14461 ins_encode %{
14462 __ movw(as_Register($dst$$reg), as_Register($src$$reg));
14463 %}
14464
14465 ins_pipe(ialu_reg);
14466 %}
14467
14468 instruct convI2B(iRegINoSp dst, iRegIorL2I src, rFlagsReg cr)
14469 %{
14470 match(Set dst (Conv2B src));
14471 effect(KILL cr);
14472
14473 format %{
14474 "cmpw $src, zr\n\t"
14475 "cset $dst, ne"
14476 %}
14477
14478 ins_encode %{
14479 __ cmpw(as_Register($src$$reg), zr);
14480 __ cset(as_Register($dst$$reg), Assembler::NE);
14481 %}
14482
14483 ins_pipe(ialu_reg);
14484 %}
14485
14486 instruct convP2B(iRegINoSp dst, iRegP src, rFlagsReg cr)
14487 %{
14488 match(Set dst (Conv2B src));
14489 effect(KILL cr);
14490
14491 format %{
14492 "cmp $src, zr\n\t"
14493 "cset $dst, ne"
14494 %}
14495
14496 ins_encode %{
14497 __ cmp(as_Register($src$$reg), zr);
14498 __ cset(as_Register($dst$$reg), Assembler::NE);
14499 %}
14500
14501 ins_pipe(ialu_reg);
14502 %}
14503
14504 instruct convD2F_reg(vRegF dst, vRegD src) %{
14505 match(Set dst (ConvD2F src));
14506
14507 ins_cost(INSN_COST * 5);
14508 format %{ "fcvtd $dst, $src \t// d2f" %}
14509
14510 ins_encode %{
14511 __ fcvtd(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg));
14512 %}
14513
14514 ins_pipe(fp_d2f);
14515 %}
14516
14517 instruct convF2D_reg(vRegD dst, vRegF src) %{
14518 match(Set dst (ConvF2D src));
14519
14520 ins_cost(INSN_COST * 5);
14521 format %{ "fcvts $dst, $src \t// f2d" %}
14522
14523 ins_encode %{
14524 __ fcvts(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg));
14525 %}
14526
14527 ins_pipe(fp_f2d);
14528 %}
14529
14530 instruct convF2I_reg_reg(iRegINoSp dst, vRegF src) %{
14531 match(Set dst (ConvF2I src));
14532
14533 ins_cost(INSN_COST * 5);
14534 format %{ "fcvtzsw $dst, $src \t// f2i" %}
14535
14536 ins_encode %{
14537 __ fcvtzsw(as_Register($dst$$reg), as_FloatRegister($src$$reg));
14538 %}
14539
14540 ins_pipe(fp_f2i);
14541 %}
14542
14543 instruct convF2L_reg_reg(iRegLNoSp dst, vRegF src) %{
14544 match(Set dst (ConvF2L src));
14545
14546 ins_cost(INSN_COST * 5);
14547 format %{ "fcvtzs $dst, $src \t// f2l" %}
14548
14549 ins_encode %{
14550 __ fcvtzs(as_Register($dst$$reg), as_FloatRegister($src$$reg));
14551 %}
14552
14553 ins_pipe(fp_f2l);
14554 %}
14555
14556 instruct convI2F_reg_reg(vRegF dst, iRegIorL2I src) %{
14557 match(Set dst (ConvI2F src));
14558
14559 ins_cost(INSN_COST * 5);
14560 format %{ "scvtfws $dst, $src \t// i2f" %}
14561
14562 ins_encode %{
14563 __ scvtfws(as_FloatRegister($dst$$reg), as_Register($src$$reg));
14564 %}
14565
14566 ins_pipe(fp_i2f);
14567 %}
14568
14569 instruct convL2F_reg_reg(vRegF dst, iRegL src) %{
14570 match(Set dst (ConvL2F src));
14571
14572 ins_cost(INSN_COST * 5);
14573 format %{ "scvtfs $dst, $src \t// l2f" %}
14574
14575 ins_encode %{
14576 __ scvtfs(as_FloatRegister($dst$$reg), as_Register($src$$reg));
14577 %}
14578
14579 ins_pipe(fp_l2f);
14580 %}
14581
14582 instruct convD2I_reg_reg(iRegINoSp dst, vRegD src) %{
14583 match(Set dst (ConvD2I src));
14584
14585 ins_cost(INSN_COST * 5);
14586 format %{ "fcvtzdw $dst, $src \t// d2i" %}
14587
14588 ins_encode %{
14589 __ fcvtzdw(as_Register($dst$$reg), as_FloatRegister($src$$reg));
14590 %}
14591
14592 ins_pipe(fp_d2i);
14593 %}
14594
14595 instruct convD2L_reg_reg(iRegLNoSp dst, vRegD src) %{
14596 match(Set dst (ConvD2L src));
14597
14598 ins_cost(INSN_COST * 5);
14599 format %{ "fcvtzd $dst, $src \t// d2l" %}
14600
14601 ins_encode %{
14602 __ fcvtzd(as_Register($dst$$reg), as_FloatRegister($src$$reg));
14603 %}
14604
14605 ins_pipe(fp_d2l);
14606 %}
14607
14608 instruct convI2D_reg_reg(vRegD dst, iRegIorL2I src) %{
14609 match(Set dst (ConvI2D src));
14610
14611 ins_cost(INSN_COST * 5);
14612 format %{ "scvtfwd $dst, $src \t// i2d" %}
14613
14614 ins_encode %{
14615 __ scvtfwd(as_FloatRegister($dst$$reg), as_Register($src$$reg));
14616 %}
14617
14618 ins_pipe(fp_i2d);
14619 %}
14620
14621 instruct convL2D_reg_reg(vRegD dst, iRegL src) %{
14622 match(Set dst (ConvL2D src));
14623
14624 ins_cost(INSN_COST * 5);
14625 format %{ "scvtfd $dst, $src \t// l2d" %}
14626
14627 ins_encode %{
14628 __ scvtfd(as_FloatRegister($dst$$reg), as_Register($src$$reg));
14629 %}
14630
14631 ins_pipe(fp_l2d);
14632 %}
14633
14634 instruct round_double_reg(iRegLNoSp dst, vRegD src, vRegD ftmp, rFlagsReg cr)
14635 %{
14636 match(Set dst (RoundD src));
14637 effect(TEMP_DEF dst, TEMP ftmp, KILL cr);
14638 format %{ "java_round_double $dst,$src"%}
14639 ins_encode %{
14640 __ java_round_double($dst$$Register, as_FloatRegister($src$$reg),
14641 as_FloatRegister($ftmp$$reg));
14642 %}
14643 ins_pipe(pipe_slow);
14644 %}
14645
14646 instruct round_float_reg(iRegINoSp dst, vRegF src, vRegF ftmp, rFlagsReg cr)
14647 %{
14648 match(Set dst (RoundF src));
14649 effect(TEMP_DEF dst, TEMP ftmp, KILL cr);
14650 format %{ "java_round_float $dst,$src"%}
14651 ins_encode %{
14652 __ java_round_float($dst$$Register, as_FloatRegister($src$$reg),
14653 as_FloatRegister($ftmp$$reg));
14654 %}
14655 ins_pipe(pipe_slow);
14656 %}
14657
14658 // stack <-> reg and reg <-> reg shuffles with no conversion
14659
14660 instruct MoveF2I_stack_reg(iRegINoSp dst, stackSlotF src) %{
14661
14662 match(Set dst (MoveF2I src));
14663
14664 effect(DEF dst, USE src);
14665
14666 ins_cost(4 * INSN_COST);
14667
14668 format %{ "ldrw $dst, $src\t# MoveF2I_stack_reg" %}
14669
14670 ins_encode %{
14671 __ ldrw($dst$$Register, Address(sp, $src$$disp));
14672 %}
14673
14674 ins_pipe(iload_reg_reg);
14675
14676 %}
14677
14678 instruct MoveI2F_stack_reg(vRegF dst, stackSlotI src) %{
14679
14680 match(Set dst (MoveI2F src));
14681
14682 effect(DEF dst, USE src);
14683
14684 ins_cost(4 * INSN_COST);
14685
14686 format %{ "ldrs $dst, $src\t# MoveI2F_stack_reg" %}
14687
14688 ins_encode %{
14689 __ ldrs(as_FloatRegister($dst$$reg), Address(sp, $src$$disp));
14690 %}
14691
14692 ins_pipe(pipe_class_memory);
14693
14694 %}
14695
14696 instruct MoveD2L_stack_reg(iRegLNoSp dst, stackSlotD src) %{
14697
14698 match(Set dst (MoveD2L src));
14699
14700 effect(DEF dst, USE src);
14701
14702 ins_cost(4 * INSN_COST);
14703
14704 format %{ "ldr $dst, $src\t# MoveD2L_stack_reg" %}
14705
14706 ins_encode %{
14707 __ ldr($dst$$Register, Address(sp, $src$$disp));
14708 %}
14709
14710 ins_pipe(iload_reg_reg);
14711
14712 %}
14713
14714 instruct MoveL2D_stack_reg(vRegD dst, stackSlotL src) %{
14715
14716 match(Set dst (MoveL2D src));
14717
14718 effect(DEF dst, USE src);
14719
14720 ins_cost(4 * INSN_COST);
14721
14722 format %{ "ldrd $dst, $src\t# MoveL2D_stack_reg" %}
14723
14724 ins_encode %{
14725 __ ldrd(as_FloatRegister($dst$$reg), Address(sp, $src$$disp));
14726 %}
14727
14728 ins_pipe(pipe_class_memory);
14729
14730 %}
14731
14732 instruct MoveF2I_reg_stack(stackSlotI dst, vRegF src) %{
14733
14734 match(Set dst (MoveF2I src));
14735
14736 effect(DEF dst, USE src);
14737
14738 ins_cost(INSN_COST);
14739
14740 format %{ "strs $src, $dst\t# MoveF2I_reg_stack" %}
14741
14742 ins_encode %{
14743 __ strs(as_FloatRegister($src$$reg), Address(sp, $dst$$disp));
14744 %}
14745
14746 ins_pipe(pipe_class_memory);
14747
14748 %}
14749
14750 instruct MoveI2F_reg_stack(stackSlotF dst, iRegI src) %{
14751
14752 match(Set dst (MoveI2F src));
14753
14754 effect(DEF dst, USE src);
14755
14756 ins_cost(INSN_COST);
14757
14758 format %{ "strw $src, $dst\t# MoveI2F_reg_stack" %}
14759
14760 ins_encode %{
14761 __ strw($src$$Register, Address(sp, $dst$$disp));
14762 %}
14763
14764 ins_pipe(istore_reg_reg);
14765
14766 %}
14767
14768 instruct MoveD2L_reg_stack(stackSlotL dst, vRegD src) %{
14769
14770 match(Set dst (MoveD2L src));
14771
14772 effect(DEF dst, USE src);
14773
14774 ins_cost(INSN_COST);
14775
14776 format %{ "strd $dst, $src\t# MoveD2L_reg_stack" %}
14777
14778 ins_encode %{
14779 __ strd(as_FloatRegister($src$$reg), Address(sp, $dst$$disp));
14780 %}
14781
14782 ins_pipe(pipe_class_memory);
14783
14784 %}
14785
14786 instruct MoveL2D_reg_stack(stackSlotD dst, iRegL src) %{
14787
14788 match(Set dst (MoveL2D src));
14789
14790 effect(DEF dst, USE src);
14791
14792 ins_cost(INSN_COST);
14793
14794 format %{ "str $src, $dst\t# MoveL2D_reg_stack" %}
14795
14796 ins_encode %{
14797 __ str($src$$Register, Address(sp, $dst$$disp));
14798 %}
14799
14800 ins_pipe(istore_reg_reg);
14801
14802 %}
14803
14804 instruct MoveF2I_reg_reg(iRegINoSp dst, vRegF src) %{
14805
14806 match(Set dst (MoveF2I src));
14807
14808 effect(DEF dst, USE src);
14809
14810 ins_cost(INSN_COST);
14811
14812 format %{ "fmovs $dst, $src\t# MoveF2I_reg_reg" %}
14813
14814 ins_encode %{
14815 __ fmovs($dst$$Register, as_FloatRegister($src$$reg));
14816 %}
14817
14818 ins_pipe(fp_f2i);
14819
14820 %}
14821
14822 instruct MoveI2F_reg_reg(vRegF dst, iRegI src) %{
14823
14824 match(Set dst (MoveI2F src));
14825
14826 effect(DEF dst, USE src);
14827
14828 ins_cost(INSN_COST);
14829
14830 format %{ "fmovs $dst, $src\t# MoveI2F_reg_reg" %}
14831
14832 ins_encode %{
14833 __ fmovs(as_FloatRegister($dst$$reg), $src$$Register);
14834 %}
14835
14836 ins_pipe(fp_i2f);
14837
14838 %}
14839
14840 instruct MoveD2L_reg_reg(iRegLNoSp dst, vRegD src) %{
14841
14842 match(Set dst (MoveD2L src));
14843
14844 effect(DEF dst, USE src);
14845
14846 ins_cost(INSN_COST);
14847
14848 format %{ "fmovd $dst, $src\t# MoveD2L_reg_reg" %}
14849
14850 ins_encode %{
14851 __ fmovd($dst$$Register, as_FloatRegister($src$$reg));
14852 %}
14853
14854 ins_pipe(fp_d2l);
14855
14856 %}
14857
14858 instruct MoveL2D_reg_reg(vRegD dst, iRegL src) %{
14859
14860 match(Set dst (MoveL2D src));
14861
14862 effect(DEF dst, USE src);
14863
14864 ins_cost(INSN_COST);
14865
14866 format %{ "fmovd $dst, $src\t# MoveL2D_reg_reg" %}
14867
14868 ins_encode %{
14869 __ fmovd(as_FloatRegister($dst$$reg), $src$$Register);
14870 %}
14871
14872 ins_pipe(fp_l2d);
14873
14874 %}
14875
14876 // ============================================================================
14877 // clearing of an array
14878
14879 instruct clearArray_reg_reg(iRegL_R11 cnt, iRegP_R10 base, Universe dummy, rFlagsReg cr)
14880 %{
14881 match(Set dummy (ClearArray cnt base));
14882 effect(USE_KILL cnt, USE_KILL base, KILL cr);
14883
14884 ins_cost(4 * INSN_COST);
14885 format %{ "ClearArray $cnt, $base" %}
14886
14887 ins_encode %{
14888 address tpc = __ zero_words($base$$Register, $cnt$$Register);
14889 if (tpc == NULL) {
14890 ciEnv::current()->record_failure("CodeCache is full");
14891 return;
14892 }
14893 %}
14894
14895 ins_pipe(pipe_class_memory);
14896 %}
14897
14898 instruct clearArray_imm_reg(immL cnt, iRegP_R10 base, iRegL_R11 temp, Universe dummy, rFlagsReg cr)
14899 %{
14900 predicate((uint64_t)n->in(2)->get_long()
14901 < (uint64_t)(BlockZeroingLowLimit >> LogBytesPerWord));
14902 match(Set dummy (ClearArray cnt base));
14903 effect(TEMP temp, USE_KILL base, KILL cr);
14904
14905 ins_cost(4 * INSN_COST);
14906 format %{ "ClearArray $cnt, $base" %}
14907
14908 ins_encode %{
14909 address tpc = __ zero_words($base$$Register, (uint64_t)$cnt$$constant);
14910 if (tpc == NULL) {
14911 ciEnv::current()->record_failure("CodeCache is full");
14912 return;
14913 }
14914 %}
14915
14916 ins_pipe(pipe_class_memory);
14917 %}
14918
14919 // ============================================================================
14920 // Overflow Math Instructions
14921
14922 instruct overflowAddI_reg_reg(rFlagsReg cr, iRegIorL2I op1, iRegIorL2I op2)
14923 %{
14924 match(Set cr (OverflowAddI op1 op2));
14925
14926 format %{ "cmnw $op1, $op2\t# overflow check int" %}
14927 ins_cost(INSN_COST);
14928 ins_encode %{
14929 __ cmnw($op1$$Register, $op2$$Register);
14930 %}
14931
14932 ins_pipe(icmp_reg_reg);
14933 %}
14934
14935 instruct overflowAddI_reg_imm(rFlagsReg cr, iRegIorL2I op1, immIAddSub op2)
14936 %{
14937 match(Set cr (OverflowAddI op1 op2));
14938
14939 format %{ "cmnw $op1, $op2\t# overflow check int" %}
14940 ins_cost(INSN_COST);
14941 ins_encode %{
14942 __ cmnw($op1$$Register, $op2$$constant);
14943 %}
14944
14945 ins_pipe(icmp_reg_imm);
14946 %}
14947
14948 instruct overflowAddL_reg_reg(rFlagsReg cr, iRegL op1, iRegL op2)
14949 %{
14950 match(Set cr (OverflowAddL op1 op2));
14951
14952 format %{ "cmn $op1, $op2\t# overflow check long" %}
14953 ins_cost(INSN_COST);
14954 ins_encode %{
14955 __ cmn($op1$$Register, $op2$$Register);
14956 %}
14957
14958 ins_pipe(icmp_reg_reg);
14959 %}
14960
14961 instruct overflowAddL_reg_imm(rFlagsReg cr, iRegL op1, immLAddSub op2)
14962 %{
14963 match(Set cr (OverflowAddL op1 op2));
14964
14965 format %{ "adds zr, $op1, $op2\t# overflow check long" %}
14966 ins_cost(INSN_COST);
14967 ins_encode %{
14968 __ adds(zr, $op1$$Register, $op2$$constant);
14969 %}
14970
14971 ins_pipe(icmp_reg_imm);
14972 %}
14973
14974 instruct overflowSubI_reg_reg(rFlagsReg cr, iRegIorL2I op1, iRegIorL2I op2)
14975 %{
14976 match(Set cr (OverflowSubI op1 op2));
14977
14978 format %{ "cmpw $op1, $op2\t# overflow check int" %}
14979 ins_cost(INSN_COST);
14980 ins_encode %{
14981 __ cmpw($op1$$Register, $op2$$Register);
14982 %}
14983
14984 ins_pipe(icmp_reg_reg);
14985 %}
14986
14987 instruct overflowSubI_reg_imm(rFlagsReg cr, iRegIorL2I op1, immIAddSub op2)
14988 %{
14989 match(Set cr (OverflowSubI op1 op2));
14990
14991 format %{ "cmpw $op1, $op2\t# overflow check int" %}
14992 ins_cost(INSN_COST);
14993 ins_encode %{
14994 __ cmpw($op1$$Register, $op2$$constant);
14995 %}
14996
14997 ins_pipe(icmp_reg_imm);
14998 %}
14999
15000 instruct overflowSubL_reg_reg(rFlagsReg cr, iRegL op1, iRegL op2)
15001 %{
15002 match(Set cr (OverflowSubL op1 op2));
15003
15004 format %{ "cmp $op1, $op2\t# overflow check long" %}
15005 ins_cost(INSN_COST);
15006 ins_encode %{
15007 __ cmp($op1$$Register, $op2$$Register);
15008 %}
15009
15010 ins_pipe(icmp_reg_reg);
15011 %}
15012
15013 instruct overflowSubL_reg_imm(rFlagsReg cr, iRegL op1, immLAddSub op2)
15014 %{
15015 match(Set cr (OverflowSubL op1 op2));
15016
15017 format %{ "cmp $op1, $op2\t# overflow check long" %}
15018 ins_cost(INSN_COST);
15019 ins_encode %{
15020 __ subs(zr, $op1$$Register, $op2$$constant);
15021 %}
15022
15023 ins_pipe(icmp_reg_imm);
15024 %}
15025
15026 instruct overflowNegI_reg(rFlagsReg cr, immI0 zero, iRegIorL2I op1)
15027 %{
15028 match(Set cr (OverflowSubI zero op1));
15029
15030 format %{ "cmpw zr, $op1\t# overflow check int" %}
15031 ins_cost(INSN_COST);
15032 ins_encode %{
15033 __ cmpw(zr, $op1$$Register);
15034 %}
15035
15036 ins_pipe(icmp_reg_imm);
15037 %}
15038
15039 instruct overflowNegL_reg(rFlagsReg cr, immI0 zero, iRegL op1)
15040 %{
15041 match(Set cr (OverflowSubL zero op1));
15042
15043 format %{ "cmp zr, $op1\t# overflow check long" %}
15044 ins_cost(INSN_COST);
15045 ins_encode %{
15046 __ cmp(zr, $op1$$Register);
15047 %}
15048
15049 ins_pipe(icmp_reg_imm);
15050 %}
15051
15052 instruct overflowMulI_reg(rFlagsReg cr, iRegIorL2I op1, iRegIorL2I op2)
15053 %{
15054 match(Set cr (OverflowMulI op1 op2));
15055
15056 format %{ "smull rscratch1, $op1, $op2\t# overflow check int\n\t"
15057 "cmp rscratch1, rscratch1, sxtw\n\t"
15058 "movw rscratch1, #0x80000000\n\t"
15059 "cselw rscratch1, rscratch1, zr, NE\n\t"
15060 "cmpw rscratch1, #1" %}
15061 ins_cost(5 * INSN_COST);
15062 ins_encode %{
15063 __ smull(rscratch1, $op1$$Register, $op2$$Register);
15064 __ subs(zr, rscratch1, rscratch1, ext::sxtw); // NE => overflow
15065 __ movw(rscratch1, 0x80000000); // Develop 0 (EQ),
15066 __ cselw(rscratch1, rscratch1, zr, Assembler::NE); // or 0x80000000 (NE)
15067 __ cmpw(rscratch1, 1); // 0x80000000 - 1 => VS
15068 %}
15069
15070 ins_pipe(pipe_slow);
15071 %}
15072
15073 instruct overflowMulI_reg_branch(cmpOp cmp, iRegIorL2I op1, iRegIorL2I op2, label labl, rFlagsReg cr)
15074 %{
15075 match(If cmp (OverflowMulI op1 op2));
15076 predicate(n->in(1)->as_Bool()->_test._test == BoolTest::overflow
15077 || n->in(1)->as_Bool()->_test._test == BoolTest::no_overflow);
15078 effect(USE labl, KILL cr);
15079
15080 format %{ "smull rscratch1, $op1, $op2\t# overflow check int\n\t"
15081 "cmp rscratch1, rscratch1, sxtw\n\t"
15082 "b$cmp $labl" %}
15083 ins_cost(3 * INSN_COST); // Branch is rare so treat as INSN_COST
15084 ins_encode %{
15085 Label* L = $labl$$label;
15086 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15087 __ smull(rscratch1, $op1$$Register, $op2$$Register);
15088 __ subs(zr, rscratch1, rscratch1, ext::sxtw); // NE => overflow
15089 __ br(cond == Assembler::VS ? Assembler::NE : Assembler::EQ, *L);
15090 %}
15091
15092 ins_pipe(pipe_serial);
15093 %}
15094
15095 instruct overflowMulL_reg(rFlagsReg cr, iRegL op1, iRegL op2)
15096 %{
15097 match(Set cr (OverflowMulL op1 op2));
15098
15099 format %{ "mul rscratch1, $op1, $op2\t#overflow check long\n\t"
15100 "smulh rscratch2, $op1, $op2\n\t"
15101 "cmp rscratch2, rscratch1, ASR #63\n\t"
15102 "movw rscratch1, #0x80000000\n\t"
15103 "cselw rscratch1, rscratch1, zr, NE\n\t"
15104 "cmpw rscratch1, #1" %}
15105 ins_cost(6 * INSN_COST);
15106 ins_encode %{
15107 __ mul(rscratch1, $op1$$Register, $op2$$Register); // Result bits 0..63
15108 __ smulh(rscratch2, $op1$$Register, $op2$$Register); // Result bits 64..127
15109 __ cmp(rscratch2, rscratch1, Assembler::ASR, 63); // Top is pure sign ext
15110 __ movw(rscratch1, 0x80000000); // Develop 0 (EQ),
15111 __ cselw(rscratch1, rscratch1, zr, Assembler::NE); // or 0x80000000 (NE)
15112 __ cmpw(rscratch1, 1); // 0x80000000 - 1 => VS
15113 %}
15114
15115 ins_pipe(pipe_slow);
15116 %}
15117
15118 instruct overflowMulL_reg_branch(cmpOp cmp, iRegL op1, iRegL op2, label labl, rFlagsReg cr)
15119 %{
15120 match(If cmp (OverflowMulL op1 op2));
15121 predicate(n->in(1)->as_Bool()->_test._test == BoolTest::overflow
15122 || n->in(1)->as_Bool()->_test._test == BoolTest::no_overflow);
15123 effect(USE labl, KILL cr);
15124
15125 format %{ "mul rscratch1, $op1, $op2\t#overflow check long\n\t"
15126 "smulh rscratch2, $op1, $op2\n\t"
15127 "cmp rscratch2, rscratch1, ASR #63\n\t"
15128 "b$cmp $labl" %}
15129 ins_cost(4 * INSN_COST); // Branch is rare so treat as INSN_COST
15130 ins_encode %{
15131 Label* L = $labl$$label;
15132 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15133 __ mul(rscratch1, $op1$$Register, $op2$$Register); // Result bits 0..63
15134 __ smulh(rscratch2, $op1$$Register, $op2$$Register); // Result bits 64..127
15135 __ cmp(rscratch2, rscratch1, Assembler::ASR, 63); // Top is pure sign ext
15136 __ br(cond == Assembler::VS ? Assembler::NE : Assembler::EQ, *L);
15137 %}
15138
15139 ins_pipe(pipe_serial);
15140 %}
15141
15142 // ============================================================================
15143 // Compare Instructions
15144
15145 instruct compI_reg_reg(rFlagsReg cr, iRegI op1, iRegI op2)
15146 %{
15147 match(Set cr (CmpI op1 op2));
15148
15149 effect(DEF cr, USE op1, USE op2);
15150
15151 ins_cost(INSN_COST);
15152 format %{ "cmpw $op1, $op2" %}
15153
15154 ins_encode(aarch64_enc_cmpw(op1, op2));
15155
15156 ins_pipe(icmp_reg_reg);
15157 %}
15158
15159 instruct compI_reg_immI0(rFlagsReg cr, iRegI op1, immI0 zero)
15160 %{
15161 match(Set cr (CmpI op1 zero));
15162
15163 effect(DEF cr, USE op1);
15164
15165 ins_cost(INSN_COST);
15166 format %{ "cmpw $op1, 0" %}
15167
15168 ins_encode(aarch64_enc_cmpw_imm_addsub(op1, zero));
15169
15170 ins_pipe(icmp_reg_imm);
15171 %}
15172
15173 instruct compI_reg_immIAddSub(rFlagsReg cr, iRegI op1, immIAddSub op2)
15174 %{
15175 match(Set cr (CmpI op1 op2));
15176
15177 effect(DEF cr, USE op1);
15178
15179 ins_cost(INSN_COST);
15180 format %{ "cmpw $op1, $op2" %}
15181
15182 ins_encode(aarch64_enc_cmpw_imm_addsub(op1, op2));
15183
15184 ins_pipe(icmp_reg_imm);
15185 %}
15186
15187 instruct compI_reg_immI(rFlagsReg cr, iRegI op1, immI op2)
15188 %{
15189 match(Set cr (CmpI op1 op2));
15190
15191 effect(DEF cr, USE op1);
15192
15193 ins_cost(INSN_COST * 2);
15194 format %{ "cmpw $op1, $op2" %}
15195
15196 ins_encode(aarch64_enc_cmpw_imm(op1, op2));
15197
15198 ins_pipe(icmp_reg_imm);
15199 %}
15200
15201 // Unsigned compare Instructions; really, same as signed compare
15202 // except it should only be used to feed an If or a CMovI which takes a
15203 // cmpOpU.
15204
15205 instruct compU_reg_reg(rFlagsRegU cr, iRegI op1, iRegI op2)
15206 %{
15207 match(Set cr (CmpU op1 op2));
15208
15209 effect(DEF cr, USE op1, USE op2);
15210
15211 ins_cost(INSN_COST);
15212 format %{ "cmpw $op1, $op2\t# unsigned" %}
15213
15214 ins_encode(aarch64_enc_cmpw(op1, op2));
15215
15216 ins_pipe(icmp_reg_reg);
15217 %}
15218
15219 instruct compU_reg_immI0(rFlagsRegU cr, iRegI op1, immI0 zero)
15220 %{
15221 match(Set cr (CmpU op1 zero));
15222
15223 effect(DEF cr, USE op1);
15224
15225 ins_cost(INSN_COST);
15226 format %{ "cmpw $op1, #0\t# unsigned" %}
15227
15228 ins_encode(aarch64_enc_cmpw_imm_addsub(op1, zero));
15229
15230 ins_pipe(icmp_reg_imm);
15231 %}
15232
15233 instruct compU_reg_immIAddSub(rFlagsRegU cr, iRegI op1, immIAddSub op2)
15234 %{
15235 match(Set cr (CmpU op1 op2));
15236
15237 effect(DEF cr, USE op1);
15238
15239 ins_cost(INSN_COST);
15240 format %{ "cmpw $op1, $op2\t# unsigned" %}
15241
15242 ins_encode(aarch64_enc_cmpw_imm_addsub(op1, op2));
15243
15244 ins_pipe(icmp_reg_imm);
15245 %}
15246
15247 instruct compU_reg_immI(rFlagsRegU cr, iRegI op1, immI op2)
15248 %{
15249 match(Set cr (CmpU op1 op2));
15250
15251 effect(DEF cr, USE op1);
15252
15253 ins_cost(INSN_COST * 2);
15254 format %{ "cmpw $op1, $op2\t# unsigned" %}
15255
15256 ins_encode(aarch64_enc_cmpw_imm(op1, op2));
15257
15258 ins_pipe(icmp_reg_imm);
15259 %}
15260
15261 instruct compL_reg_reg(rFlagsReg cr, iRegL op1, iRegL op2)
15262 %{
15263 match(Set cr (CmpL op1 op2));
15264
15265 effect(DEF cr, USE op1, USE op2);
15266
15267 ins_cost(INSN_COST);
15268 format %{ "cmp $op1, $op2" %}
15269
15270 ins_encode(aarch64_enc_cmp(op1, op2));
15271
15272 ins_pipe(icmp_reg_reg);
15273 %}
15274
15275 instruct compL_reg_immL0(rFlagsReg cr, iRegL op1, immL0 zero)
15276 %{
15277 match(Set cr (CmpL op1 zero));
15278
15279 effect(DEF cr, USE op1);
15280
15281 ins_cost(INSN_COST);
15282 format %{ "tst $op1" %}
15283
15284 ins_encode(aarch64_enc_cmp_imm_addsub(op1, zero));
15285
15286 ins_pipe(icmp_reg_imm);
15287 %}
15288
15289 instruct compL_reg_immLAddSub(rFlagsReg cr, iRegL op1, immLAddSub op2)
15290 %{
15291 match(Set cr (CmpL op1 op2));
15292
15293 effect(DEF cr, USE op1);
15294
15295 ins_cost(INSN_COST);
15296 format %{ "cmp $op1, $op2" %}
15297
15298 ins_encode(aarch64_enc_cmp_imm_addsub(op1, op2));
15299
15300 ins_pipe(icmp_reg_imm);
15301 %}
15302
15303 instruct compL_reg_immL(rFlagsReg cr, iRegL op1, immL op2)
15304 %{
15305 match(Set cr (CmpL op1 op2));
15306
15307 effect(DEF cr, USE op1);
15308
15309 ins_cost(INSN_COST * 2);
15310 format %{ "cmp $op1, $op2" %}
15311
15312 ins_encode(aarch64_enc_cmp_imm(op1, op2));
15313
15314 ins_pipe(icmp_reg_imm);
15315 %}
15316
15317 instruct compUL_reg_reg(rFlagsRegU cr, iRegL op1, iRegL op2)
15318 %{
15319 match(Set cr (CmpUL op1 op2));
15320
15321 effect(DEF cr, USE op1, USE op2);
15322
15323 ins_cost(INSN_COST);
15324 format %{ "cmp $op1, $op2" %}
15325
15326 ins_encode(aarch64_enc_cmp(op1, op2));
15327
15328 ins_pipe(icmp_reg_reg);
15329 %}
15330
15331 instruct compUL_reg_immL0(rFlagsRegU cr, iRegL op1, immL0 zero)
15332 %{
15333 match(Set cr (CmpUL op1 zero));
15334
15335 effect(DEF cr, USE op1);
15336
15337 ins_cost(INSN_COST);
15338 format %{ "tst $op1" %}
15339
15340 ins_encode(aarch64_enc_cmp_imm_addsub(op1, zero));
15341
15342 ins_pipe(icmp_reg_imm);
15343 %}
15344
15345 instruct compUL_reg_immLAddSub(rFlagsRegU cr, iRegL op1, immLAddSub op2)
15346 %{
15347 match(Set cr (CmpUL op1 op2));
15348
15349 effect(DEF cr, USE op1);
15350
15351 ins_cost(INSN_COST);
15352 format %{ "cmp $op1, $op2" %}
15353
15354 ins_encode(aarch64_enc_cmp_imm_addsub(op1, op2));
15355
15356 ins_pipe(icmp_reg_imm);
15357 %}
15358
15359 instruct compUL_reg_immL(rFlagsRegU cr, iRegL op1, immL op2)
15360 %{
15361 match(Set cr (CmpUL op1 op2));
15362
15363 effect(DEF cr, USE op1);
15364
15365 ins_cost(INSN_COST * 2);
15366 format %{ "cmp $op1, $op2" %}
15367
15368 ins_encode(aarch64_enc_cmp_imm(op1, op2));
15369
15370 ins_pipe(icmp_reg_imm);
15371 %}
15372
15373 instruct compP_reg_reg(rFlagsRegU cr, iRegP op1, iRegP op2)
15374 %{
15375 match(Set cr (CmpP op1 op2));
15376
15377 effect(DEF cr, USE op1, USE op2);
15378
15379 ins_cost(INSN_COST);
15380 format %{ "cmp $op1, $op2\t // ptr" %}
15381
15382 ins_encode(aarch64_enc_cmpp(op1, op2));
15383
15384 ins_pipe(icmp_reg_reg);
15385 %}
15386
15387 instruct compN_reg_reg(rFlagsRegU cr, iRegN op1, iRegN op2)
15388 %{
15389 match(Set cr (CmpN op1 op2));
15390
15391 effect(DEF cr, USE op1, USE op2);
15392
15393 ins_cost(INSN_COST);
15394 format %{ "cmp $op1, $op2\t // compressed ptr" %}
15395
15396 ins_encode(aarch64_enc_cmpn(op1, op2));
15397
15398 ins_pipe(icmp_reg_reg);
15399 %}
15400
15401 instruct testP_reg(rFlagsRegU cr, iRegP op1, immP0 zero)
15402 %{
15403 match(Set cr (CmpP op1 zero));
15404
15405 effect(DEF cr, USE op1, USE zero);
15406
15407 ins_cost(INSN_COST);
15408 format %{ "cmp $op1, 0\t // ptr" %}
15409
15410 ins_encode(aarch64_enc_testp(op1));
15411
15412 ins_pipe(icmp_reg_imm);
15413 %}
15414
15415 instruct testN_reg(rFlagsRegU cr, iRegN op1, immN0 zero)
15416 %{
15417 match(Set cr (CmpN op1 zero));
15418
15419 effect(DEF cr, USE op1, USE zero);
15420
15421 ins_cost(INSN_COST);
15422 format %{ "cmp $op1, 0\t // compressed ptr" %}
15423
15424 ins_encode(aarch64_enc_testn(op1));
15425
15426 ins_pipe(icmp_reg_imm);
15427 %}
15428
15429 // FP comparisons
15430 //
15431 // n.b. CmpF/CmpD set a normal flags reg which then gets compared
15432 // using normal cmpOp. See declaration of rFlagsReg for details.
15433
15434 instruct compF_reg_reg(rFlagsReg cr, vRegF src1, vRegF src2)
15435 %{
15436 match(Set cr (CmpF src1 src2));
15437
15438 ins_cost(3 * INSN_COST);
15439 format %{ "fcmps $src1, $src2" %}
15440
15441 ins_encode %{
15442 __ fcmps(as_FloatRegister($src1$$reg), as_FloatRegister($src2$$reg));
15443 %}
15444
15445 ins_pipe(pipe_class_compare);
15446 %}
15447
15448 instruct compF_reg_zero(rFlagsReg cr, vRegF src1, immF0 src2)
15449 %{
15450 match(Set cr (CmpF src1 src2));
15451
15452 ins_cost(3 * INSN_COST);
15453 format %{ "fcmps $src1, 0.0" %}
15454
15455 ins_encode %{
15456 __ fcmps(as_FloatRegister($src1$$reg), 0.0);
15457 %}
15458
15459 ins_pipe(pipe_class_compare);
15460 %}
15461 // FROM HERE
15462
15463 instruct compD_reg_reg(rFlagsReg cr, vRegD src1, vRegD src2)
15464 %{
15465 match(Set cr (CmpD src1 src2));
15466
15467 ins_cost(3 * INSN_COST);
15468 format %{ "fcmpd $src1, $src2" %}
15469
15470 ins_encode %{
15471 __ fcmpd(as_FloatRegister($src1$$reg), as_FloatRegister($src2$$reg));
15472 %}
15473
15474 ins_pipe(pipe_class_compare);
15475 %}
15476
15477 instruct compD_reg_zero(rFlagsReg cr, vRegD src1, immD0 src2)
15478 %{
15479 match(Set cr (CmpD src1 src2));
15480
15481 ins_cost(3 * INSN_COST);
15482 format %{ "fcmpd $src1, 0.0" %}
15483
15484 ins_encode %{
15485 __ fcmpd(as_FloatRegister($src1$$reg), 0.0);
15486 %}
15487
15488 ins_pipe(pipe_class_compare);
15489 %}
15490
15491 instruct compF3_reg_reg(iRegINoSp dst, vRegF src1, vRegF src2, rFlagsReg cr)
15492 %{
15493 match(Set dst (CmpF3 src1 src2));
15494 effect(KILL cr);
15495
15496 ins_cost(5 * INSN_COST);
15497 format %{ "fcmps $src1, $src2\n\t"
15498 "csinvw($dst, zr, zr, eq\n\t"
15499 "csnegw($dst, $dst, $dst, lt)"
15500 %}
15501
15502 ins_encode %{
15503 Label done;
15504 FloatRegister s1 = as_FloatRegister($src1$$reg);
15505 FloatRegister s2 = as_FloatRegister($src2$$reg);
15506 Register d = as_Register($dst$$reg);
15507 __ fcmps(s1, s2);
15508 // installs 0 if EQ else -1
15509 __ csinvw(d, zr, zr, Assembler::EQ);
15510 // keeps -1 if less or unordered else installs 1
15511 __ csnegw(d, d, d, Assembler::LT);
15512 __ bind(done);
15513 %}
15514
15515 ins_pipe(pipe_class_default);
15516
15517 %}
15518
15519 instruct compD3_reg_reg(iRegINoSp dst, vRegD src1, vRegD src2, rFlagsReg cr)
15520 %{
15521 match(Set dst (CmpD3 src1 src2));
15522 effect(KILL cr);
15523
15524 ins_cost(5 * INSN_COST);
15525 format %{ "fcmpd $src1, $src2\n\t"
15526 "csinvw($dst, zr, zr, eq\n\t"
15527 "csnegw($dst, $dst, $dst, lt)"
15528 %}
15529
15530 ins_encode %{
15531 Label done;
15532 FloatRegister s1 = as_FloatRegister($src1$$reg);
15533 FloatRegister s2 = as_FloatRegister($src2$$reg);
15534 Register d = as_Register($dst$$reg);
15535 __ fcmpd(s1, s2);
15536 // installs 0 if EQ else -1
15537 __ csinvw(d, zr, zr, Assembler::EQ);
15538 // keeps -1 if less or unordered else installs 1
15539 __ csnegw(d, d, d, Assembler::LT);
15540 __ bind(done);
15541 %}
15542 ins_pipe(pipe_class_default);
15543
15544 %}
15545
15546 instruct compF3_reg_immF0(iRegINoSp dst, vRegF src1, immF0 zero, rFlagsReg cr)
15547 %{
15548 match(Set dst (CmpF3 src1 zero));
15549 effect(KILL cr);
15550
15551 ins_cost(5 * INSN_COST);
15552 format %{ "fcmps $src1, 0.0\n\t"
15553 "csinvw($dst, zr, zr, eq\n\t"
15554 "csnegw($dst, $dst, $dst, lt)"
15555 %}
15556
15557 ins_encode %{
15558 Label done;
15559 FloatRegister s1 = as_FloatRegister($src1$$reg);
15560 Register d = as_Register($dst$$reg);
15561 __ fcmps(s1, 0.0);
15562 // installs 0 if EQ else -1
15563 __ csinvw(d, zr, zr, Assembler::EQ);
15564 // keeps -1 if less or unordered else installs 1
15565 __ csnegw(d, d, d, Assembler::LT);
15566 __ bind(done);
15567 %}
15568
15569 ins_pipe(pipe_class_default);
15570
15571 %}
15572
15573 instruct compD3_reg_immD0(iRegINoSp dst, vRegD src1, immD0 zero, rFlagsReg cr)
15574 %{
15575 match(Set dst (CmpD3 src1 zero));
15576 effect(KILL cr);
15577
15578 ins_cost(5 * INSN_COST);
15579 format %{ "fcmpd $src1, 0.0\n\t"
15580 "csinvw($dst, zr, zr, eq\n\t"
15581 "csnegw($dst, $dst, $dst, lt)"
15582 %}
15583
15584 ins_encode %{
15585 Label done;
15586 FloatRegister s1 = as_FloatRegister($src1$$reg);
15587 Register d = as_Register($dst$$reg);
15588 __ fcmpd(s1, 0.0);
15589 // installs 0 if EQ else -1
15590 __ csinvw(d, zr, zr, Assembler::EQ);
15591 // keeps -1 if less or unordered else installs 1
15592 __ csnegw(d, d, d, Assembler::LT);
15593 __ bind(done);
15594 %}
15595 ins_pipe(pipe_class_default);
15596
15597 %}
15598
15599 instruct cmpLTMask_reg_reg(iRegINoSp dst, iRegIorL2I p, iRegIorL2I q, rFlagsReg cr)
15600 %{
15601 match(Set dst (CmpLTMask p q));
15602 effect(KILL cr);
15603
15604 ins_cost(3 * INSN_COST);
15605
15606 format %{ "cmpw $p, $q\t# cmpLTMask\n\t"
15607 "csetw $dst, lt\n\t"
15608 "subw $dst, zr, $dst"
15609 %}
15610
15611 ins_encode %{
15612 __ cmpw(as_Register($p$$reg), as_Register($q$$reg));
15613 __ csetw(as_Register($dst$$reg), Assembler::LT);
15614 __ subw(as_Register($dst$$reg), zr, as_Register($dst$$reg));
15615 %}
15616
15617 ins_pipe(ialu_reg_reg);
15618 %}
15619
15620 instruct cmpLTMask_reg_zero(iRegINoSp dst, iRegIorL2I src, immI0 zero, rFlagsReg cr)
15621 %{
15622 match(Set dst (CmpLTMask src zero));
15623 effect(KILL cr);
15624
15625 ins_cost(INSN_COST);
15626
15627 format %{ "asrw $dst, $src, #31\t# cmpLTMask0" %}
15628
15629 ins_encode %{
15630 __ asrw(as_Register($dst$$reg), as_Register($src$$reg), 31);
15631 %}
15632
15633 ins_pipe(ialu_reg_shift);
15634 %}
15635
15636 // ============================================================================
15637 // Max and Min
15638
15639 instruct cmovI_reg_reg_lt(iRegINoSp dst, iRegI src1, iRegI src2, rFlagsReg cr)
15640 %{
15641 effect( DEF dst, USE src1, USE src2, USE cr );
15642
15643 ins_cost(INSN_COST * 2);
15644 format %{ "cselw $dst, $src1, $src2 lt\t" %}
15645
15646 ins_encode %{
15647 __ cselw(as_Register($dst$$reg),
15648 as_Register($src1$$reg),
15649 as_Register($src2$$reg),
15650 Assembler::LT);
15651 %}
15652
15653 ins_pipe(icond_reg_reg);
15654 %}
15655
15656 instruct minI_rReg(iRegINoSp dst, iRegI src1, iRegI src2)
15657 %{
15658 match(Set dst (MinI src1 src2));
15659 ins_cost(INSN_COST * 3);
15660
15661 expand %{
15662 rFlagsReg cr;
15663 compI_reg_reg(cr, src1, src2);
15664 cmovI_reg_reg_lt(dst, src1, src2, cr);
15665 %}
15666
15667 %}
15668 // FROM HERE
15669
15670 instruct cmovI_reg_reg_gt(iRegINoSp dst, iRegI src1, iRegI src2, rFlagsReg cr)
15671 %{
15672 effect( DEF dst, USE src1, USE src2, USE cr );
15673
15674 ins_cost(INSN_COST * 2);
15675 format %{ "cselw $dst, $src1, $src2 gt\t" %}
15676
15677 ins_encode %{
15678 __ cselw(as_Register($dst$$reg),
15679 as_Register($src1$$reg),
15680 as_Register($src2$$reg),
15681 Assembler::GT);
15682 %}
15683
15684 ins_pipe(icond_reg_reg);
15685 %}
15686
15687 instruct maxI_rReg(iRegINoSp dst, iRegI src1, iRegI src2)
15688 %{
15689 match(Set dst (MaxI src1 src2));
15690 ins_cost(INSN_COST * 3);
15691 expand %{
15692 rFlagsReg cr;
15693 compI_reg_reg(cr, src1, src2);
15694 cmovI_reg_reg_gt(dst, src1, src2, cr);
15695 %}
15696 %}
15697
15698 // ============================================================================
15699 // Branch Instructions
15700
15701 // Direct Branch.
15702 instruct branch(label lbl)
15703 %{
15704 match(Goto);
15705
15706 effect(USE lbl);
15707
15708 ins_cost(BRANCH_COST);
15709 format %{ "b $lbl" %}
15710
15711 ins_encode(aarch64_enc_b(lbl));
15712
15713 ins_pipe(pipe_branch);
15714 %}
15715
15716 // Conditional Near Branch
15717 instruct branchCon(cmpOp cmp, rFlagsReg cr, label lbl)
15718 %{
15719 // Same match rule as `branchConFar'.
15720 match(If cmp cr);
15721
15722 effect(USE lbl);
15723
15724 ins_cost(BRANCH_COST);
15725 // If set to 1 this indicates that the current instruction is a
15726 // short variant of a long branch. This avoids using this
15727 // instruction in first-pass matching. It will then only be used in
15728 // the `Shorten_branches' pass.
15729 // ins_short_branch(1);
15730 format %{ "b$cmp $lbl" %}
15731
15732 ins_encode(aarch64_enc_br_con(cmp, lbl));
15733
15734 ins_pipe(pipe_branch_cond);
15735 %}
15736
15737 // Conditional Near Branch Unsigned
15738 instruct branchConU(cmpOpU cmp, rFlagsRegU cr, label lbl)
15739 %{
15740 // Same match rule as `branchConFar'.
15741 match(If cmp cr);
15742
15743 effect(USE lbl);
15744
15745 ins_cost(BRANCH_COST);
15746 // If set to 1 this indicates that the current instruction is a
15747 // short variant of a long branch. This avoids using this
15748 // instruction in first-pass matching. It will then only be used in
15749 // the `Shorten_branches' pass.
15750 // ins_short_branch(1);
15751 format %{ "b$cmp $lbl\t# unsigned" %}
15752
15753 ins_encode(aarch64_enc_br_conU(cmp, lbl));
15754
15755 ins_pipe(pipe_branch_cond);
15756 %}
15757
15758 // Make use of CBZ and CBNZ. These instructions, as well as being
15759 // shorter than (cmp; branch), have the additional benefit of not
15760 // killing the flags.
15761
15762 instruct cmpI_imm0_branch(cmpOpEqNe cmp, iRegIorL2I op1, immI0 op2, label labl, rFlagsReg cr) %{
15763 match(If cmp (CmpI op1 op2));
15764 effect(USE labl);
15765
15766 ins_cost(BRANCH_COST);
15767 format %{ "cbw$cmp $op1, $labl" %}
15768 ins_encode %{
15769 Label* L = $labl$$label;
15770 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15771 if (cond == Assembler::EQ)
15772 __ cbzw($op1$$Register, *L);
15773 else
15774 __ cbnzw($op1$$Register, *L);
15775 %}
15776 ins_pipe(pipe_cmp_branch);
15777 %}
15778
15779 instruct cmpL_imm0_branch(cmpOpEqNe cmp, iRegL op1, immL0 op2, label labl, rFlagsReg cr) %{
15780 match(If cmp (CmpL op1 op2));
15781 effect(USE labl);
15782
15783 ins_cost(BRANCH_COST);
15784 format %{ "cb$cmp $op1, $labl" %}
15785 ins_encode %{
15786 Label* L = $labl$$label;
15787 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15788 if (cond == Assembler::EQ)
15789 __ cbz($op1$$Register, *L);
15790 else
15791 __ cbnz($op1$$Register, *L);
15792 %}
15793 ins_pipe(pipe_cmp_branch);
15794 %}
15795
15796 instruct cmpP_imm0_branch(cmpOpEqNe cmp, iRegP op1, immP0 op2, label labl, rFlagsReg cr) %{
15797 match(If cmp (CmpP op1 op2));
15798 effect(USE labl);
15799
15800 ins_cost(BRANCH_COST);
15801 format %{ "cb$cmp $op1, $labl" %}
15802 ins_encode %{
15803 Label* L = $labl$$label;
15804 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15805 if (cond == Assembler::EQ)
15806 __ cbz($op1$$Register, *L);
15807 else
15808 __ cbnz($op1$$Register, *L);
15809 %}
15810 ins_pipe(pipe_cmp_branch);
15811 %}
15812
15813 instruct cmpN_imm0_branch(cmpOpEqNe cmp, iRegN op1, immN0 op2, label labl, rFlagsReg cr) %{
15814 match(If cmp (CmpN op1 op2));
15815 effect(USE labl);
15816
15817 ins_cost(BRANCH_COST);
15818 format %{ "cbw$cmp $op1, $labl" %}
15819 ins_encode %{
15820 Label* L = $labl$$label;
15821 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15822 if (cond == Assembler::EQ)
15823 __ cbzw($op1$$Register, *L);
15824 else
15825 __ cbnzw($op1$$Register, *L);
15826 %}
15827 ins_pipe(pipe_cmp_branch);
15828 %}
15829
15830 instruct cmpP_narrowOop_imm0_branch(cmpOpEqNe cmp, iRegN oop, immP0 zero, label labl, rFlagsReg cr) %{
15831 match(If cmp (CmpP (DecodeN oop) zero));
15832 effect(USE labl);
15833
15834 ins_cost(BRANCH_COST);
15835 format %{ "cb$cmp $oop, $labl" %}
15836 ins_encode %{
15837 Label* L = $labl$$label;
15838 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15839 if (cond == Assembler::EQ)
15840 __ cbzw($oop$$Register, *L);
15841 else
15842 __ cbnzw($oop$$Register, *L);
15843 %}
15844 ins_pipe(pipe_cmp_branch);
15845 %}
15846
15847 instruct cmpUI_imm0_branch(cmpOpUEqNeLtGe cmp, iRegIorL2I op1, immI0 op2, label labl, rFlagsRegU cr) %{
15848 match(If cmp (CmpU op1 op2));
15849 effect(USE labl);
15850
15851 ins_cost(BRANCH_COST);
15852 format %{ "cbw$cmp $op1, $labl" %}
15853 ins_encode %{
15854 Label* L = $labl$$label;
15855 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15856 if (cond == Assembler::EQ || cond == Assembler::LS)
15857 __ cbzw($op1$$Register, *L);
15858 else
15859 __ cbnzw($op1$$Register, *L);
15860 %}
15861 ins_pipe(pipe_cmp_branch);
15862 %}
15863
15864 instruct cmpUL_imm0_branch(cmpOpUEqNeLtGe cmp, iRegL op1, immL0 op2, label labl, rFlagsRegU cr) %{
15865 match(If cmp (CmpUL op1 op2));
15866 effect(USE labl);
15867
15868 ins_cost(BRANCH_COST);
15869 format %{ "cb$cmp $op1, $labl" %}
15870 ins_encode %{
15871 Label* L = $labl$$label;
15872 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15873 if (cond == Assembler::EQ || cond == Assembler::LS)
15874 __ cbz($op1$$Register, *L);
15875 else
15876 __ cbnz($op1$$Register, *L);
15877 %}
15878 ins_pipe(pipe_cmp_branch);
15879 %}
15880
15881 // Test bit and Branch
15882
15883 // Patterns for short (< 32KiB) variants
15884 instruct cmpL_branch_sign(cmpOpLtGe cmp, iRegL op1, immL0 op2, label labl) %{
15885 match(If cmp (CmpL op1 op2));
15886 effect(USE labl);
15887
15888 ins_cost(BRANCH_COST);
15889 format %{ "cb$cmp $op1, $labl # long" %}
15890 ins_encode %{
15891 Label* L = $labl$$label;
15892 Assembler::Condition cond =
15893 ((Assembler::Condition)$cmp$$cmpcode == Assembler::LT) ? Assembler::NE : Assembler::EQ;
15894 __ tbr(cond, $op1$$Register, 63, *L);
15895 %}
15896 ins_pipe(pipe_cmp_branch);
15897 ins_short_branch(1);
15898 %}
15899
15900 instruct cmpI_branch_sign(cmpOpLtGe cmp, iRegIorL2I op1, immI0 op2, label labl) %{
15901 match(If cmp (CmpI op1 op2));
15902 effect(USE labl);
15903
15904 ins_cost(BRANCH_COST);
15905 format %{ "cb$cmp $op1, $labl # int" %}
15906 ins_encode %{
15907 Label* L = $labl$$label;
15908 Assembler::Condition cond =
15909 ((Assembler::Condition)$cmp$$cmpcode == Assembler::LT) ? Assembler::NE : Assembler::EQ;
15910 __ tbr(cond, $op1$$Register, 31, *L);
15911 %}
15912 ins_pipe(pipe_cmp_branch);
15913 ins_short_branch(1);
15914 %}
15915
15916 instruct cmpL_branch_bit(cmpOpEqNe cmp, iRegL op1, immL op2, immL0 op3, label labl) %{
15917 match(If cmp (CmpL (AndL op1 op2) op3));
15918 predicate(is_power_of_2((julong)n->in(2)->in(1)->in(2)->get_long()));
15919 effect(USE labl);
15920
15921 ins_cost(BRANCH_COST);
15922 format %{ "tb$cmp $op1, $op2, $labl" %}
15923 ins_encode %{
15924 Label* L = $labl$$label;
15925 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15926 int bit = exact_log2_long($op2$$constant);
15927 __ tbr(cond, $op1$$Register, bit, *L);
15928 %}
15929 ins_pipe(pipe_cmp_branch);
15930 ins_short_branch(1);
15931 %}
15932
15933 instruct cmpI_branch_bit(cmpOpEqNe cmp, iRegIorL2I op1, immI op2, immI0 op3, label labl) %{
15934 match(If cmp (CmpI (AndI op1 op2) op3));
15935 predicate(is_power_of_2((juint)n->in(2)->in(1)->in(2)->get_int()));
15936 effect(USE labl);
15937
15938 ins_cost(BRANCH_COST);
15939 format %{ "tb$cmp $op1, $op2, $labl" %}
15940 ins_encode %{
15941 Label* L = $labl$$label;
15942 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15943 int bit = exact_log2((juint)$op2$$constant);
15944 __ tbr(cond, $op1$$Register, bit, *L);
15945 %}
15946 ins_pipe(pipe_cmp_branch);
15947 ins_short_branch(1);
15948 %}
15949
15950 // And far variants
15951 instruct far_cmpL_branch_sign(cmpOpLtGe cmp, iRegL op1, immL0 op2, label labl) %{
15952 match(If cmp (CmpL op1 op2));
15953 effect(USE labl);
15954
15955 ins_cost(BRANCH_COST);
15956 format %{ "cb$cmp $op1, $labl # long" %}
15957 ins_encode %{
15958 Label* L = $labl$$label;
15959 Assembler::Condition cond =
15960 ((Assembler::Condition)$cmp$$cmpcode == Assembler::LT) ? Assembler::NE : Assembler::EQ;
15961 __ tbr(cond, $op1$$Register, 63, *L, /*far*/true);
15962 %}
15963 ins_pipe(pipe_cmp_branch);
15964 %}
15965
15966 instruct far_cmpI_branch_sign(cmpOpLtGe cmp, iRegIorL2I op1, immI0 op2, label labl) %{
15967 match(If cmp (CmpI op1 op2));
15968 effect(USE labl);
15969
15970 ins_cost(BRANCH_COST);
15971 format %{ "cb$cmp $op1, $labl # int" %}
15972 ins_encode %{
15973 Label* L = $labl$$label;
15974 Assembler::Condition cond =
15975 ((Assembler::Condition)$cmp$$cmpcode == Assembler::LT) ? Assembler::NE : Assembler::EQ;
15976 __ tbr(cond, $op1$$Register, 31, *L, /*far*/true);
15977 %}
15978 ins_pipe(pipe_cmp_branch);
15979 %}
15980
15981 instruct far_cmpL_branch_bit(cmpOpEqNe cmp, iRegL op1, immL op2, immL0 op3, label labl) %{
15982 match(If cmp (CmpL (AndL op1 op2) op3));
15983 predicate(is_power_of_2((julong)n->in(2)->in(1)->in(2)->get_long()));
15984 effect(USE labl);
15985
15986 ins_cost(BRANCH_COST);
15987 format %{ "tb$cmp $op1, $op2, $labl" %}
15988 ins_encode %{
15989 Label* L = $labl$$label;
15990 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15991 int bit = exact_log2_long($op2$$constant);
15992 __ tbr(cond, $op1$$Register, bit, *L, /*far*/true);
15993 %}
15994 ins_pipe(pipe_cmp_branch);
15995 %}
15996
15997 instruct far_cmpI_branch_bit(cmpOpEqNe cmp, iRegIorL2I op1, immI op2, immI0 op3, label labl) %{
15998 match(If cmp (CmpI (AndI op1 op2) op3));
15999 predicate(is_power_of_2((juint)n->in(2)->in(1)->in(2)->get_int()));
16000 effect(USE labl);
16001
16002 ins_cost(BRANCH_COST);
16003 format %{ "tb$cmp $op1, $op2, $labl" %}
16004 ins_encode %{
16005 Label* L = $labl$$label;
16006 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
16007 int bit = exact_log2((juint)$op2$$constant);
16008 __ tbr(cond, $op1$$Register, bit, *L, /*far*/true);
16009 %}
16010 ins_pipe(pipe_cmp_branch);
16011 %}
16012
16013 // Test bits
16014
16015 instruct cmpL_and(cmpOp cmp, iRegL op1, immL op2, immL0 op3, rFlagsReg cr) %{
16016 match(Set cr (CmpL (AndL op1 op2) op3));
16017 predicate(Assembler::operand_valid_for_logical_immediate
16018 (/*is_32*/false, n->in(1)->in(2)->get_long()));
16019
16020 ins_cost(INSN_COST);
16021 format %{ "tst $op1, $op2 # long" %}
16022 ins_encode %{
16023 __ tst($op1$$Register, $op2$$constant);
16024 %}
16025 ins_pipe(ialu_reg_reg);
16026 %}
16027
16028 instruct cmpI_and(cmpOp cmp, iRegIorL2I op1, immI op2, immI0 op3, rFlagsReg cr) %{
16029 match(Set cr (CmpI (AndI op1 op2) op3));
16030 predicate(Assembler::operand_valid_for_logical_immediate
16031 (/*is_32*/true, n->in(1)->in(2)->get_int()));
16032
16033 ins_cost(INSN_COST);
16034 format %{ "tst $op1, $op2 # int" %}
16035 ins_encode %{
16036 __ tstw($op1$$Register, $op2$$constant);
16037 %}
16038 ins_pipe(ialu_reg_reg);
16039 %}
16040
16041 instruct cmpL_and_reg(cmpOp cmp, iRegL op1, iRegL op2, immL0 op3, rFlagsReg cr) %{
16042 match(Set cr (CmpL (AndL op1 op2) op3));
16043
16044 ins_cost(INSN_COST);
16045 format %{ "tst $op1, $op2 # long" %}
16046 ins_encode %{
16047 __ tst($op1$$Register, $op2$$Register);
16048 %}
16049 ins_pipe(ialu_reg_reg);
16050 %}
16051
16052 instruct cmpI_and_reg(cmpOp cmp, iRegIorL2I op1, iRegIorL2I op2, immI0 op3, rFlagsReg cr) %{
16053 match(Set cr (CmpI (AndI op1 op2) op3));
16054
16055 ins_cost(INSN_COST);
16056 format %{ "tstw $op1, $op2 # int" %}
16057 ins_encode %{
16058 __ tstw($op1$$Register, $op2$$Register);
16059 %}
16060 ins_pipe(ialu_reg_reg);
16061 %}
16062
16063
16064 // Conditional Far Branch
16065 // Conditional Far Branch Unsigned
16066 // TODO: fixme
16067
16068 // counted loop end branch near
16069 instruct branchLoopEnd(cmpOp cmp, rFlagsReg cr, label lbl)
16070 %{
16071 match(CountedLoopEnd cmp cr);
16072
16073 effect(USE lbl);
16074
16075 ins_cost(BRANCH_COST);
16076 // short variant.
16077 // ins_short_branch(1);
16078 format %{ "b$cmp $lbl \t// counted loop end" %}
16079
16080 ins_encode(aarch64_enc_br_con(cmp, lbl));
16081
16082 ins_pipe(pipe_branch);
16083 %}
16084
16085 // counted loop end branch near Unsigned
16086 instruct branchLoopEndU(cmpOpU cmp, rFlagsRegU cr, label lbl)
16087 %{
16088 match(CountedLoopEnd cmp cr);
16089
16090 effect(USE lbl);
16091
16092 ins_cost(BRANCH_COST);
16093 // short variant.
16094 // ins_short_branch(1);
16095 format %{ "b$cmp $lbl \t// counted loop end unsigned" %}
16096
16097 ins_encode(aarch64_enc_br_conU(cmp, lbl));
16098
16099 ins_pipe(pipe_branch);
16100 %}
16101
16102 // counted loop end branch far
16103 // counted loop end branch far unsigned
16104 // TODO: fixme
16105
16106 // ============================================================================
16107 // inlined locking and unlocking
16108
16109 instruct cmpFastLock(rFlagsReg cr, iRegP object, iRegP box, iRegPNoSp tmp, iRegPNoSp tmp2)
16110 %{
16111 match(Set cr (FastLock object));
16112 effect(TEMP tmp, TEMP tmp2, TEMP box);
16113
16114 // TODO
16115 // identify correct cost
16116 ins_cost(5 * INSN_COST);
16117 format %{ "fastlock $object,$box\t! kills $tmp,$tmp2" %}
16118
16119 ins_encode(aarch64_enc_fast_lock(object, box, tmp, tmp2));
16120
16121 ins_pipe(pipe_serial);
16122 %}
16123
16124 instruct cmpFastUnlock(rFlagsReg cr, iRegP object, iRegP box, iRegPNoSp tmp, iRegPNoSp tmp2)
16125 %{
16126 match(Set cr (FastUnlock object));
16127 effect(TEMP box, TEMP tmp, TEMP tmp2);
16128
16129 ins_cost(5 * INSN_COST);
16130 format %{ "fastunlock $object,$box\t! kills $tmp, $tmp2" %}
16131
16132 ins_encode(aarch64_enc_fast_unlock(object, box, tmp, tmp2));
16133
16134 ins_pipe(pipe_serial);
16135 %}
16136
16137
16138 // ============================================================================
16139 // Safepoint Instructions
16140
16141 // TODO
16142 // provide a near and far version of this code
16143
16144 instruct safePoint(rFlagsReg cr, iRegP poll)
16145 %{
16146 match(SafePoint poll);
16147 effect(KILL cr);
16148
16149 format %{
16150 "ldrw zr, [$poll]\t# Safepoint: poll for GC"
16151 %}
16152 ins_encode %{
16153 __ read_polling_page(as_Register($poll$$reg), relocInfo::poll_type);
16154 %}
16155 ins_pipe(pipe_serial); // ins_pipe(iload_reg_mem);
16156 %}
16157
16158
16159 // ============================================================================
16160 // Procedure Call/Return Instructions
16161
16162 // Call Java Static Instruction
16163
16164 instruct CallStaticJavaDirect(method meth)
16165 %{
16166 match(CallStaticJava);
16167
16168 effect(USE meth);
16169
16170 ins_cost(CALL_COST);
16171
16172 format %{ "call,static $meth \t// ==> " %}
16173
16174 ins_encode(aarch64_enc_java_static_call(meth),
16175 aarch64_enc_call_epilog);
16176
16177 ins_pipe(pipe_class_call);
16178 %}
16179
16180 // TO HERE
16181
16182 // Call Java Dynamic Instruction
16183 instruct CallDynamicJavaDirect(method meth)
16184 %{
16185 match(CallDynamicJava);
16186
16187 effect(USE meth);
16188
16189 ins_cost(CALL_COST);
16190
16191 format %{ "CALL,dynamic $meth \t// ==> " %}
16192
16193 ins_encode(aarch64_enc_java_dynamic_call(meth),
16194 aarch64_enc_call_epilog);
16195
16196 ins_pipe(pipe_class_call);
16197 %}
16198
16199 // Call Runtime Instruction
16200
16201 instruct CallRuntimeDirect(method meth)
16202 %{
16203 match(CallRuntime);
16204
16205 effect(USE meth);
16206
16207 ins_cost(CALL_COST);
16208
16209 format %{ "CALL, runtime $meth" %}
16210
16211 ins_encode( aarch64_enc_java_to_runtime(meth) );
16212
16213 ins_pipe(pipe_class_call);
16214 %}
16215
16216 // Call Runtime Instruction
16217
16218 instruct CallLeafDirect(method meth)
16219 %{
16220 match(CallLeaf);
16221
16222 effect(USE meth);
16223
16224 ins_cost(CALL_COST);
16225
16226 format %{ "CALL, runtime leaf $meth" %}
16227
16228 ins_encode( aarch64_enc_java_to_runtime(meth) );
16229
16230 ins_pipe(pipe_class_call);
16231 %}
16232
16233 // Call Runtime Instruction
16234
16235 instruct CallLeafNoFPDirect(method meth)
16236 %{
16237 match(CallLeafNoFP);
16238
16239 effect(USE meth);
16240
16241 ins_cost(CALL_COST);
16242
16243 format %{ "CALL, runtime leaf nofp $meth" %}
16244
16245 ins_encode( aarch64_enc_java_to_runtime(meth) );
16246
16247 ins_pipe(pipe_class_call);
16248 %}
16249
16250 // Tail Call; Jump from runtime stub to Java code.
16251 // Also known as an 'interprocedural jump'.
16252 // Target of jump will eventually return to caller.
16253 // TailJump below removes the return address.
16254 instruct TailCalljmpInd(iRegPNoSp jump_target, inline_cache_RegP method_ptr)
16255 %{
16256 match(TailCall jump_target method_ptr);
16257
16258 ins_cost(CALL_COST);
16259
16260 format %{ "br $jump_target\t# $method_ptr holds method" %}
16261
16262 ins_encode(aarch64_enc_tail_call(jump_target));
16263
16264 ins_pipe(pipe_class_call);
16265 %}
16266
16267 instruct TailjmpInd(iRegPNoSp jump_target, iRegP_R0 ex_oop)
16268 %{
16269 match(TailJump jump_target ex_oop);
16270
16271 ins_cost(CALL_COST);
16272
16273 format %{ "br $jump_target\t# $ex_oop holds exception oop" %}
16274
16275 ins_encode(aarch64_enc_tail_jmp(jump_target));
16276
16277 ins_pipe(pipe_class_call);
16278 %}
16279
16280 // Create exception oop: created by stack-crawling runtime code.
16281 // Created exception is now available to this handler, and is setup
16282 // just prior to jumping to this handler. No code emitted.
16283 // TODO check
16284 // should ex_oop be in r0? intel uses rax, ppc cannot use r0 so uses rarg1
16285 instruct CreateException(iRegP_R0 ex_oop)
16286 %{
16287 match(Set ex_oop (CreateEx));
16288
16289 format %{ " -- \t// exception oop; no code emitted" %}
16290
16291 size(0);
16292
16293 ins_encode( /*empty*/ );
16294
16295 ins_pipe(pipe_class_empty);
16296 %}
16297
16298 // Rethrow exception: The exception oop will come in the first
16299 // argument position. Then JUMP (not call) to the rethrow stub code.
16300 instruct RethrowException() %{
16301 match(Rethrow);
16302 ins_cost(CALL_COST);
16303
16304 format %{ "b rethrow_stub" %}
16305
16306 ins_encode( aarch64_enc_rethrow() );
16307
16308 ins_pipe(pipe_class_call);
16309 %}
16310
16311
16312 // Return Instruction
16313 // epilog node loads ret address into lr as part of frame pop
16314 instruct Ret()
16315 %{
16316 match(Return);
16317
16318 format %{ "ret\t// return register" %}
16319
16320 ins_encode( aarch64_enc_ret() );
16321
16322 ins_pipe(pipe_branch);
16323 %}
16324
16325 // Die now.
16326 instruct ShouldNotReachHere() %{
16327 match(Halt);
16328
16329 ins_cost(CALL_COST);
16330 format %{ "ShouldNotReachHere" %}
16331
16332 ins_encode %{
16333 if (is_reachable()) {
16334 __ stop(_halt_reason);
16335 }
16336 %}
16337
16338 ins_pipe(pipe_class_default);
16339 %}
16340
16341 // ============================================================================
16342 // Partial Subtype Check
16343 //
16344 // superklass array for an instance of the superklass. Set a hidden
16345 // internal cache on a hit (cache is checked with exposed code in
16346 // gen_subtype_check()). Return NZ for a miss or zero for a hit. The
16347 // encoding ALSO sets flags.
16348
16349 instruct partialSubtypeCheck(iRegP_R4 sub, iRegP_R0 super, iRegP_R2 temp, iRegP_R5 result, rFlagsReg cr)
16350 %{
16351 match(Set result (PartialSubtypeCheck sub super));
16352 effect(KILL cr, KILL temp);
16353
16354 ins_cost(1100); // slightly larger than the next version
16355 format %{ "partialSubtypeCheck $result, $sub, $super" %}
16356
16357 ins_encode(aarch64_enc_partial_subtype_check(sub, super, temp, result));
16358
16359 opcode(0x1); // Force zero of result reg on hit
16360
16361 ins_pipe(pipe_class_memory);
16362 %}
16363
16364 instruct partialSubtypeCheckVsZero(iRegP_R4 sub, iRegP_R0 super, iRegP_R2 temp, iRegP_R5 result, immP0 zero, rFlagsReg cr)
16365 %{
16366 match(Set cr (CmpP (PartialSubtypeCheck sub super) zero));
16367 effect(KILL temp, KILL result);
16368
16369 ins_cost(1100); // slightly larger than the next version
16370 format %{ "partialSubtypeCheck $result, $sub, $super == 0" %}
16371
16372 ins_encode(aarch64_enc_partial_subtype_check(sub, super, temp, result));
16373
16374 opcode(0x0); // Don't zero result reg on hit
16375
16376 ins_pipe(pipe_class_memory);
16377 %}
16378
16379 // Intrisics for String.compareTo()
16380
16381 instruct string_compareU(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
16382 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2, rFlagsReg cr)
16383 %{
16384 predicate((UseSVE == 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::UU));
16385 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
16386 effect(KILL tmp1, KILL tmp2, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
16387
16388 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # KILL $tmp1" %}
16389 ins_encode %{
16390 // Count is in 8-bit bytes; non-Compact chars are 16 bits.
16391 __ string_compare($str1$$Register, $str2$$Register,
16392 $cnt1$$Register, $cnt2$$Register, $result$$Register,
16393 $tmp1$$Register, $tmp2$$Register,
16394 fnoreg, fnoreg, fnoreg, pnoreg, pnoreg, StrIntrinsicNode::UU);
16395 %}
16396 ins_pipe(pipe_class_memory);
16397 %}
16398
16399 instruct string_compareL(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
16400 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2, rFlagsReg cr)
16401 %{
16402 predicate((UseSVE == 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::LL));
16403 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
16404 effect(KILL tmp1, KILL tmp2, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
16405
16406 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # KILL $tmp1" %}
16407 ins_encode %{
16408 __ string_compare($str1$$Register, $str2$$Register,
16409 $cnt1$$Register, $cnt2$$Register, $result$$Register,
16410 $tmp1$$Register, $tmp2$$Register,
16411 fnoreg, fnoreg, fnoreg, pnoreg, pnoreg, StrIntrinsicNode::LL);
16412 %}
16413 ins_pipe(pipe_class_memory);
16414 %}
16415
16416 instruct string_compareUL(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
16417 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
16418 vRegD_V0 vtmp1, vRegD_V1 vtmp2, vRegD_V2 vtmp3, rFlagsReg cr)
16419 %{
16420 predicate((UseSVE == 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::UL));
16421 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
16422 effect(KILL tmp1, KILL tmp2, KILL vtmp1, KILL vtmp2, KILL vtmp3,
16423 USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
16424
16425 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # KILL $tmp1, $tmp2, $vtmp1, $vtmp2, $vtmp3" %}
16426 ins_encode %{
16427 __ string_compare($str1$$Register, $str2$$Register,
16428 $cnt1$$Register, $cnt2$$Register, $result$$Register,
16429 $tmp1$$Register, $tmp2$$Register,
16430 $vtmp1$$FloatRegister, $vtmp2$$FloatRegister,
16431 $vtmp3$$FloatRegister, pnoreg, pnoreg, StrIntrinsicNode::UL);
16432 %}
16433 ins_pipe(pipe_class_memory);
16434 %}
16435
16436 instruct string_compareLU(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
16437 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
16438 vRegD_V0 vtmp1, vRegD_V1 vtmp2, vRegD_V2 vtmp3, rFlagsReg cr)
16439 %{
16440 predicate((UseSVE == 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::LU));
16441 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
16442 effect(KILL tmp1, KILL tmp2, KILL vtmp1, KILL vtmp2, KILL vtmp3,
16443 USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
16444
16445 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # KILL $tmp1, $tmp2, $vtmp1, $vtmp2, $vtmp3" %}
16446 ins_encode %{
16447 __ string_compare($str1$$Register, $str2$$Register,
16448 $cnt1$$Register, $cnt2$$Register, $result$$Register,
16449 $tmp1$$Register, $tmp2$$Register,
16450 $vtmp1$$FloatRegister, $vtmp2$$FloatRegister,
16451 $vtmp3$$FloatRegister, pnoreg, pnoreg, StrIntrinsicNode::LU);
16452 %}
16453 ins_pipe(pipe_class_memory);
16454 %}
16455
16456 // Note that Z registers alias the corresponding NEON registers, we declare the vector operands of
16457 // these string_compare variants as NEON register type for convenience so that the prototype of
16458 // string_compare can be shared with all variants.
16459
16460 instruct string_compareLL_sve(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
16461 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
16462 vRegD_V0 vtmp1, vRegD_V1 vtmp2, pRegGov_P0 pgtmp1,
16463 pRegGov_P1 pgtmp2, rFlagsReg cr)
16464 %{
16465 predicate((UseSVE > 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::LL));
16466 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
16467 effect(TEMP tmp1, TEMP tmp2, TEMP vtmp1, TEMP vtmp2, TEMP pgtmp1, TEMP pgtmp2,
16468 USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
16469
16470 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # USE sve" %}
16471 ins_encode %{
16472 // Count is in 8-bit bytes; non-Compact chars are 16 bits.
16473 __ string_compare($str1$$Register, $str2$$Register,
16474 $cnt1$$Register, $cnt2$$Register, $result$$Register,
16475 $tmp1$$Register, $tmp2$$Register,
16476 $vtmp1$$FloatRegister, $vtmp2$$FloatRegister, fnoreg,
16477 as_PRegister($pgtmp1$$reg), as_PRegister($pgtmp2$$reg),
16478 StrIntrinsicNode::LL);
16479 %}
16480 ins_pipe(pipe_class_memory);
16481 %}
16482
16483 instruct string_compareLU_sve(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
16484 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
16485 vRegD_V0 vtmp1, vRegD_V1 vtmp2, pRegGov_P0 pgtmp1,
16486 pRegGov_P1 pgtmp2, rFlagsReg cr)
16487 %{
16488 predicate((UseSVE > 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::LU));
16489 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
16490 effect(TEMP tmp1, TEMP tmp2, TEMP vtmp1, TEMP vtmp2, TEMP pgtmp1, TEMP pgtmp2,
16491 USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
16492
16493 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # USE sve" %}
16494 ins_encode %{
16495 // Count is in 8-bit bytes; non-Compact chars are 16 bits.
16496 __ string_compare($str1$$Register, $str2$$Register,
16497 $cnt1$$Register, $cnt2$$Register, $result$$Register,
16498 $tmp1$$Register, $tmp2$$Register,
16499 $vtmp1$$FloatRegister, $vtmp2$$FloatRegister, fnoreg,
16500 as_PRegister($pgtmp1$$reg), as_PRegister($pgtmp2$$reg),
16501 StrIntrinsicNode::LU);
16502 %}
16503 ins_pipe(pipe_class_memory);
16504 %}
16505
16506 instruct string_compareUL_sve(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
16507 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
16508 vRegD_V0 vtmp1, vRegD_V1 vtmp2, pRegGov_P0 pgtmp1,
16509 pRegGov_P1 pgtmp2, rFlagsReg cr)
16510 %{
16511 predicate((UseSVE > 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::UL));
16512 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
16513 effect(TEMP tmp1, TEMP tmp2, TEMP vtmp1, TEMP vtmp2, TEMP pgtmp1, TEMP pgtmp2,
16514 USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
16515
16516 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # USE sve" %}
16517 ins_encode %{
16518 // Count is in 8-bit bytes; non-Compact chars are 16 bits.
16519 __ string_compare($str1$$Register, $str2$$Register,
16520 $cnt1$$Register, $cnt2$$Register, $result$$Register,
16521 $tmp1$$Register, $tmp2$$Register,
16522 $vtmp1$$FloatRegister, $vtmp2$$FloatRegister, fnoreg,
16523 as_PRegister($pgtmp1$$reg), as_PRegister($pgtmp2$$reg),
16524 StrIntrinsicNode::UL);
16525 %}
16526 ins_pipe(pipe_class_memory);
16527 %}
16528
16529 instruct string_compareUU_sve(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
16530 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
16531 vRegD_V0 vtmp1, vRegD_V1 vtmp2, pRegGov_P0 pgtmp1,
16532 pRegGov_P1 pgtmp2, rFlagsReg cr)
16533 %{
16534 predicate((UseSVE > 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::UU));
16535 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
16536 effect(TEMP tmp1, TEMP tmp2, TEMP vtmp1, TEMP vtmp2, TEMP pgtmp1, TEMP pgtmp2,
16537 USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
16538
16539 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # USE sve" %}
16540 ins_encode %{
16541 // Count is in 8-bit bytes; non-Compact chars are 16 bits.
16542 __ string_compare($str1$$Register, $str2$$Register,
16543 $cnt1$$Register, $cnt2$$Register, $result$$Register,
16544 $tmp1$$Register, $tmp2$$Register,
16545 $vtmp1$$FloatRegister, $vtmp2$$FloatRegister, fnoreg,
16546 as_PRegister($pgtmp1$$reg), as_PRegister($pgtmp2$$reg),
16547 StrIntrinsicNode::UU);
16548 %}
16549 ins_pipe(pipe_class_memory);
16550 %}
16551
16552 instruct string_indexofUU(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2, iRegI_R2 cnt2,
16553 iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2, iRegINoSp tmp3,
16554 iRegINoSp tmp4, iRegINoSp tmp5, iRegINoSp tmp6, rFlagsReg cr)
16555 %{
16556 predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UU);
16557 match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 cnt2)));
16558 effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2,
16559 TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, TEMP tmp5, TEMP tmp6, KILL cr);
16560 format %{ "String IndexOf $str1,$cnt1,$str2,$cnt2 -> $result (UU)" %}
16561
16562 ins_encode %{
16563 __ string_indexof($str1$$Register, $str2$$Register,
16564 $cnt1$$Register, $cnt2$$Register,
16565 $tmp1$$Register, $tmp2$$Register,
16566 $tmp3$$Register, $tmp4$$Register,
16567 $tmp5$$Register, $tmp6$$Register,
16568 -1, $result$$Register, StrIntrinsicNode::UU);
16569 %}
16570 ins_pipe(pipe_class_memory);
16571 %}
16572
16573 instruct string_indexofLL(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2, iRegI_R2 cnt2,
16574 iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2, iRegINoSp tmp3,
16575 iRegINoSp tmp4, iRegINoSp tmp5, iRegINoSp tmp6, rFlagsReg cr)
16576 %{
16577 predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::LL);
16578 match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 cnt2)));
16579 effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2,
16580 TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, TEMP tmp5, TEMP tmp6, KILL cr);
16581 format %{ "String IndexOf $str1,$cnt1,$str2,$cnt2 -> $result (LL)" %}
16582
16583 ins_encode %{
16584 __ string_indexof($str1$$Register, $str2$$Register,
16585 $cnt1$$Register, $cnt2$$Register,
16586 $tmp1$$Register, $tmp2$$Register,
16587 $tmp3$$Register, $tmp4$$Register,
16588 $tmp5$$Register, $tmp6$$Register,
16589 -1, $result$$Register, StrIntrinsicNode::LL);
16590 %}
16591 ins_pipe(pipe_class_memory);
16592 %}
16593
16594 instruct string_indexofUL(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2, iRegI_R2 cnt2,
16595 iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2, iRegINoSp tmp3,
16596 iRegINoSp tmp4, iRegINoSp tmp5, iRegINoSp tmp6, rFlagsReg cr)
16597 %{
16598 predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UL);
16599 match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 cnt2)));
16600 effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2,
16601 TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, TEMP tmp5, TEMP tmp6, KILL cr);
16602 format %{ "String IndexOf $str1,$cnt1,$str2,$cnt2 -> $result (UL)" %}
16603
16604 ins_encode %{
16605 __ string_indexof($str1$$Register, $str2$$Register,
16606 $cnt1$$Register, $cnt2$$Register,
16607 $tmp1$$Register, $tmp2$$Register,
16608 $tmp3$$Register, $tmp4$$Register,
16609 $tmp5$$Register, $tmp6$$Register,
16610 -1, $result$$Register, StrIntrinsicNode::UL);
16611 %}
16612 ins_pipe(pipe_class_memory);
16613 %}
16614
16615 instruct string_indexof_conUU(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2,
16616 immI_le_4 int_cnt2, iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2,
16617 iRegINoSp tmp3, iRegINoSp tmp4, rFlagsReg cr)
16618 %{
16619 predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UU);
16620 match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 int_cnt2)));
16621 effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1,
16622 TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, KILL cr);
16623 format %{ "String IndexOf $str1,$cnt1,$str2,$int_cnt2 -> $result (UU)" %}
16624
16625 ins_encode %{
16626 int icnt2 = (int)$int_cnt2$$constant;
16627 __ string_indexof($str1$$Register, $str2$$Register,
16628 $cnt1$$Register, zr,
16629 $tmp1$$Register, $tmp2$$Register,
16630 $tmp3$$Register, $tmp4$$Register, zr, zr,
16631 icnt2, $result$$Register, StrIntrinsicNode::UU);
16632 %}
16633 ins_pipe(pipe_class_memory);
16634 %}
16635
16636 instruct string_indexof_conLL(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2,
16637 immI_le_4 int_cnt2, iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2,
16638 iRegINoSp tmp3, iRegINoSp tmp4, rFlagsReg cr)
16639 %{
16640 predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::LL);
16641 match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 int_cnt2)));
16642 effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1,
16643 TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, KILL cr);
16644 format %{ "String IndexOf $str1,$cnt1,$str2,$int_cnt2 -> $result (LL)" %}
16645
16646 ins_encode %{
16647 int icnt2 = (int)$int_cnt2$$constant;
16648 __ string_indexof($str1$$Register, $str2$$Register,
16649 $cnt1$$Register, zr,
16650 $tmp1$$Register, $tmp2$$Register,
16651 $tmp3$$Register, $tmp4$$Register, zr, zr,
16652 icnt2, $result$$Register, StrIntrinsicNode::LL);
16653 %}
16654 ins_pipe(pipe_class_memory);
16655 %}
16656
16657 instruct string_indexof_conUL(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2,
16658 immI_1 int_cnt2, iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2,
16659 iRegINoSp tmp3, iRegINoSp tmp4, rFlagsReg cr)
16660 %{
16661 predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UL);
16662 match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 int_cnt2)));
16663 effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1,
16664 TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, KILL cr);
16665 format %{ "String IndexOf $str1,$cnt1,$str2,$int_cnt2 -> $result (UL)" %}
16666
16667 ins_encode %{
16668 int icnt2 = (int)$int_cnt2$$constant;
16669 __ string_indexof($str1$$Register, $str2$$Register,
16670 $cnt1$$Register, zr,
16671 $tmp1$$Register, $tmp2$$Register,
16672 $tmp3$$Register, $tmp4$$Register, zr, zr,
16673 icnt2, $result$$Register, StrIntrinsicNode::UL);
16674 %}
16675 ins_pipe(pipe_class_memory);
16676 %}
16677
16678 instruct string_indexof_char(iRegP_R1 str1, iRegI_R2 cnt1, iRegI_R3 ch,
16679 iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2,
16680 iRegINoSp tmp3, rFlagsReg cr)
16681 %{
16682 match(Set result (StrIndexOfChar (Binary str1 cnt1) ch));
16683 predicate((UseSVE == 0) && (((StrIndexOfCharNode*)n)->encoding() == StrIntrinsicNode::U));
16684 effect(USE_KILL str1, USE_KILL cnt1, USE_KILL ch,
16685 TEMP tmp1, TEMP tmp2, TEMP tmp3, KILL cr);
16686
16687 format %{ "StringUTF16 IndexOf char[] $str1,$cnt1,$ch -> $result" %}
16688
16689 ins_encode %{
16690 __ string_indexof_char($str1$$Register, $cnt1$$Register, $ch$$Register,
16691 $result$$Register, $tmp1$$Register, $tmp2$$Register,
16692 $tmp3$$Register);
16693 %}
16694 ins_pipe(pipe_class_memory);
16695 %}
16696
16697 instruct stringL_indexof_char(iRegP_R1 str1, iRegI_R2 cnt1, iRegI_R3 ch,
16698 iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2,
16699 iRegINoSp tmp3, rFlagsReg cr)
16700 %{
16701 match(Set result (StrIndexOfChar (Binary str1 cnt1) ch));
16702 predicate((UseSVE == 0) && (((StrIndexOfCharNode*)n)->encoding() == StrIntrinsicNode::L));
16703 effect(USE_KILL str1, USE_KILL cnt1, USE_KILL ch,
16704 TEMP tmp1, TEMP tmp2, TEMP tmp3, KILL cr);
16705
16706 format %{ "StringLatin1 IndexOf char[] $str1,$cnt1,$ch -> $result" %}
16707
16708 ins_encode %{
16709 __ stringL_indexof_char($str1$$Register, $cnt1$$Register, $ch$$Register,
16710 $result$$Register, $tmp1$$Register, $tmp2$$Register,
16711 $tmp3$$Register);
16712 %}
16713 ins_pipe(pipe_class_memory);
16714 %}
16715
16716 instruct stringL_indexof_char_sve(iRegP_R1 str1, iRegI_R2 cnt1, iRegI_R3 ch,
16717 iRegI_R0 result, vecA ztmp1, vecA ztmp2,
16718 pRegGov pgtmp, pReg ptmp, rFlagsReg cr) %{
16719 predicate(UseSVE > 0 && ((StrIndexOfCharNode*)n)->encoding() == StrIntrinsicNode::L);
16720 match(Set result (StrIndexOfChar (Binary str1 cnt1) ch));
16721 effect(TEMP ztmp1, TEMP ztmp2, TEMP pgtmp, TEMP ptmp, KILL cr);
16722 format %{ "StringLatin1 IndexOf char[] $str1,$cnt1,$ch -> $result # use sve" %}
16723 ins_encode %{
16724 __ string_indexof_char_sve($str1$$Register, $cnt1$$Register, $ch$$Register,
16725 $result$$Register, $ztmp1$$FloatRegister,
16726 $ztmp2$$FloatRegister, $pgtmp$$PRegister,
16727 $ptmp$$PRegister, true /* isL */);
16728 %}
16729 ins_pipe(pipe_class_memory);
16730 %}
16731
16732 instruct stringU_indexof_char_sve(iRegP_R1 str1, iRegI_R2 cnt1, iRegI_R3 ch,
16733 iRegI_R0 result, vecA ztmp1, vecA ztmp2,
16734 pRegGov pgtmp, pReg ptmp, rFlagsReg cr) %{
16735 predicate(UseSVE > 0 && ((StrIndexOfCharNode*)n)->encoding() == StrIntrinsicNode::U);
16736 match(Set result (StrIndexOfChar (Binary str1 cnt1) ch));
16737 effect(TEMP ztmp1, TEMP ztmp2, TEMP pgtmp, TEMP ptmp, KILL cr);
16738 format %{ "StringUTF16 IndexOf char[] $str1,$cnt1,$ch -> $result # use sve" %}
16739 ins_encode %{
16740 __ string_indexof_char_sve($str1$$Register, $cnt1$$Register, $ch$$Register,
16741 $result$$Register, $ztmp1$$FloatRegister,
16742 $ztmp2$$FloatRegister, $pgtmp$$PRegister,
16743 $ptmp$$PRegister, false /* isL */);
16744 %}
16745 ins_pipe(pipe_class_memory);
16746 %}
16747
16748 instruct string_equalsL(iRegP_R1 str1, iRegP_R3 str2, iRegI_R4 cnt,
16749 iRegI_R0 result, rFlagsReg cr)
16750 %{
16751 predicate(((StrEqualsNode*)n)->encoding() == StrIntrinsicNode::LL);
16752 match(Set result (StrEquals (Binary str1 str2) cnt));
16753 effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt, KILL cr);
16754
16755 format %{ "String Equals $str1,$str2,$cnt -> $result" %}
16756 ins_encode %{
16757 // Count is in 8-bit bytes; non-Compact chars are 16 bits.
16758 __ string_equals($str1$$Register, $str2$$Register,
16759 $result$$Register, $cnt$$Register, 1);
16760 %}
16761 ins_pipe(pipe_class_memory);
16762 %}
16763
16764 instruct string_equalsU(iRegP_R1 str1, iRegP_R3 str2, iRegI_R4 cnt,
16765 iRegI_R0 result, rFlagsReg cr)
16766 %{
16767 predicate(((StrEqualsNode*)n)->encoding() == StrIntrinsicNode::UU);
16768 match(Set result (StrEquals (Binary str1 str2) cnt));
16769 effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt, KILL cr);
16770
16771 format %{ "String Equals $str1,$str2,$cnt -> $result" %}
16772 ins_encode %{
16773 // Count is in 8-bit bytes; non-Compact chars are 16 bits.
16774 __ string_equals($str1$$Register, $str2$$Register,
16775 $result$$Register, $cnt$$Register, 2);
16776 %}
16777 ins_pipe(pipe_class_memory);
16778 %}
16779
16780 instruct array_equalsB(iRegP_R1 ary1, iRegP_R2 ary2, iRegI_R0 result,
16781 iRegP_R3 tmp1, iRegP_R4 tmp2, iRegP_R5 tmp3,
16782 iRegP_R10 tmp, rFlagsReg cr)
16783 %{
16784 predicate(((AryEqNode*)n)->encoding() == StrIntrinsicNode::LL);
16785 match(Set result (AryEq ary1 ary2));
16786 effect(KILL tmp, USE_KILL ary1, USE_KILL ary2, TEMP tmp1, TEMP tmp2, TEMP tmp3, KILL cr);
16787
16788 format %{ "Array Equals $ary1,ary2 -> $result // KILL $tmp" %}
16789 ins_encode %{
16790 address tpc = __ arrays_equals($ary1$$Register, $ary2$$Register,
16791 $tmp1$$Register, $tmp2$$Register, $tmp3$$Register,
16792 $result$$Register, $tmp$$Register, 1);
16793 if (tpc == NULL) {
16794 ciEnv::current()->record_failure("CodeCache is full");
16795 return;
16796 }
16797 %}
16798 ins_pipe(pipe_class_memory);
16799 %}
16800
16801 instruct array_equalsC(iRegP_R1 ary1, iRegP_R2 ary2, iRegI_R0 result,
16802 iRegP_R3 tmp1, iRegP_R4 tmp2, iRegP_R5 tmp3,
16803 iRegP_R10 tmp, rFlagsReg cr)
16804 %{
16805 predicate(((AryEqNode*)n)->encoding() == StrIntrinsicNode::UU);
16806 match(Set result (AryEq ary1 ary2));
16807 effect(KILL tmp, USE_KILL ary1, USE_KILL ary2, TEMP tmp1, TEMP tmp2, TEMP tmp3, KILL cr);
16808
16809 format %{ "Array Equals $ary1,ary2 -> $result // KILL $tmp" %}
16810 ins_encode %{
16811 address tpc = __ arrays_equals($ary1$$Register, $ary2$$Register,
16812 $tmp1$$Register, $tmp2$$Register, $tmp3$$Register,
16813 $result$$Register, $tmp$$Register, 2);
16814 if (tpc == NULL) {
16815 ciEnv::current()->record_failure("CodeCache is full");
16816 return;
16817 }
16818 %}
16819 ins_pipe(pipe_class_memory);
16820 %}
16821
16822 instruct count_positives(iRegP_R1 ary1, iRegI_R2 len, iRegI_R0 result, rFlagsReg cr)
16823 %{
16824 match(Set result (CountPositives ary1 len));
16825 effect(USE_KILL ary1, USE_KILL len, KILL cr);
16826 format %{ "count positives byte[] $ary1,$len -> $result" %}
16827 ins_encode %{
16828 address tpc = __ count_positives($ary1$$Register, $len$$Register, $result$$Register);
16829 if (tpc == NULL) {
16830 ciEnv::current()->record_failure("CodeCache is full");
16831 return;
16832 }
16833 %}
16834 ins_pipe( pipe_slow );
16835 %}
16836
16837 // fast char[] to byte[] compression
16838 instruct string_compress(iRegP_R2 src, iRegP_R1 dst, iRegI_R3 len,
16839 vRegD_V0 tmp1, vRegD_V1 tmp2,
16840 vRegD_V2 tmp3, vRegD_V3 tmp4,
16841 iRegI_R0 result, rFlagsReg cr)
16842 %{
16843 match(Set result (StrCompressedCopy src (Binary dst len)));
16844 effect(TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4,
16845 USE_KILL src, USE_KILL dst, USE len, KILL cr);
16846
16847 format %{ "String Compress $src,$dst,$len -> $result // KILL $src,$dst" %}
16848 ins_encode %{
16849 __ char_array_compress($src$$Register, $dst$$Register, $len$$Register,
16850 $result$$Register,
16851 $tmp1$$FloatRegister, $tmp2$$FloatRegister,
16852 $tmp3$$FloatRegister, $tmp4$$FloatRegister);
16853 %}
16854 ins_pipe(pipe_slow);
16855 %}
16856
16857 // fast byte[] to char[] inflation
16858 instruct string_inflate(Universe dummy, iRegP_R0 src, iRegP_R1 dst, iRegI_R2 len,
16859 vRegD_V0 tmp1, vRegD_V1 tmp2, vRegD_V2 tmp3, iRegP_R3 tmp4, rFlagsReg cr)
16860 %{
16861 match(Set dummy (StrInflatedCopy src (Binary dst len)));
16862 effect(TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, USE_KILL src, USE_KILL dst, USE_KILL len, KILL cr);
16863
16864 format %{ "String Inflate $src,$dst // KILL $tmp1, $tmp2" %}
16865 ins_encode %{
16866 address tpc = __ byte_array_inflate($src$$Register, $dst$$Register, $len$$Register,
16867 $tmp1$$FloatRegister, $tmp2$$FloatRegister,
16868 $tmp3$$FloatRegister, $tmp4$$Register);
16869 if (tpc == NULL) {
16870 ciEnv::current()->record_failure("CodeCache is full");
16871 return;
16872 }
16873 %}
16874 ins_pipe(pipe_class_memory);
16875 %}
16876
16877 // encode char[] to byte[] in ISO_8859_1
16878 instruct encode_iso_array(iRegP_R2 src, iRegP_R1 dst, iRegI_R3 len,
16879 vRegD_V0 vtmp0, vRegD_V1 vtmp1,
16880 vRegD_V2 vtmp2, vRegD_V3 vtmp3,
16881 iRegI_R0 result, rFlagsReg cr)
16882 %{
16883 predicate(!((EncodeISOArrayNode*)n)->is_ascii());
16884 match(Set result (EncodeISOArray src (Binary dst len)));
16885 effect(USE_KILL src, USE_KILL dst, USE len,
16886 KILL vtmp0, KILL vtmp1, KILL vtmp2, KILL vtmp3, KILL cr);
16887
16888 format %{ "Encode ISO array $src,$dst,$len -> $result" %}
16889 ins_encode %{
16890 __ encode_iso_array($src$$Register, $dst$$Register, $len$$Register,
16891 $result$$Register, false,
16892 $vtmp0$$FloatRegister, $vtmp1$$FloatRegister,
16893 $vtmp2$$FloatRegister, $vtmp3$$FloatRegister);
16894 %}
16895 ins_pipe(pipe_class_memory);
16896 %}
16897
16898 instruct encode_ascii_array(iRegP_R2 src, iRegP_R1 dst, iRegI_R3 len,
16899 vRegD_V0 vtmp0, vRegD_V1 vtmp1,
16900 vRegD_V2 vtmp2, vRegD_V3 vtmp3,
16901 iRegI_R0 result, rFlagsReg cr)
16902 %{
16903 predicate(((EncodeISOArrayNode*)n)->is_ascii());
16904 match(Set result (EncodeISOArray src (Binary dst len)));
16905 effect(USE_KILL src, USE_KILL dst, USE len,
16906 KILL vtmp0, KILL vtmp1, KILL vtmp2, KILL vtmp3, KILL cr);
16907
16908 format %{ "Encode ASCII array $src,$dst,$len -> $result" %}
16909 ins_encode %{
16910 __ encode_iso_array($src$$Register, $dst$$Register, $len$$Register,
16911 $result$$Register, true,
16912 $vtmp0$$FloatRegister, $vtmp1$$FloatRegister,
16913 $vtmp2$$FloatRegister, $vtmp3$$FloatRegister);
16914 %}
16915 ins_pipe(pipe_class_memory);
16916 %}
16917
16918 // ============================================================================
16919 // This name is KNOWN by the ADLC and cannot be changed.
16920 // The ADLC forces a 'TypeRawPtr::BOTTOM' output type
16921 // for this guy.
16922 instruct tlsLoadP(thread_RegP dst)
16923 %{
16924 match(Set dst (ThreadLocal));
16925
16926 ins_cost(0);
16927
16928 format %{ " -- \t// $dst=Thread::current(), empty" %}
16929
16930 size(0);
16931
16932 ins_encode( /*empty*/ );
16933
16934 ins_pipe(pipe_class_empty);
16935 %}
16936
16937 //----------PEEPHOLE RULES-----------------------------------------------------
16938 // These must follow all instruction definitions as they use the names
16939 // defined in the instructions definitions.
16940 //
16941 // peepmatch ( root_instr_name [preceding_instruction]* );
16942 //
16943 // peepconstraint %{
16944 // (instruction_number.operand_name relational_op instruction_number.operand_name
16945 // [, ...] );
16946 // // instruction numbers are zero-based using left to right order in peepmatch
16947 //
16948 // peepreplace ( instr_name ( [instruction_number.operand_name]* ) );
16949 // // provide an instruction_number.operand_name for each operand that appears
16950 // // in the replacement instruction's match rule
16951 //
16952 // ---------VM FLAGS---------------------------------------------------------
16953 //
16954 // All peephole optimizations can be turned off using -XX:-OptoPeephole
16955 //
16956 // Each peephole rule is given an identifying number starting with zero and
16957 // increasing by one in the order seen by the parser. An individual peephole
16958 // can be enabled, and all others disabled, by using -XX:OptoPeepholeAt=#
16959 // on the command-line.
16960 //
16961 // ---------CURRENT LIMITATIONS----------------------------------------------
16962 //
16963 // Only match adjacent instructions in same basic block
16964 // Only equality constraints
16965 // Only constraints between operands, not (0.dest_reg == RAX_enc)
16966 // Only one replacement instruction
16967 //
16968 // ---------EXAMPLE----------------------------------------------------------
16969 //
16970 // // pertinent parts of existing instructions in architecture description
16971 // instruct movI(iRegINoSp dst, iRegI src)
16972 // %{
16973 // match(Set dst (CopyI src));
16974 // %}
16975 //
16976 // instruct incI_iReg(iRegINoSp dst, immI1 src, rFlagsReg cr)
16977 // %{
16978 // match(Set dst (AddI dst src));
16979 // effect(KILL cr);
16980 // %}
16981 //
16982 // // Change (inc mov) to lea
16983 // peephole %{
16984 // // increment preceded by register-register move
16985 // peepmatch ( incI_iReg movI );
16986 // // require that the destination register of the increment
16987 // // match the destination register of the move
16988 // peepconstraint ( 0.dst == 1.dst );
16989 // // construct a replacement instruction that sets
16990 // // the destination to ( move's source register + one )
16991 // peepreplace ( leaI_iReg_immI( 0.dst 1.src 0.src ) );
16992 // %}
16993 //
16994
16995 // Implementation no longer uses movX instructions since
16996 // machine-independent system no longer uses CopyX nodes.
16997 //
16998 // peephole
16999 // %{
17000 // peepmatch (incI_iReg movI);
17001 // peepconstraint (0.dst == 1.dst);
17002 // peepreplace (leaI_iReg_immI(0.dst 1.src 0.src));
17003 // %}
17004
17005 // peephole
17006 // %{
17007 // peepmatch (decI_iReg movI);
17008 // peepconstraint (0.dst == 1.dst);
17009 // peepreplace (leaI_iReg_immI(0.dst 1.src 0.src));
17010 // %}
17011
17012 // peephole
17013 // %{
17014 // peepmatch (addI_iReg_imm movI);
17015 // peepconstraint (0.dst == 1.dst);
17016 // peepreplace (leaI_iReg_immI(0.dst 1.src 0.src));
17017 // %}
17018
17019 // peephole
17020 // %{
17021 // peepmatch (incL_iReg movL);
17022 // peepconstraint (0.dst == 1.dst);
17023 // peepreplace (leaL_iReg_immL(0.dst 1.src 0.src));
17024 // %}
17025
17026 // peephole
17027 // %{
17028 // peepmatch (decL_iReg movL);
17029 // peepconstraint (0.dst == 1.dst);
17030 // peepreplace (leaL_iReg_immL(0.dst 1.src 0.src));
17031 // %}
17032
17033 // peephole
17034 // %{
17035 // peepmatch (addL_iReg_imm movL);
17036 // peepconstraint (0.dst == 1.dst);
17037 // peepreplace (leaL_iReg_immL(0.dst 1.src 0.src));
17038 // %}
17039
17040 // peephole
17041 // %{
17042 // peepmatch (addP_iReg_imm movP);
17043 // peepconstraint (0.dst == 1.dst);
17044 // peepreplace (leaP_iReg_imm(0.dst 1.src 0.src));
17045 // %}
17046
17047 // // Change load of spilled value to only a spill
17048 // instruct storeI(memory mem, iRegI src)
17049 // %{
17050 // match(Set mem (StoreI mem src));
17051 // %}
17052 //
17053 // instruct loadI(iRegINoSp dst, memory mem)
17054 // %{
17055 // match(Set dst (LoadI mem));
17056 // %}
17057 //
17058
17059 //----------SMARTSPILL RULES---------------------------------------------------
17060 // These must follow all instruction definitions as they use the names
17061 // defined in the instructions definitions.
17062
17063 // Local Variables:
17064 // mode: c++
17065 // End: