1 //
2 // Copyright (c) 2003, 2019, Oracle and/or its affiliates. All rights reserved.
3 // Copyright (c) 2014, 2019, Red Hat Inc.
4 // All rights reserved.
5 // DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
6 //
7 // This code is free software; you can redistribute it and/or modify it
8 // under the terms of the GNU General Public License version 2 only, as
9 // published by the Free Software Foundation.
10 //
11 // This code is distributed in the hope that it will be useful, but WITHOUT
12 // ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
13 // FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
14 // version 2 for more details (a copy is included in the LICENSE file that
15 // accompanied this code).
16 //
17 // You should have received a copy of the GNU General Public License version
18 // 2 along with this work; if not, write to the Free Software Foundation,
19 // Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
20 //
21 // Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
22 // or visit www.oracle.com if you need additional information or have any
23 // questions.
24 //
25 //
26
27 // AArch64 Architecture Description File
28
29 //----------REGISTER DEFINITION BLOCK------------------------------------------
30 // This information is used by the matcher and the register allocator to
31 // describe individual registers and classes of registers within the target
32 // archtecture.
33
34 register %{
35 //----------Architecture Description Register Definitions----------------------
36 // General Registers
37 // "reg_def" name ( register save type, C convention save type,
38 // ideal register type, encoding );
39 // Register Save Types:
40 //
41 // NS = No-Save: The register allocator assumes that these registers
42 // can be used without saving upon entry to the method, &
43 // that they do not need to be saved at call sites.
44 //
45 // SOC = Save-On-Call: The register allocator assumes that these registers
46 // can be used without saving upon entry to the method,
47 // but that they must be saved at call sites.
48 //
49 // SOE = Save-On-Entry: The register allocator assumes that these registers
50 // must be saved before using them upon entry to the
51 // method, but they do not need to be saved at call
52 // sites.
53 //
54 // AS = Always-Save: The register allocator assumes that these registers
55 // must be saved before using them upon entry to the
56 // method, & that they must be saved at call sites.
57 //
58 // Ideal Register Type is used to determine how to save & restore a
59 // register. Op_RegI will get spilled with LoadI/StoreI, Op_RegP will get
60 // spilled with LoadP/StoreP. If the register supports both, use Op_RegI.
61 //
62 // The encoding number is the actual bit-pattern placed into the opcodes.
63
64 // We must define the 64 bit int registers in two 32 bit halves, the
65 // real lower register and a virtual upper half register. upper halves
66 // are used by the register allocator but are not actually supplied as
67 // operands to memory ops.
68 //
69 // follow the C1 compiler in making registers
70 //
71 // r0-r7,r10-r26 volatile (caller save)
72 // r27-r32 system (no save, no allocate)
73 // r8-r9 invisible to the allocator (so we can use them as scratch regs)
74 //
75 // as regards Java usage. we don't use any callee save registers
76 // because this makes it difficult to de-optimise a frame (see comment
77 // in x86 implementation of Deoptimization::unwind_callee_save_values)
78 //
79
80 // General Registers
81
82 reg_def R0 ( SOC, SOC, Op_RegI, 0, r0->as_VMReg() );
83 reg_def R0_H ( SOC, SOC, Op_RegI, 0, r0->as_VMReg()->next() );
84 reg_def R1 ( SOC, SOC, Op_RegI, 1, r1->as_VMReg() );
85 reg_def R1_H ( SOC, SOC, Op_RegI, 1, r1->as_VMReg()->next() );
86 reg_def R2 ( SOC, SOC, Op_RegI, 2, r2->as_VMReg() );
87 reg_def R2_H ( SOC, SOC, Op_RegI, 2, r2->as_VMReg()->next() );
88 reg_def R3 ( SOC, SOC, Op_RegI, 3, r3->as_VMReg() );
89 reg_def R3_H ( SOC, SOC, Op_RegI, 3, r3->as_VMReg()->next() );
90 reg_def R4 ( SOC, SOC, Op_RegI, 4, r4->as_VMReg() );
91 reg_def R4_H ( SOC, SOC, Op_RegI, 4, r4->as_VMReg()->next() );
92 reg_def R5 ( SOC, SOC, Op_RegI, 5, r5->as_VMReg() );
93 reg_def R5_H ( SOC, SOC, Op_RegI, 5, r5->as_VMReg()->next() );
94 reg_def R6 ( SOC, SOC, Op_RegI, 6, r6->as_VMReg() );
95 reg_def R6_H ( SOC, SOC, Op_RegI, 6, r6->as_VMReg()->next() );
96 reg_def R7 ( SOC, SOC, Op_RegI, 7, r7->as_VMReg() );
97 reg_def R7_H ( SOC, SOC, Op_RegI, 7, r7->as_VMReg()->next() );
98 reg_def R10 ( SOC, SOC, Op_RegI, 10, r10->as_VMReg() );
99 reg_def R10_H ( SOC, SOC, Op_RegI, 10, r10->as_VMReg()->next());
100 reg_def R11 ( SOC, SOC, Op_RegI, 11, r11->as_VMReg() );
101 reg_def R11_H ( SOC, SOC, Op_RegI, 11, r11->as_VMReg()->next());
102 reg_def R12 ( SOC, SOC, Op_RegI, 12, r12->as_VMReg() );
103 reg_def R12_H ( SOC, SOC, Op_RegI, 12, r12->as_VMReg()->next());
104 reg_def R13 ( SOC, SOC, Op_RegI, 13, r13->as_VMReg() );
105 reg_def R13_H ( SOC, SOC, Op_RegI, 13, r13->as_VMReg()->next());
106 reg_def R14 ( SOC, SOC, Op_RegI, 14, r14->as_VMReg() );
107 reg_def R14_H ( SOC, SOC, Op_RegI, 14, r14->as_VMReg()->next());
108 reg_def R15 ( SOC, SOC, Op_RegI, 15, r15->as_VMReg() );
109 reg_def R15_H ( SOC, SOC, Op_RegI, 15, r15->as_VMReg()->next());
110 reg_def R16 ( SOC, SOC, Op_RegI, 16, r16->as_VMReg() );
111 reg_def R16_H ( SOC, SOC, Op_RegI, 16, r16->as_VMReg()->next());
112 reg_def R17 ( SOC, SOC, Op_RegI, 17, r17->as_VMReg() );
113 reg_def R17_H ( SOC, SOC, Op_RegI, 17, r17->as_VMReg()->next());
114 reg_def R18 ( SOC, SOC, Op_RegI, 18, r18->as_VMReg() );
115 reg_def R18_H ( SOC, SOC, Op_RegI, 18, r18->as_VMReg()->next());
116 reg_def R19 ( SOC, SOE, Op_RegI, 19, r19->as_VMReg() );
117 reg_def R19_H ( SOC, SOE, Op_RegI, 19, r19->as_VMReg()->next());
118 reg_def R20 ( SOC, SOE, Op_RegI, 20, r20->as_VMReg() ); // caller esp
119 reg_def R20_H ( SOC, SOE, Op_RegI, 20, r20->as_VMReg()->next());
120 reg_def R21 ( SOC, SOE, Op_RegI, 21, r21->as_VMReg() );
121 reg_def R21_H ( SOC, SOE, Op_RegI, 21, r21->as_VMReg()->next());
122 reg_def R22 ( SOC, SOE, Op_RegI, 22, r22->as_VMReg() );
123 reg_def R22_H ( SOC, SOE, Op_RegI, 22, r22->as_VMReg()->next());
124 reg_def R23 ( SOC, SOE, Op_RegI, 23, r23->as_VMReg() );
125 reg_def R23_H ( SOC, SOE, Op_RegI, 23, r23->as_VMReg()->next());
126 reg_def R24 ( SOC, SOE, Op_RegI, 24, r24->as_VMReg() );
127 reg_def R24_H ( SOC, SOE, Op_RegI, 24, r24->as_VMReg()->next());
128 reg_def R25 ( SOC, SOE, Op_RegI, 25, r25->as_VMReg() );
129 reg_def R25_H ( SOC, SOE, Op_RegI, 25, r25->as_VMReg()->next());
130 reg_def R26 ( SOC, SOE, Op_RegI, 26, r26->as_VMReg() );
131 reg_def R26_H ( SOC, SOE, Op_RegI, 26, r26->as_VMReg()->next());
132 reg_def R27 ( NS, SOE, Op_RegI, 27, r27->as_VMReg() ); // heapbase
133 reg_def R27_H ( NS, SOE, Op_RegI, 27, r27->as_VMReg()->next());
134 reg_def R28 ( NS, SOE, Op_RegI, 28, r28->as_VMReg() ); // thread
135 reg_def R28_H ( NS, SOE, Op_RegI, 28, r28->as_VMReg()->next());
136 reg_def R29 ( NS, NS, Op_RegI, 29, r29->as_VMReg() ); // fp
137 reg_def R29_H ( NS, NS, Op_RegI, 29, r29->as_VMReg()->next());
138 reg_def R30 ( NS, NS, Op_RegI, 30, r30->as_VMReg() ); // lr
139 reg_def R30_H ( NS, NS, Op_RegI, 30, r30->as_VMReg()->next());
140 reg_def R31 ( NS, NS, Op_RegI, 31, r31_sp->as_VMReg() ); // sp
141 reg_def R31_H ( NS, NS, Op_RegI, 31, r31_sp->as_VMReg()->next());
142
143 // ----------------------------
144 // Float/Double Registers
145 // ----------------------------
146
147 // Double Registers
148
149 // The rules of ADL require that double registers be defined in pairs.
150 // Each pair must be two 32-bit values, but not necessarily a pair of
151 // single float registers. In each pair, ADLC-assigned register numbers
152 // must be adjacent, with the lower number even. Finally, when the
153 // CPU stores such a register pair to memory, the word associated with
154 // the lower ADLC-assigned number must be stored to the lower address.
155
156 // AArch64 has 32 floating-point registers. Each can store a vector of
157 // single or double precision floating-point values up to 8 * 32
158 // floats, 4 * 64 bit floats or 2 * 128 bit floats. We currently only
159 // use the first float or double element of the vector.
160
161 // for Java use float registers v0-v15 are always save on call whereas
162 // the platform ABI treats v8-v15 as callee save). float registers
163 // v16-v31 are SOC as per the platform spec
164
165 reg_def V0 ( SOC, SOC, Op_RegF, 0, v0->as_VMReg() );
166 reg_def V0_H ( SOC, SOC, Op_RegF, 0, v0->as_VMReg()->next() );
167 reg_def V0_J ( SOC, SOC, Op_RegF, 0, v0->as_VMReg()->next(2) );
168 reg_def V0_K ( SOC, SOC, Op_RegF, 0, v0->as_VMReg()->next(3) );
169
170 reg_def V1 ( SOC, SOC, Op_RegF, 1, v1->as_VMReg() );
171 reg_def V1_H ( SOC, SOC, Op_RegF, 1, v1->as_VMReg()->next() );
172 reg_def V1_J ( SOC, SOC, Op_RegF, 1, v1->as_VMReg()->next(2) );
173 reg_def V1_K ( SOC, SOC, Op_RegF, 1, v1->as_VMReg()->next(3) );
174
175 reg_def V2 ( SOC, SOC, Op_RegF, 2, v2->as_VMReg() );
176 reg_def V2_H ( SOC, SOC, Op_RegF, 2, v2->as_VMReg()->next() );
177 reg_def V2_J ( SOC, SOC, Op_RegF, 2, v2->as_VMReg()->next(2) );
178 reg_def V2_K ( SOC, SOC, Op_RegF, 2, v2->as_VMReg()->next(3) );
179
180 reg_def V3 ( SOC, SOC, Op_RegF, 3, v3->as_VMReg() );
181 reg_def V3_H ( SOC, SOC, Op_RegF, 3, v3->as_VMReg()->next() );
182 reg_def V3_J ( SOC, SOC, Op_RegF, 3, v3->as_VMReg()->next(2) );
183 reg_def V3_K ( SOC, SOC, Op_RegF, 3, v3->as_VMReg()->next(3) );
184
185 reg_def V4 ( SOC, SOC, Op_RegF, 4, v4->as_VMReg() );
186 reg_def V4_H ( SOC, SOC, Op_RegF, 4, v4->as_VMReg()->next() );
187 reg_def V4_J ( SOC, SOC, Op_RegF, 4, v4->as_VMReg()->next(2) );
188 reg_def V4_K ( SOC, SOC, Op_RegF, 4, v4->as_VMReg()->next(3) );
189
190 reg_def V5 ( SOC, SOC, Op_RegF, 5, v5->as_VMReg() );
191 reg_def V5_H ( SOC, SOC, Op_RegF, 5, v5->as_VMReg()->next() );
192 reg_def V5_J ( SOC, SOC, Op_RegF, 5, v5->as_VMReg()->next(2) );
193 reg_def V5_K ( SOC, SOC, Op_RegF, 5, v5->as_VMReg()->next(3) );
194
195 reg_def V6 ( SOC, SOC, Op_RegF, 6, v6->as_VMReg() );
196 reg_def V6_H ( SOC, SOC, Op_RegF, 6, v6->as_VMReg()->next() );
197 reg_def V6_J ( SOC, SOC, Op_RegF, 6, v6->as_VMReg()->next(2) );
198 reg_def V6_K ( SOC, SOC, Op_RegF, 6, v6->as_VMReg()->next(3) );
199
200 reg_def V7 ( SOC, SOC, Op_RegF, 7, v7->as_VMReg() );
201 reg_def V7_H ( SOC, SOC, Op_RegF, 7, v7->as_VMReg()->next() );
202 reg_def V7_J ( SOC, SOC, Op_RegF, 7, v7->as_VMReg()->next(2) );
203 reg_def V7_K ( SOC, SOC, Op_RegF, 7, v7->as_VMReg()->next(3) );
204
205 reg_def V8 ( SOC, SOC, Op_RegF, 8, v8->as_VMReg() );
206 reg_def V8_H ( SOC, SOC, Op_RegF, 8, v8->as_VMReg()->next() );
207 reg_def V8_J ( SOC, SOC, Op_RegF, 8, v8->as_VMReg()->next(2) );
208 reg_def V8_K ( SOC, SOC, Op_RegF, 8, v8->as_VMReg()->next(3) );
209
210 reg_def V9 ( SOC, SOC, Op_RegF, 9, v9->as_VMReg() );
211 reg_def V9_H ( SOC, SOC, Op_RegF, 9, v9->as_VMReg()->next() );
212 reg_def V9_J ( SOC, SOC, Op_RegF, 9, v9->as_VMReg()->next(2) );
213 reg_def V9_K ( SOC, SOC, Op_RegF, 9, v9->as_VMReg()->next(3) );
214
215 reg_def V10 ( SOC, SOC, Op_RegF, 10, v10->as_VMReg() );
216 reg_def V10_H( SOC, SOC, Op_RegF, 10, v10->as_VMReg()->next() );
217 reg_def V10_J( SOC, SOC, Op_RegF, 10, v10->as_VMReg()->next(2));
218 reg_def V10_K( SOC, SOC, Op_RegF, 10, v10->as_VMReg()->next(3));
219
220 reg_def V11 ( SOC, SOC, Op_RegF, 11, v11->as_VMReg() );
221 reg_def V11_H( SOC, SOC, Op_RegF, 11, v11->as_VMReg()->next() );
222 reg_def V11_J( SOC, SOC, Op_RegF, 11, v11->as_VMReg()->next(2));
223 reg_def V11_K( SOC, SOC, Op_RegF, 11, v11->as_VMReg()->next(3));
224
225 reg_def V12 ( SOC, SOC, Op_RegF, 12, v12->as_VMReg() );
226 reg_def V12_H( SOC, SOC, Op_RegF, 12, v12->as_VMReg()->next() );
227 reg_def V12_J( SOC, SOC, Op_RegF, 12, v12->as_VMReg()->next(2));
228 reg_def V12_K( SOC, SOC, Op_RegF, 12, v12->as_VMReg()->next(3));
229
230 reg_def V13 ( SOC, SOC, Op_RegF, 13, v13->as_VMReg() );
231 reg_def V13_H( SOC, SOC, Op_RegF, 13, v13->as_VMReg()->next() );
232 reg_def V13_J( SOC, SOC, Op_RegF, 13, v13->as_VMReg()->next(2));
233 reg_def V13_K( SOC, SOC, Op_RegF, 13, v13->as_VMReg()->next(3));
234
235 reg_def V14 ( SOC, SOC, Op_RegF, 14, v14->as_VMReg() );
236 reg_def V14_H( SOC, SOC, Op_RegF, 14, v14->as_VMReg()->next() );
237 reg_def V14_J( SOC, SOC, Op_RegF, 14, v14->as_VMReg()->next(2));
238 reg_def V14_K( SOC, SOC, Op_RegF, 14, v14->as_VMReg()->next(3));
239
240 reg_def V15 ( SOC, SOC, Op_RegF, 15, v15->as_VMReg() );
241 reg_def V15_H( SOC, SOC, Op_RegF, 15, v15->as_VMReg()->next() );
242 reg_def V15_J( SOC, SOC, Op_RegF, 15, v15->as_VMReg()->next(2));
243 reg_def V15_K( SOC, SOC, Op_RegF, 15, v15->as_VMReg()->next(3));
244
245 reg_def V16 ( SOC, SOC, Op_RegF, 16, v16->as_VMReg() );
246 reg_def V16_H( SOC, SOC, Op_RegF, 16, v16->as_VMReg()->next() );
247 reg_def V16_J( SOC, SOC, Op_RegF, 16, v16->as_VMReg()->next(2));
248 reg_def V16_K( SOC, SOC, Op_RegF, 16, v16->as_VMReg()->next(3));
249
250 reg_def V17 ( SOC, SOC, Op_RegF, 17, v17->as_VMReg() );
251 reg_def V17_H( SOC, SOC, Op_RegF, 17, v17->as_VMReg()->next() );
252 reg_def V17_J( SOC, SOC, Op_RegF, 17, v17->as_VMReg()->next(2));
253 reg_def V17_K( SOC, SOC, Op_RegF, 17, v17->as_VMReg()->next(3));
254
255 reg_def V18 ( SOC, SOC, Op_RegF, 18, v18->as_VMReg() );
256 reg_def V18_H( SOC, SOC, Op_RegF, 18, v18->as_VMReg()->next() );
257 reg_def V18_J( SOC, SOC, Op_RegF, 18, v18->as_VMReg()->next(2));
258 reg_def V18_K( SOC, SOC, Op_RegF, 18, v18->as_VMReg()->next(3));
259
260 reg_def V19 ( SOC, SOC, Op_RegF, 19, v19->as_VMReg() );
261 reg_def V19_H( SOC, SOC, Op_RegF, 19, v19->as_VMReg()->next() );
262 reg_def V19_J( SOC, SOC, Op_RegF, 19, v19->as_VMReg()->next(2));
263 reg_def V19_K( SOC, SOC, Op_RegF, 19, v19->as_VMReg()->next(3));
264
265 reg_def V20 ( SOC, SOC, Op_RegF, 20, v20->as_VMReg() );
266 reg_def V20_H( SOC, SOC, Op_RegF, 20, v20->as_VMReg()->next() );
267 reg_def V20_J( SOC, SOC, Op_RegF, 20, v20->as_VMReg()->next(2));
268 reg_def V20_K( SOC, SOC, Op_RegF, 20, v20->as_VMReg()->next(3));
269
270 reg_def V21 ( SOC, SOC, Op_RegF, 21, v21->as_VMReg() );
271 reg_def V21_H( SOC, SOC, Op_RegF, 21, v21->as_VMReg()->next() );
272 reg_def V21_J( SOC, SOC, Op_RegF, 21, v21->as_VMReg()->next(2));
273 reg_def V21_K( SOC, SOC, Op_RegF, 21, v21->as_VMReg()->next(3));
274
275 reg_def V22 ( SOC, SOC, Op_RegF, 22, v22->as_VMReg() );
276 reg_def V22_H( SOC, SOC, Op_RegF, 22, v22->as_VMReg()->next() );
277 reg_def V22_J( SOC, SOC, Op_RegF, 22, v22->as_VMReg()->next(2));
278 reg_def V22_K( SOC, SOC, Op_RegF, 22, v22->as_VMReg()->next(3));
279
280 reg_def V23 ( SOC, SOC, Op_RegF, 23, v23->as_VMReg() );
281 reg_def V23_H( SOC, SOC, Op_RegF, 23, v23->as_VMReg()->next() );
282 reg_def V23_J( SOC, SOC, Op_RegF, 23, v23->as_VMReg()->next(2));
283 reg_def V23_K( SOC, SOC, Op_RegF, 23, v23->as_VMReg()->next(3));
284
285 reg_def V24 ( SOC, SOC, Op_RegF, 24, v24->as_VMReg() );
286 reg_def V24_H( SOC, SOC, Op_RegF, 24, v24->as_VMReg()->next() );
287 reg_def V24_J( SOC, SOC, Op_RegF, 24, v24->as_VMReg()->next(2));
288 reg_def V24_K( SOC, SOC, Op_RegF, 24, v24->as_VMReg()->next(3));
289
290 reg_def V25 ( SOC, SOC, Op_RegF, 25, v25->as_VMReg() );
291 reg_def V25_H( SOC, SOC, Op_RegF, 25, v25->as_VMReg()->next() );
292 reg_def V25_J( SOC, SOC, Op_RegF, 25, v25->as_VMReg()->next(2));
293 reg_def V25_K( SOC, SOC, Op_RegF, 25, v25->as_VMReg()->next(3));
294
295 reg_def V26 ( SOC, SOC, Op_RegF, 26, v26->as_VMReg() );
296 reg_def V26_H( SOC, SOC, Op_RegF, 26, v26->as_VMReg()->next() );
297 reg_def V26_J( SOC, SOC, Op_RegF, 26, v26->as_VMReg()->next(2));
298 reg_def V26_K( SOC, SOC, Op_RegF, 26, v26->as_VMReg()->next(3));
299
300 reg_def V27 ( SOC, SOC, Op_RegF, 27, v27->as_VMReg() );
301 reg_def V27_H( SOC, SOC, Op_RegF, 27, v27->as_VMReg()->next() );
302 reg_def V27_J( SOC, SOC, Op_RegF, 27, v27->as_VMReg()->next(2));
303 reg_def V27_K( SOC, SOC, Op_RegF, 27, v27->as_VMReg()->next(3));
304
305 reg_def V28 ( SOC, SOC, Op_RegF, 28, v28->as_VMReg() );
306 reg_def V28_H( SOC, SOC, Op_RegF, 28, v28->as_VMReg()->next() );
307 reg_def V28_J( SOC, SOC, Op_RegF, 28, v28->as_VMReg()->next(2));
308 reg_def V28_K( SOC, SOC, Op_RegF, 28, v28->as_VMReg()->next(3));
309
310 reg_def V29 ( SOC, SOC, Op_RegF, 29, v29->as_VMReg() );
311 reg_def V29_H( SOC, SOC, Op_RegF, 29, v29->as_VMReg()->next() );
312 reg_def V29_J( SOC, SOC, Op_RegF, 29, v29->as_VMReg()->next(2));
313 reg_def V29_K( SOC, SOC, Op_RegF, 29, v29->as_VMReg()->next(3));
314
315 reg_def V30 ( SOC, SOC, Op_RegF, 30, v30->as_VMReg() );
316 reg_def V30_H( SOC, SOC, Op_RegF, 30, v30->as_VMReg()->next() );
317 reg_def V30_J( SOC, SOC, Op_RegF, 30, v30->as_VMReg()->next(2));
318 reg_def V30_K( SOC, SOC, Op_RegF, 30, v30->as_VMReg()->next(3));
319
320 reg_def V31 ( SOC, SOC, Op_RegF, 31, v31->as_VMReg() );
321 reg_def V31_H( SOC, SOC, Op_RegF, 31, v31->as_VMReg()->next() );
322 reg_def V31_J( SOC, SOC, Op_RegF, 31, v31->as_VMReg()->next(2));
323 reg_def V31_K( SOC, SOC, Op_RegF, 31, v31->as_VMReg()->next(3));
324
325 // ----------------------------
326 // Special Registers
327 // ----------------------------
328
329 // the AArch64 CSPR status flag register is not directly acessible as
330 // instruction operand. the FPSR status flag register is a system
331 // register which can be written/read using MSR/MRS but again does not
332 // appear as an operand (a code identifying the FSPR occurs as an
333 // immediate value in the instruction).
334
335 reg_def RFLAGS(SOC, SOC, 0, 32, VMRegImpl::Bad());
336
337
338 // Specify priority of register selection within phases of register
339 // allocation. Highest priority is first. A useful heuristic is to
340 // give registers a low priority when they are required by machine
341 // instructions, like EAX and EDX on I486, and choose no-save registers
342 // before save-on-call, & save-on-call before save-on-entry. Registers
343 // which participate in fixed calling sequences should come last.
344 // Registers which are used as pairs must fall on an even boundary.
345
346 alloc_class chunk0(
347 // volatiles
348 R10, R10_H,
349 R11, R11_H,
350 R12, R12_H,
351 R13, R13_H,
352 R14, R14_H,
353 R15, R15_H,
354 R16, R16_H,
355 R17, R17_H,
356 R18, R18_H,
357
358 // arg registers
359 R0, R0_H,
360 R1, R1_H,
361 R2, R2_H,
362 R3, R3_H,
363 R4, R4_H,
364 R5, R5_H,
365 R6, R6_H,
366 R7, R7_H,
367
368 // non-volatiles
369 R19, R19_H,
370 R20, R20_H,
371 R21, R21_H,
372 R22, R22_H,
373 R23, R23_H,
374 R24, R24_H,
375 R25, R25_H,
376 R26, R26_H,
377
378 // non-allocatable registers
379
380 R27, R27_H, // heapbase
381 R28, R28_H, // thread
382 R29, R29_H, // fp
383 R30, R30_H, // lr
384 R31, R31_H, // sp
385 );
386
387 alloc_class chunk1(
388
389 // no save
390 V16, V16_H, V16_J, V16_K,
391 V17, V17_H, V17_J, V17_K,
392 V18, V18_H, V18_J, V18_K,
393 V19, V19_H, V19_J, V19_K,
394 V20, V20_H, V20_J, V20_K,
395 V21, V21_H, V21_J, V21_K,
396 V22, V22_H, V22_J, V22_K,
397 V23, V23_H, V23_J, V23_K,
398 V24, V24_H, V24_J, V24_K,
399 V25, V25_H, V25_J, V25_K,
400 V26, V26_H, V26_J, V26_K,
401 V27, V27_H, V27_J, V27_K,
402 V28, V28_H, V28_J, V28_K,
403 V29, V29_H, V29_J, V29_K,
404 V30, V30_H, V30_J, V30_K,
405 V31, V31_H, V31_J, V31_K,
406
407 // arg registers
408 V0, V0_H, V0_J, V0_K,
409 V1, V1_H, V1_J, V1_K,
410 V2, V2_H, V2_J, V2_K,
411 V3, V3_H, V3_J, V3_K,
412 V4, V4_H, V4_J, V4_K,
413 V5, V5_H, V5_J, V5_K,
414 V6, V6_H, V6_J, V6_K,
415 V7, V7_H, V7_J, V7_K,
416
417 // non-volatiles
418 V8, V8_H, V8_J, V8_K,
419 V9, V9_H, V9_J, V9_K,
420 V10, V10_H, V10_J, V10_K,
421 V11, V11_H, V11_J, V11_K,
422 V12, V12_H, V12_J, V12_K,
423 V13, V13_H, V13_J, V13_K,
424 V14, V14_H, V14_J, V14_K,
425 V15, V15_H, V15_J, V15_K,
426 );
427
428 alloc_class chunk2(RFLAGS);
429
430 //----------Architecture Description Register Classes--------------------------
431 // Several register classes are automatically defined based upon information in
432 // this architecture description.
433 // 1) reg_class inline_cache_reg ( /* as def'd in frame section */ )
434 // 2) reg_class compiler_method_oop_reg ( /* as def'd in frame section */ )
435 // 2) reg_class interpreter_method_oop_reg ( /* as def'd in frame section */ )
436 // 3) reg_class stack_slots( /* one chunk of stack-based "registers" */ )
437 //
438
439 // Class for all 32 bit integer registers -- excludes SP which will
440 // never be used as an integer register
441 reg_class any_reg32(
442 R0,
443 R1,
444 R2,
445 R3,
446 R4,
447 R5,
448 R6,
449 R7,
450 R10,
451 R11,
452 R12,
453 R13,
454 R14,
455 R15,
456 R16,
457 R17,
458 R18,
459 R19,
460 R20,
461 R21,
462 R22,
463 R23,
464 R24,
465 R25,
466 R26,
467 R27,
468 R28,
469 R29,
470 R30
471 );
472
473 // Singleton class for R0 int register
474 reg_class int_r0_reg(R0);
475
476 // Singleton class for R2 int register
477 reg_class int_r2_reg(R2);
478
479 // Singleton class for R3 int register
480 reg_class int_r3_reg(R3);
481
482 // Singleton class for R4 int register
483 reg_class int_r4_reg(R4);
484
485 // Class for all long integer registers (including RSP)
486 reg_class any_reg(
487 R0, R0_H,
488 R1, R1_H,
489 R2, R2_H,
490 R3, R3_H,
491 R4, R4_H,
492 R5, R5_H,
493 R6, R6_H,
494 R7, R7_H,
495 R10, R10_H,
496 R11, R11_H,
497 R12, R12_H,
498 R13, R13_H,
499 R14, R14_H,
500 R15, R15_H,
501 R16, R16_H,
502 R17, R17_H,
503 R18, R18_H,
504 R19, R19_H,
505 R20, R20_H,
506 R21, R21_H,
507 R22, R22_H,
508 R23, R23_H,
509 R24, R24_H,
510 R25, R25_H,
511 R26, R26_H,
512 R27, R27_H,
513 R28, R28_H,
514 R29, R29_H,
515 R30, R30_H,
516 R31, R31_H
517 );
518
519 // Class for all non-special integer registers
520 reg_class no_special_reg32(
521 R0,
522 R1,
523 R2,
524 R3,
525 R4,
526 R5,
527 R6,
528 R7,
529 R10,
530 R11,
531 R12, // rmethod
532 R13,
533 R14,
534 R15,
535 R16,
536 R17,
537 R18,
538 R19,
539 R20,
540 R21,
541 R22,
542 R23,
543 R24,
544 R25,
545 R26
546 /* R27, */ // heapbase
547 /* R28, */ // thread
548 /* R29, */ // fp
549 /* R30, */ // lr
550 /* R31 */ // sp
551 );
552
553 // Class for all non-special long integer registers
554 reg_class no_special_reg(
555 R0, R0_H,
556 R1, R1_H,
557 R2, R2_H,
558 R3, R3_H,
559 R4, R4_H,
560 R5, R5_H,
561 R6, R6_H,
562 R7, R7_H,
563 R10, R10_H,
564 R11, R11_H,
565 R12, R12_H, // rmethod
566 R13, R13_H,
567 R14, R14_H,
568 R15, R15_H,
569 R16, R16_H,
570 R17, R17_H,
571 R18, R18_H,
572 R19, R19_H,
573 R20, R20_H,
574 R21, R21_H,
575 R22, R22_H,
576 R23, R23_H,
577 R24, R24_H,
578 R25, R25_H,
579 R26, R26_H,
580 /* R27, R27_H, */ // heapbase
581 /* R28, R28_H, */ // thread
582 /* R29, R29_H, */ // fp
583 /* R30, R30_H, */ // lr
584 /* R31, R31_H */ // sp
585 );
586
587 // Class for 64 bit register r0
588 reg_class r0_reg(
589 R0, R0_H
590 );
591
592 // Class for 64 bit register r1
593 reg_class r1_reg(
594 R1, R1_H
595 );
596
597 // Class for 64 bit register r2
598 reg_class r2_reg(
599 R2, R2_H
600 );
601
602 // Class for 64 bit register r3
603 reg_class r3_reg(
604 R3, R3_H
605 );
606
607 // Class for 64 bit register r4
608 reg_class r4_reg(
609 R4, R4_H
610 );
611
612 // Class for 64 bit register r5
613 reg_class r5_reg(
614 R5, R5_H
615 );
616
617 // Class for 64 bit register r10
618 reg_class r10_reg(
619 R10, R10_H
620 );
621
622 // Class for 64 bit register r11
623 reg_class r11_reg(
624 R11, R11_H
625 );
626
627 // Class for method register
628 reg_class method_reg(
629 R12, R12_H
630 );
631
632 // Class for heapbase register
633 reg_class heapbase_reg(
634 R27, R27_H
635 );
636
637 // Class for thread register
638 reg_class thread_reg(
639 R28, R28_H
640 );
641
642 // Class for frame pointer register
643 reg_class fp_reg(
644 R29, R29_H
645 );
646
647 // Class for link register
648 reg_class lr_reg(
649 R30, R30_H
650 );
651
652 // Class for long sp register
653 reg_class sp_reg(
654 R31, R31_H
655 );
656
657 // Class for all pointer registers
658 reg_class ptr_reg(
659 R0, R0_H,
660 R1, R1_H,
661 R2, R2_H,
662 R3, R3_H,
663 R4, R4_H,
664 R5, R5_H,
665 R6, R6_H,
666 R7, R7_H,
667 R10, R10_H,
668 R11, R11_H,
669 R12, R12_H,
670 R13, R13_H,
671 R14, R14_H,
672 R15, R15_H,
673 R16, R16_H,
674 R17, R17_H,
675 R18, R18_H,
676 R19, R19_H,
677 R20, R20_H,
678 R21, R21_H,
679 R22, R22_H,
680 R23, R23_H,
681 R24, R24_H,
682 R25, R25_H,
683 R26, R26_H,
684 R27, R27_H,
685 R28, R28_H,
686 R29, R29_H,
687 R30, R30_H,
688 R31, R31_H
689 );
690
691 // Class for all non_special pointer registers
692 reg_class no_special_ptr_reg(
693 R0, R0_H,
694 R1, R1_H,
695 R2, R2_H,
696 R3, R3_H,
697 R4, R4_H,
698 R5, R5_H,
699 R6, R6_H,
700 R7, R7_H,
701 R10, R10_H,
702 R11, R11_H,
703 R12, R12_H,
704 R13, R13_H,
705 R14, R14_H,
706 R15, R15_H,
707 R16, R16_H,
708 R17, R17_H,
709 R18, R18_H,
710 R19, R19_H,
711 R20, R20_H,
712 R21, R21_H,
713 R22, R22_H,
714 R23, R23_H,
715 R24, R24_H,
716 R25, R25_H,
717 R26, R26_H,
718 /* R27, R27_H, */ // heapbase
719 /* R28, R28_H, */ // thread
720 /* R29, R29_H, */ // fp
721 /* R30, R30_H, */ // lr
722 /* R31, R31_H */ // sp
723 );
724
725 // Class for all float registers
726 reg_class float_reg(
727 V0,
728 V1,
729 V2,
730 V3,
731 V4,
732 V5,
733 V6,
734 V7,
735 V8,
736 V9,
737 V10,
738 V11,
739 V12,
740 V13,
741 V14,
742 V15,
743 V16,
744 V17,
745 V18,
746 V19,
747 V20,
748 V21,
749 V22,
750 V23,
751 V24,
752 V25,
753 V26,
754 V27,
755 V28,
756 V29,
757 V30,
758 V31
759 );
760
761 // Double precision float registers have virtual `high halves' that
762 // are needed by the allocator.
763 // Class for all double registers
764 reg_class double_reg(
765 V0, V0_H,
766 V1, V1_H,
767 V2, V2_H,
768 V3, V3_H,
769 V4, V4_H,
770 V5, V5_H,
771 V6, V6_H,
772 V7, V7_H,
773 V8, V8_H,
774 V9, V9_H,
775 V10, V10_H,
776 V11, V11_H,
777 V12, V12_H,
778 V13, V13_H,
779 V14, V14_H,
780 V15, V15_H,
781 V16, V16_H,
782 V17, V17_H,
783 V18, V18_H,
784 V19, V19_H,
785 V20, V20_H,
786 V21, V21_H,
787 V22, V22_H,
788 V23, V23_H,
789 V24, V24_H,
790 V25, V25_H,
791 V26, V26_H,
792 V27, V27_H,
793 V28, V28_H,
794 V29, V29_H,
795 V30, V30_H,
796 V31, V31_H
797 );
798
799 // Class for all 64bit vector registers
800 reg_class vectord_reg(
801 V0, V0_H,
802 V1, V1_H,
803 V2, V2_H,
804 V3, V3_H,
805 V4, V4_H,
806 V5, V5_H,
807 V6, V6_H,
808 V7, V7_H,
809 V8, V8_H,
810 V9, V9_H,
811 V10, V10_H,
812 V11, V11_H,
813 V12, V12_H,
814 V13, V13_H,
815 V14, V14_H,
816 V15, V15_H,
817 V16, V16_H,
818 V17, V17_H,
819 V18, V18_H,
820 V19, V19_H,
821 V20, V20_H,
822 V21, V21_H,
823 V22, V22_H,
824 V23, V23_H,
825 V24, V24_H,
826 V25, V25_H,
827 V26, V26_H,
828 V27, V27_H,
829 V28, V28_H,
830 V29, V29_H,
831 V30, V30_H,
832 V31, V31_H
833 );
834
835 // Class for all 128bit vector registers
836 reg_class vectorx_reg(
837 V0, V0_H, V0_J, V0_K,
838 V1, V1_H, V1_J, V1_K,
839 V2, V2_H, V2_J, V2_K,
840 V3, V3_H, V3_J, V3_K,
841 V4, V4_H, V4_J, V4_K,
842 V5, V5_H, V5_J, V5_K,
843 V6, V6_H, V6_J, V6_K,
844 V7, V7_H, V7_J, V7_K,
845 V8, V8_H, V8_J, V8_K,
846 V9, V9_H, V9_J, V9_K,
847 V10, V10_H, V10_J, V10_K,
848 V11, V11_H, V11_J, V11_K,
849 V12, V12_H, V12_J, V12_K,
850 V13, V13_H, V13_J, V13_K,
851 V14, V14_H, V14_J, V14_K,
852 V15, V15_H, V15_J, V15_K,
853 V16, V16_H, V16_J, V16_K,
854 V17, V17_H, V17_J, V17_K,
855 V18, V18_H, V18_J, V18_K,
856 V19, V19_H, V19_J, V19_K,
857 V20, V20_H, V20_J, V20_K,
858 V21, V21_H, V21_J, V21_K,
859 V22, V22_H, V22_J, V22_K,
860 V23, V23_H, V23_J, V23_K,
861 V24, V24_H, V24_J, V24_K,
862 V25, V25_H, V25_J, V25_K,
863 V26, V26_H, V26_J, V26_K,
864 V27, V27_H, V27_J, V27_K,
865 V28, V28_H, V28_J, V28_K,
866 V29, V29_H, V29_J, V29_K,
867 V30, V30_H, V30_J, V30_K,
868 V31, V31_H, V31_J, V31_K
869 );
870
871 // Class for 128 bit register v0
872 reg_class v0_reg(
873 V0, V0_H
874 );
875
876 // Class for 128 bit register v1
877 reg_class v1_reg(
878 V1, V1_H
879 );
880
881 // Class for 128 bit register v2
882 reg_class v2_reg(
883 V2, V2_H
884 );
885
886 // Class for 128 bit register v3
887 reg_class v3_reg(
888 V3, V3_H
889 );
890
891 // Singleton class for condition codes
892 reg_class int_flags(RFLAGS);
893
894 %}
895
896 //----------DEFINITION BLOCK---------------------------------------------------
897 // Define name --> value mappings to inform the ADLC of an integer valued name
898 // Current support includes integer values in the range [0, 0x7FFFFFFF]
899 // Format:
900 // int_def <name> ( <int_value>, <expression>);
901 // Generated Code in ad_<arch>.hpp
902 // #define <name> (<expression>)
903 // // value == <int_value>
904 // Generated code in ad_<arch>.cpp adlc_verification()
905 // assert( <name> == <int_value>, "Expect (<expression>) to equal <int_value>");
906 //
907
908 // we follow the ppc-aix port in using a simple cost model which ranks
909 // register operations as cheap, memory ops as more expensive and
910 // branches as most expensive. the first two have a low as well as a
911 // normal cost. huge cost appears to be a way of saying don't do
912 // something
913
914 definitions %{
915 // The default cost (of a register move instruction).
916 int_def INSN_COST ( 100, 100);
917 int_def BRANCH_COST ( 200, 2 * INSN_COST);
918 int_def CALL_COST ( 200, 2 * INSN_COST);
919 int_def VOLATILE_REF_COST ( 1000, 10 * INSN_COST);
920 %}
921
922
923 //----------SOURCE BLOCK-------------------------------------------------------
924 // This is a block of C++ code which provides values, functions, and
925 // definitions necessary in the rest of the architecture description
926
927 source_hpp %{
928
929 #include "opto/addnode.hpp"
930 #if INCLUDE_ALL_GCS
931 #include "shenandoahBarrierSetAssembler_aarch64.hpp"
932 #endif
933
934 class CallStubImpl {
935
936 //--------------------------------------------------------------
937 //---< Used for optimization in Compile::shorten_branches >---
938 //--------------------------------------------------------------
939
940 public:
941 // Size of call trampoline stub.
942 static uint size_call_trampoline() {
943 return 0; // no call trampolines on this platform
944 }
945
946 // number of relocations needed by a call trampoline stub
947 static uint reloc_call_trampoline() {
948 return 0; // no call trampolines on this platform
949 }
950 };
951
952 class HandlerImpl {
953
954 public:
955
956 static int emit_exception_handler(CodeBuffer &cbuf);
957 static int emit_deopt_handler(CodeBuffer& cbuf);
958
959 static uint size_exception_handler() {
960 return MacroAssembler::far_branch_size();
961 }
962
963 static uint size_deopt_handler() {
964 // count one adr and one far branch instruction
965 // return 4 * NativeInstruction::instruction_size;
966 return NativeInstruction::instruction_size + MacroAssembler::far_branch_size();
967 }
968 };
969
970 bool is_CAS(int opcode);
971
972 // predicates controlling emit of ldr<x>/ldar<x> and associated dmb
973
974 bool unnecessary_acquire(const Node *barrier);
975 bool needs_acquiring_load(const Node *load);
976
977 // predicates controlling emit of str<x>/stlr<x> and associated dmbs
978
979 bool unnecessary_release(const Node *barrier);
980 bool unnecessary_volatile(const Node *barrier);
981 bool needs_releasing_store(const Node *store);
982
983 // predicate controlling translation of CompareAndSwapX
984 bool needs_acquiring_load_exclusive(const Node *load);
985
986 // predicate controlling translation of StoreCM
987 bool unnecessary_storestore(const Node *storecm);
988
989 // predicate controlling addressing modes
990 bool size_fits_all_mem_uses(AddPNode* addp, int shift);
991 %}
992
993 source %{
994
995 // Optimizaton of volatile gets and puts
996 // -------------------------------------
997 //
998 // AArch64 has ldar<x> and stlr<x> instructions which we can safely
999 // use to implement volatile reads and writes. For a volatile read
1000 // we simply need
1001 //
1002 // ldar<x>
1003 //
1004 // and for a volatile write we need
1005 //
1006 // stlr<x>
1007 //
1008 // Alternatively, we can implement them by pairing a normal
1009 // load/store with a memory barrier. For a volatile read we need
1010 //
1011 // ldr<x>
1012 // dmb ishld
1013 //
1014 // for a volatile write
1015 //
1016 // dmb ish
1017 // str<x>
1018 // dmb ish
1019 //
1020 // We can also use ldaxr and stlxr to implement compare and swap CAS
1021 // sequences. These are normally translated to an instruction
1022 // sequence like the following
1023 //
1024 // dmb ish
1025 // retry:
1026 // ldxr<x> rval raddr
1027 // cmp rval rold
1028 // b.ne done
1029 // stlxr<x> rval, rnew, rold
1030 // cbnz rval retry
1031 // done:
1032 // cset r0, eq
1033 // dmb ishld
1034 //
1035 // Note that the exclusive store is already using an stlxr
1036 // instruction. That is required to ensure visibility to other
1037 // threads of the exclusive write (assuming it succeeds) before that
1038 // of any subsequent writes.
1039 //
1040 // The following instruction sequence is an improvement on the above
1041 //
1042 // retry:
1043 // ldaxr<x> rval raddr
1044 // cmp rval rold
1045 // b.ne done
1046 // stlxr<x> rval, rnew, rold
1047 // cbnz rval retry
1048 // done:
1049 // cset r0, eq
1050 //
1051 // We don't need the leading dmb ish since the stlxr guarantees
1052 // visibility of prior writes in the case that the swap is
1053 // successful. Crucially we don't have to worry about the case where
1054 // the swap is not successful since no valid program should be
1055 // relying on visibility of prior changes by the attempting thread
1056 // in the case where the CAS fails.
1057 //
1058 // Similarly, we don't need the trailing dmb ishld if we substitute
1059 // an ldaxr instruction since that will provide all the guarantees we
1060 // require regarding observation of changes made by other threads
1061 // before any change to the CAS address observed by the load.
1062 //
1063 // In order to generate the desired instruction sequence we need to
1064 // be able to identify specific 'signature' ideal graph node
1065 // sequences which i) occur as a translation of a volatile reads or
1066 // writes or CAS operations and ii) do not occur through any other
1067 // translation or graph transformation. We can then provide
1068 // alternative aldc matching rules which translate these node
1069 // sequences to the desired machine code sequences. Selection of the
1070 // alternative rules can be implemented by predicates which identify
1071 // the relevant node sequences.
1072 //
1073 // The ideal graph generator translates a volatile read to the node
1074 // sequence
1075 //
1076 // LoadX[mo_acquire]
1077 // MemBarAcquire
1078 //
1079 // As a special case when using the compressed oops optimization we
1080 // may also see this variant
1081 //
1082 // LoadN[mo_acquire]
1083 // DecodeN
1084 // MemBarAcquire
1085 //
1086 // A volatile write is translated to the node sequence
1087 //
1088 // MemBarRelease
1089 // StoreX[mo_release] {CardMark}-optional
1090 // MemBarVolatile
1091 //
1092 // n.b. the above node patterns are generated with a strict
1093 // 'signature' configuration of input and output dependencies (see
1094 // the predicates below for exact details). The card mark may be as
1095 // simple as a few extra nodes or, in a few GC configurations, may
1096 // include more complex control flow between the leading and
1097 // trailing memory barriers. However, whatever the card mark
1098 // configuration these signatures are unique to translated volatile
1099 // reads/stores -- they will not appear as a result of any other
1100 // bytecode translation or inlining nor as a consequence of
1101 // optimizing transforms.
1102 //
1103 // We also want to catch inlined unsafe volatile gets and puts and
1104 // be able to implement them using either ldar<x>/stlr<x> or some
1105 // combination of ldr<x>/stlr<x> and dmb instructions.
1106 //
1107 // Inlined unsafe volatiles puts manifest as a minor variant of the
1108 // normal volatile put node sequence containing an extra cpuorder
1109 // membar
1110 //
1111 // MemBarRelease
1112 // MemBarCPUOrder
1113 // StoreX[mo_release] {CardMark}-optional
1114 // MemBarVolatile
1115 //
1116 // n.b. as an aside, the cpuorder membar is not itself subject to
1117 // matching and translation by adlc rules. However, the rule
1118 // predicates need to detect its presence in order to correctly
1119 // select the desired adlc rules.
1120 //
1121 // Inlined unsafe volatile gets manifest as a somewhat different
1122 // node sequence to a normal volatile get
1123 //
1124 // MemBarCPUOrder
1125 // || \\
1126 // MemBarAcquire LoadX[mo_acquire]
1127 // ||
1128 // MemBarCPUOrder
1129 //
1130 // In this case the acquire membar does not directly depend on the
1131 // load. However, we can be sure that the load is generated from an
1132 // inlined unsafe volatile get if we see it dependent on this unique
1133 // sequence of membar nodes. Similarly, given an acquire membar we
1134 // can know that it was added because of an inlined unsafe volatile
1135 // get if it is fed and feeds a cpuorder membar and if its feed
1136 // membar also feeds an acquiring load.
1137 //
1138 // Finally an inlined (Unsafe) CAS operation is translated to the
1139 // following ideal graph
1140 //
1141 // MemBarRelease
1142 // MemBarCPUOrder
1143 // CompareAndSwapX {CardMark}-optional
1144 // MemBarCPUOrder
1145 // MemBarAcquire
1146 //
1147 // So, where we can identify these volatile read and write
1148 // signatures we can choose to plant either of the above two code
1149 // sequences. For a volatile read we can simply plant a normal
1150 // ldr<x> and translate the MemBarAcquire to a dmb. However, we can
1151 // also choose to inhibit translation of the MemBarAcquire and
1152 // inhibit planting of the ldr<x>, instead planting an ldar<x>.
1153 //
1154 // When we recognise a volatile store signature we can choose to
1155 // plant at a dmb ish as a translation for the MemBarRelease, a
1156 // normal str<x> and then a dmb ish for the MemBarVolatile.
1157 // Alternatively, we can inhibit translation of the MemBarRelease
1158 // and MemBarVolatile and instead plant a simple stlr<x>
1159 // instruction.
1160 //
1161 // when we recognise a CAS signature we can choose to plant a dmb
1162 // ish as a translation for the MemBarRelease, the conventional
1163 // macro-instruction sequence for the CompareAndSwap node (which
1164 // uses ldxr<x>) and then a dmb ishld for the MemBarAcquire.
1165 // Alternatively, we can elide generation of the dmb instructions
1166 // and plant the alternative CompareAndSwap macro-instruction
1167 // sequence (which uses ldaxr<x>).
1168 //
1169 // Of course, the above only applies when we see these signature
1170 // configurations. We still want to plant dmb instructions in any
1171 // other cases where we may see a MemBarAcquire, MemBarRelease or
1172 // MemBarVolatile. For example, at the end of a constructor which
1173 // writes final/volatile fields we will see a MemBarRelease
1174 // instruction and this needs a 'dmb ish' lest we risk the
1175 // constructed object being visible without making the
1176 // final/volatile field writes visible.
1177 //
1178 // n.b. the translation rules below which rely on detection of the
1179 // volatile signatures and insert ldar<x> or stlr<x> are failsafe.
1180 // If we see anything other than the signature configurations we
1181 // always just translate the loads and stores to ldr<x> and str<x>
1182 // and translate acquire, release and volatile membars to the
1183 // relevant dmb instructions.
1184 //
1185
1186 // is_CAS(int opcode)
1187 //
1188 // return true if opcode is one of the possible CompareAndSwapX
1189 // values otherwise false.
1190
1191 bool is_CAS(int opcode)
1192 {
1193 switch(opcode) {
1194 // We handle these
1195 case Op_CompareAndSwapI:
1196 case Op_CompareAndSwapL:
1197 case Op_CompareAndSwapP:
1198 case Op_CompareAndSwapN:
1199 case Op_GetAndSetI:
1200 case Op_GetAndSetL:
1201 case Op_GetAndSetP:
1202 case Op_GetAndSetN:
1203 case Op_GetAndAddI:
1204 case Op_GetAndAddL:
1205 return true;
1206 default:
1207 return false;
1208 }
1209 }
1210
1211 // predicates controlling emit of ldr<x>/ldar<x> and associated dmb
1212
1213 bool unnecessary_acquire(const Node *barrier)
1214 {
1215 assert(barrier->is_MemBar(), "expecting a membar");
1216
1217 if (UseBarriersForVolatile) {
1218 // we need to plant a dmb
1219 return false;
1220 }
1221
1222 MemBarNode* mb = barrier->as_MemBar();
1223
1224 if (mb->trailing_load()) {
1225 return true;
1226 }
1227
1228 if (mb->trailing_load_store()) {
1229 Node* load_store = mb->in(MemBarNode::Precedent);
1230 assert(load_store->is_LoadStore(), "unexpected graph shape");
1231 return is_CAS(load_store->Opcode());
1232 }
1233
1234 return false;
1235 }
1236
1237 bool needs_acquiring_load(const Node *n)
1238 {
1239 assert(n->is_Load(), "expecting a load");
1240 if (UseBarriersForVolatile) {
1241 // we use a normal load and a dmb
1242 return false;
1243 }
1244
1245 LoadNode *ld = n->as_Load();
1246
1247 return ld->is_acquire();
1248 }
1249
1250 bool unnecessary_release(const Node *n)
1251 {
1252 assert((n->is_MemBar() &&
1253 n->Opcode() == Op_MemBarRelease),
1254 "expecting a release membar");
1255
1256 if (UseBarriersForVolatile) {
1257 // we need to plant a dmb
1258 return false;
1259 }
1260
1261 MemBarNode *barrier = n->as_MemBar();
1262
1263 if (!barrier->leading()) {
1264 return false;
1265 } else {
1266 Node* trailing = barrier->trailing_membar();
1267 MemBarNode* trailing_mb = trailing->as_MemBar();
1268 assert(trailing_mb->trailing(), "Not a trailing membar?");
1269 assert(trailing_mb->leading_membar() == n, "inconsistent leading/trailing membars");
1270
1271 Node* mem = trailing_mb->in(MemBarNode::Precedent);
1272 if (mem->is_Store()) {
1273 assert(mem->as_Store()->is_release(), "");
1274 assert(trailing_mb->Opcode() == Op_MemBarVolatile, "");
1275 return true;
1276 } else {
1277 assert(mem->is_LoadStore(), "");
1278 assert(trailing_mb->Opcode() == Op_MemBarAcquire, "");
1279 return is_CAS(mem->Opcode());
1280 }
1281 }
1282
1283 return false;
1284 }
1285
1286 bool unnecessary_volatile(const Node *n)
1287 {
1288 // assert n->is_MemBar();
1289 if (UseBarriersForVolatile) {
1290 // we need to plant a dmb
1291 return false;
1292 }
1293
1294 MemBarNode *mbvol = n->as_MemBar();
1295
1296 bool release = mbvol->trailing_store();
1297 assert(!release || (mbvol->in(MemBarNode::Precedent)->is_Store() && mbvol->in(MemBarNode::Precedent)->as_Store()->is_release()), "");
1298 #ifdef ASSERT
1299 if (release) {
1300 Node* leading = mbvol->leading_membar();
1301 assert(leading->Opcode() == Op_MemBarRelease, "");
1302 assert(leading->as_MemBar()->leading_store(), "");
1303 assert(leading->as_MemBar()->trailing_membar() == mbvol, "");
1304 }
1305 #endif
1306
1307 return release;
1308 }
1309
1310 // predicates controlling emit of str<x>/stlr<x> and associated dmbs
1311
1312 bool needs_releasing_store(const Node *n)
1313 {
1314 // assert n->is_Store();
1315 if (UseBarriersForVolatile) {
1316 // we use a normal store and dmb combination
1317 return false;
1318 }
1319
1320 StoreNode *st = n->as_Store();
1321
1322 return st->trailing_membar() != NULL;
1323 }
1324
1325 // predicate controlling translation of CAS
1326 //
1327 // returns true if CAS needs to use an acquiring load otherwise false
1328
1329 bool needs_acquiring_load_exclusive(const Node *n)
1330 {
1331 assert(is_CAS(n->Opcode()), "expecting a compare and swap");
1332 if (UseBarriersForVolatile) {
1333 return false;
1334 }
1335
1336 LoadStoreNode* ldst = n->as_LoadStore();
1337 assert(ldst->trailing_membar() != NULL, "expected trailing membar");
1338
1339 // so we can just return true here
1340 return true;
1341 }
1342
1343 // predicate controlling translation of StoreCM
1344 //
1345 // returns true if a StoreStore must precede the card write otherwise
1346 // false
1347
1348 bool unnecessary_storestore(const Node *storecm)
1349 {
1350 assert(storecm->Opcode() == Op_StoreCM, "expecting a StoreCM");
1351
1352 // we need to generate a dmb ishst between an object put and the
1353 // associated card mark when we are using CMS without conditional
1354 // card marking
1355
1356 if (UseConcMarkSweepGC && !UseCondCardMark) {
1357 return false;
1358 }
1359
1360 // a storestore is unnecesary in all other cases
1361
1362 return true;
1363 }
1364
1365
1366 #define __ _masm.
1367
1368 // advance declaratuons for helper functions to convert register
1369 // indices to register objects
1370
1371 // the ad file has to provide implementations of certain methods
1372 // expected by the generic code
1373 //
1374 // REQUIRED FUNCTIONALITY
1375
1376 //=============================================================================
1377
1378 // !!!!! Special hack to get all types of calls to specify the byte offset
1379 // from the start of the call to the point where the return address
1380 // will point.
1381
1382 int MachCallStaticJavaNode::ret_addr_offset()
1383 {
1384 // call should be a simple bl
1385 // unless this is a method handle invoke in which case it is
1386 // mov(rfp, sp), bl, mov(sp, rfp)
1387 int off = 4;
1388 if (_method_handle_invoke) {
1389 off += 4;
1390 }
1391 return off;
1392 }
1393
1394 int MachCallDynamicJavaNode::ret_addr_offset()
1395 {
1396 return 16; // movz, movk, movk, bl
1397 }
1398
1399 int MachCallRuntimeNode::ret_addr_offset() {
1400 // for generated stubs the call will be
1401 // bl(addr)
1402 // for real runtime callouts it will be six instructions
1403 // see aarch64_enc_java_to_runtime
1404 // adr(rscratch2, retaddr)
1405 // lea(rscratch1, RuntimeAddress(addr)
1406 // stp(zr, rscratch2, Address(__ pre(sp, -2 * wordSize)))
1407 // blr(rscratch1)
1408 CodeBlob *cb = CodeCache::find_blob(_entry_point);
1409 if (cb) {
1410 return MacroAssembler::far_branch_size();
1411 } else {
1412 return 6 * NativeInstruction::instruction_size;
1413 }
1414 }
1415
1416 // Indicate if the safepoint node needs the polling page as an input
1417
1418 // the shared code plants the oop data at the start of the generated
1419 // code for the safepoint node and that needs ot be at the load
1420 // instruction itself. so we cannot plant a mov of the safepoint poll
1421 // address followed by a load. setting this to true means the mov is
1422 // scheduled as a prior instruction. that's better for scheduling
1423 // anyway.
1424
1425 bool SafePointNode::needs_polling_address_input()
1426 {
1427 return true;
1428 }
1429
1430 //=============================================================================
1431
1432 #ifndef PRODUCT
1433 void MachBreakpointNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
1434 st->print("BREAKPOINT");
1435 }
1436 #endif
1437
1438 void MachBreakpointNode::emit(CodeBuffer &cbuf, PhaseRegAlloc *ra_) const {
1439 MacroAssembler _masm(&cbuf);
1440 __ brk(0);
1441 }
1442
1443 uint MachBreakpointNode::size(PhaseRegAlloc *ra_) const {
1444 return MachNode::size(ra_);
1445 }
1446
1447 //=============================================================================
1448
1449 #ifndef PRODUCT
1450 void MachNopNode::format(PhaseRegAlloc*, outputStream* st) const {
1451 st->print("nop \t# %d bytes pad for loops and calls", _count);
1452 }
1453 #endif
1454
1455 void MachNopNode::emit(CodeBuffer &cbuf, PhaseRegAlloc*) const {
1456 MacroAssembler _masm(&cbuf);
1457 for (int i = 0; i < _count; i++) {
1458 __ nop();
1459 }
1460 }
1461
1462 uint MachNopNode::size(PhaseRegAlloc*) const {
1463 return _count * NativeInstruction::instruction_size;
1464 }
1465
1466 //=============================================================================
1467 const RegMask& MachConstantBaseNode::_out_RegMask = RegMask::Empty;
1468
1469 int Compile::ConstantTable::calculate_table_base_offset() const {
1470 return 0; // absolute addressing, no offset
1471 }
1472
1473 bool MachConstantBaseNode::requires_postalloc_expand() const { return false; }
1474 void MachConstantBaseNode::postalloc_expand(GrowableArray <Node *> *nodes, PhaseRegAlloc *ra_) {
1475 ShouldNotReachHere();
1476 }
1477
1478 void MachConstantBaseNode::emit(CodeBuffer& cbuf, PhaseRegAlloc* ra_) const {
1479 // Empty encoding
1480 }
1481
1482 uint MachConstantBaseNode::size(PhaseRegAlloc* ra_) const {
1483 return 0;
1484 }
1485
1486 #ifndef PRODUCT
1487 void MachConstantBaseNode::format(PhaseRegAlloc* ra_, outputStream* st) const {
1488 st->print("-- \t// MachConstantBaseNode (empty encoding)");
1489 }
1490 #endif
1491
1492 #ifndef PRODUCT
1493 void MachPrologNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
1494 Compile* C = ra_->C;
1495
1496 int framesize = C->frame_slots() << LogBytesPerInt;
1497
1498 if (C->need_stack_bang(framesize))
1499 st->print("# stack bang size=%d\n\t", framesize);
1500
1501 if (framesize == 0) {
1502 // Is this even possible?
1503 st->print("stp lr, rfp, [sp, #%d]!", -(2 * wordSize));
1504 } else if (framesize < ((1 << 9) + 2 * wordSize)) {
1505 st->print("sub sp, sp, #%d\n\t", framesize);
1506 st->print("stp rfp, lr, [sp, #%d]", framesize - 2 * wordSize);
1507 } else {
1508 st->print("stp lr, rfp, [sp, #%d]!\n\t", -(2 * wordSize));
1509 st->print("mov rscratch1, #%d\n\t", framesize - 2 * wordSize);
1510 st->print("sub sp, sp, rscratch1");
1511 }
1512 }
1513 #endif
1514
1515 void MachPrologNode::emit(CodeBuffer &cbuf, PhaseRegAlloc *ra_) const {
1516 Compile* C = ra_->C;
1517 MacroAssembler _masm(&cbuf);
1518
1519 // n.b. frame size includes space for return pc and rfp
1520 long framesize = ((long)C->frame_slots()) << LogBytesPerInt;
1521 assert(framesize%(2*wordSize) == 0, "must preserve 2*wordSize alignment");
1522
1523 // insert a nop at the start of the prolog so we can patch in a
1524 // branch if we need to invalidate the method later
1525 __ nop();
1526
1527 if (C->need_stack_bang(framesize))
1528 __ generate_stack_overflow_check(framesize);
1529
1530 __ build_frame(framesize);
1531
1532 if (VerifyStackAtCalls) {
1533 Unimplemented();
1534 }
1535
1536 C->set_frame_complete(cbuf.insts_size());
1537
1538 if (C->has_mach_constant_base_node()) {
1539 // NOTE: We set the table base offset here because users might be
1540 // emitted before MachConstantBaseNode.
1541 Compile::ConstantTable& constant_table = C->constant_table();
1542 constant_table.set_table_base_offset(constant_table.calculate_table_base_offset());
1543 }
1544 }
1545
1546 uint MachPrologNode::size(PhaseRegAlloc* ra_) const
1547 {
1548 return MachNode::size(ra_); // too many variables; just compute it
1549 // the hard way
1550 }
1551
1552 int MachPrologNode::reloc() const
1553 {
1554 return 0;
1555 }
1556
1557 //=============================================================================
1558
1559 #ifndef PRODUCT
1560 void MachEpilogNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
1561 Compile* C = ra_->C;
1562 int framesize = C->frame_slots() << LogBytesPerInt;
1563
1564 st->print("# pop frame %d\n\t",framesize);
1565
1566 if (framesize == 0) {
1567 st->print("ldp lr, rfp, [sp],#%d\n\t", (2 * wordSize));
1568 } else if (framesize < ((1 << 9) + 2 * wordSize)) {
1569 st->print("ldp lr, rfp, [sp,#%d]\n\t", framesize - 2 * wordSize);
1570 st->print("add sp, sp, #%d\n\t", framesize);
1571 } else {
1572 st->print("mov rscratch1, #%d\n\t", framesize - 2 * wordSize);
1573 st->print("add sp, sp, rscratch1\n\t");
1574 st->print("ldp lr, rfp, [sp],#%d\n\t", (2 * wordSize));
1575 }
1576
1577 if (do_polling() && C->is_method_compilation()) {
1578 st->print("# touch polling page\n\t");
1579 st->print("mov rscratch1, #" INTPTR_FORMAT "\n\t", p2i(os::get_polling_page()));
1580 st->print("ldr zr, [rscratch1]");
1581 }
1582 }
1583 #endif
1584
1585 void MachEpilogNode::emit(CodeBuffer &cbuf, PhaseRegAlloc *ra_) const {
1586 Compile* C = ra_->C;
1587 MacroAssembler _masm(&cbuf);
1588 int framesize = C->frame_slots() << LogBytesPerInt;
1589
1590 __ remove_frame(framesize);
1591
1592 if (do_polling() && C->is_method_compilation()) {
1593 __ read_polling_page(rscratch1, os::get_polling_page(), relocInfo::poll_return_type);
1594 }
1595 }
1596
1597 uint MachEpilogNode::size(PhaseRegAlloc *ra_) const {
1598 // Variable size. Determine dynamically.
1599 return MachNode::size(ra_);
1600 }
1601
1602 int MachEpilogNode::reloc() const {
1603 // Return number of relocatable values contained in this instruction.
1604 return 1; // 1 for polling page.
1605 }
1606
1607 const Pipeline * MachEpilogNode::pipeline() const {
1608 return MachNode::pipeline_class();
1609 }
1610
1611 // This method seems to be obsolete. It is declared in machnode.hpp
1612 // and defined in all *.ad files, but it is never called. Should we
1613 // get rid of it?
1614 int MachEpilogNode::safepoint_offset() const {
1615 assert(do_polling(), "no return for this epilog node");
1616 return 4;
1617 }
1618
1619 //=============================================================================
1620
1621 // Figure out which register class each belongs in: rc_int, rc_float or
1622 // rc_stack.
1623 enum RC { rc_bad, rc_int, rc_float, rc_stack };
1624
1625 static enum RC rc_class(OptoReg::Name reg) {
1626
1627 if (reg == OptoReg::Bad) {
1628 return rc_bad;
1629 }
1630
1631 // we have 30 int registers * 2 halves
1632 // (rscratch1 and rscratch2 are omitted)
1633
1634 if (reg < 60) {
1635 return rc_int;
1636 }
1637
1638 // we have 32 float register * 2 halves
1639 if (reg < 60 + 128) {
1640 return rc_float;
1641 }
1642
1643 // Between float regs & stack is the flags regs.
1644 assert(OptoReg::is_stack(reg), "blow up if spilling flags");
1645
1646 return rc_stack;
1647 }
1648
1649 uint MachSpillCopyNode::implementation(CodeBuffer *cbuf, PhaseRegAlloc *ra_, bool do_size, outputStream *st) const {
1650 Compile* C = ra_->C;
1651
1652 // Get registers to move.
1653 OptoReg::Name src_hi = ra_->get_reg_second(in(1));
1654 OptoReg::Name src_lo = ra_->get_reg_first(in(1));
1655 OptoReg::Name dst_hi = ra_->get_reg_second(this);
1656 OptoReg::Name dst_lo = ra_->get_reg_first(this);
1657
1658 enum RC src_hi_rc = rc_class(src_hi);
1659 enum RC src_lo_rc = rc_class(src_lo);
1660 enum RC dst_hi_rc = rc_class(dst_hi);
1661 enum RC dst_lo_rc = rc_class(dst_lo);
1662
1663 assert(src_lo != OptoReg::Bad && dst_lo != OptoReg::Bad, "must move at least 1 register");
1664
1665 if (src_hi != OptoReg::Bad) {
1666 assert((src_lo&1)==0 && src_lo+1==src_hi &&
1667 (dst_lo&1)==0 && dst_lo+1==dst_hi,
1668 "expected aligned-adjacent pairs");
1669 }
1670
1671 if (src_lo == dst_lo && src_hi == dst_hi) {
1672 return 0; // Self copy, no move.
1673 }
1674
1675 bool is64 = (src_lo & 1) == 0 && src_lo + 1 == src_hi &&
1676 (dst_lo & 1) == 0 && dst_lo + 1 == dst_hi;
1677 int src_offset = ra_->reg2offset(src_lo);
1678 int dst_offset = ra_->reg2offset(dst_lo);
1679
1680 if (bottom_type()->isa_vect() != NULL) {
1681 uint ireg = ideal_reg();
1682 assert(ireg == Op_VecD || ireg == Op_VecX, "must be 64 bit or 128 bit vector");
1683 if (cbuf) {
1684 MacroAssembler _masm(cbuf);
1685 assert((src_lo_rc != rc_int && dst_lo_rc != rc_int), "sanity");
1686 if (src_lo_rc == rc_stack && dst_lo_rc == rc_stack) {
1687 // stack->stack
1688 assert((src_offset & 7) == 0 && (dst_offset & 7) == 0, "unaligned stack offset");
1689 if (ireg == Op_VecD) {
1690 __ unspill(rscratch1, true, src_offset);
1691 __ spill(rscratch1, true, dst_offset);
1692 } else {
1693 __ spill_copy128(src_offset, dst_offset);
1694 }
1695 } else if (src_lo_rc == rc_float && dst_lo_rc == rc_float) {
1696 __ mov(as_FloatRegister(Matcher::_regEncode[dst_lo]),
1697 ireg == Op_VecD ? __ T8B : __ T16B,
1698 as_FloatRegister(Matcher::_regEncode[src_lo]));
1699 } else if (src_lo_rc == rc_float && dst_lo_rc == rc_stack) {
1700 __ spill(as_FloatRegister(Matcher::_regEncode[src_lo]),
1701 ireg == Op_VecD ? __ D : __ Q,
1702 ra_->reg2offset(dst_lo));
1703 } else if (src_lo_rc == rc_stack && dst_lo_rc == rc_float) {
1704 __ unspill(as_FloatRegister(Matcher::_regEncode[dst_lo]),
1705 ireg == Op_VecD ? __ D : __ Q,
1706 ra_->reg2offset(src_lo));
1707 } else {
1708 ShouldNotReachHere();
1709 }
1710 }
1711 } else if (cbuf) {
1712 MacroAssembler _masm(cbuf);
1713 switch (src_lo_rc) {
1714 case rc_int:
1715 if (dst_lo_rc == rc_int) { // gpr --> gpr copy
1716 if (is64) {
1717 __ mov(as_Register(Matcher::_regEncode[dst_lo]),
1718 as_Register(Matcher::_regEncode[src_lo]));
1719 } else {
1720 MacroAssembler _masm(cbuf);
1721 __ movw(as_Register(Matcher::_regEncode[dst_lo]),
1722 as_Register(Matcher::_regEncode[src_lo]));
1723 }
1724 } else if (dst_lo_rc == rc_float) { // gpr --> fpr copy
1725 if (is64) {
1726 __ fmovd(as_FloatRegister(Matcher::_regEncode[dst_lo]),
1727 as_Register(Matcher::_regEncode[src_lo]));
1728 } else {
1729 __ fmovs(as_FloatRegister(Matcher::_regEncode[dst_lo]),
1730 as_Register(Matcher::_regEncode[src_lo]));
1731 }
1732 } else { // gpr --> stack spill
1733 assert(dst_lo_rc == rc_stack, "spill to bad register class");
1734 __ spill(as_Register(Matcher::_regEncode[src_lo]), is64, dst_offset);
1735 }
1736 break;
1737 case rc_float:
1738 if (dst_lo_rc == rc_int) { // fpr --> gpr copy
1739 if (is64) {
1740 __ fmovd(as_Register(Matcher::_regEncode[dst_lo]),
1741 as_FloatRegister(Matcher::_regEncode[src_lo]));
1742 } else {
1743 __ fmovs(as_Register(Matcher::_regEncode[dst_lo]),
1744 as_FloatRegister(Matcher::_regEncode[src_lo]));
1745 }
1746 } else if (dst_lo_rc == rc_float) { // fpr --> fpr copy
1747 if (cbuf) {
1748 __ fmovd(as_FloatRegister(Matcher::_regEncode[dst_lo]),
1749 as_FloatRegister(Matcher::_regEncode[src_lo]));
1750 } else {
1751 __ fmovs(as_FloatRegister(Matcher::_regEncode[dst_lo]),
1752 as_FloatRegister(Matcher::_regEncode[src_lo]));
1753 }
1754 } else { // fpr --> stack spill
1755 assert(dst_lo_rc == rc_stack, "spill to bad register class");
1756 __ spill(as_FloatRegister(Matcher::_regEncode[src_lo]),
1757 is64 ? __ D : __ S, dst_offset);
1758 }
1759 break;
1760 case rc_stack:
1761 if (dst_lo_rc == rc_int) { // stack --> gpr load
1762 __ unspill(as_Register(Matcher::_regEncode[dst_lo]), is64, src_offset);
1763 } else if (dst_lo_rc == rc_float) { // stack --> fpr load
1764 __ unspill(as_FloatRegister(Matcher::_regEncode[dst_lo]),
1765 is64 ? __ D : __ S, src_offset);
1766 } else { // stack --> stack copy
1767 assert(dst_lo_rc == rc_stack, "spill to bad register class");
1768 __ unspill(rscratch1, is64, src_offset);
1769 __ spill(rscratch1, is64, dst_offset);
1770 }
1771 break;
1772 default:
1773 assert(false, "bad rc_class for spill");
1774 ShouldNotReachHere();
1775 }
1776 }
1777
1778 if (st) {
1779 st->print("spill ");
1780 if (src_lo_rc == rc_stack) {
1781 st->print("[sp, #%d] -> ", ra_->reg2offset(src_lo));
1782 } else {
1783 st->print("%s -> ", Matcher::regName[src_lo]);
1784 }
1785 if (dst_lo_rc == rc_stack) {
1786 st->print("[sp, #%d]", ra_->reg2offset(dst_lo));
1787 } else {
1788 st->print("%s", Matcher::regName[dst_lo]);
1789 }
1790 if (bottom_type()->isa_vect() != NULL) {
1791 st->print("\t# vector spill size = %d", ideal_reg()==Op_VecD ? 64:128);
1792 } else {
1793 st->print("\t# spill size = %d", is64 ? 64:32);
1794 }
1795 }
1796
1797 return 0;
1798
1799 }
1800
1801 #ifndef PRODUCT
1802 void MachSpillCopyNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
1803 if (!ra_)
1804 st->print("N%d = SpillCopy(N%d)", _idx, in(1)->_idx);
1805 else
1806 implementation(NULL, ra_, false, st);
1807 }
1808 #endif
1809
1810 void MachSpillCopyNode::emit(CodeBuffer &cbuf, PhaseRegAlloc *ra_) const {
1811 implementation(&cbuf, ra_, false, NULL);
1812 }
1813
1814 uint MachSpillCopyNode::size(PhaseRegAlloc *ra_) const {
1815 return MachNode::size(ra_);
1816 }
1817
1818 //=============================================================================
1819
1820 #ifndef PRODUCT
1821 void BoxLockNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
1822 int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
1823 int reg = ra_->get_reg_first(this);
1824 st->print("add %s, rsp, #%d]\t# box lock",
1825 Matcher::regName[reg], offset);
1826 }
1827 #endif
1828
1829 void BoxLockNode::emit(CodeBuffer &cbuf, PhaseRegAlloc *ra_) const {
1830 MacroAssembler _masm(&cbuf);
1831
1832 int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
1833 int reg = ra_->get_encode(this);
1834
1835 // This add will handle any 24-bit signed offset. 24 bits allows an
1836 // 8 megabyte stack frame.
1837 __ add(as_Register(reg), sp, offset);
1838 }
1839
1840 uint BoxLockNode::size(PhaseRegAlloc *ra_) const {
1841 // BoxLockNode is not a MachNode, so we can't just call MachNode::size(ra_).
1842 int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
1843
1844 if (Assembler::operand_valid_for_add_sub_immediate(offset)) {
1845 return NativeInstruction::instruction_size;
1846 } else {
1847 return 2 * NativeInstruction::instruction_size;
1848 }
1849 }
1850
1851 //=============================================================================
1852
1853 #ifndef PRODUCT
1854 void MachUEPNode::format(PhaseRegAlloc* ra_, outputStream* st) const
1855 {
1856 st->print_cr("# MachUEPNode");
1857 if (UseCompressedClassPointers) {
1858 st->print_cr("\tldrw rscratch1, j_rarg0 + oopDesc::klass_offset_in_bytes()]\t# compressed klass");
1859 if (Universe::narrow_klass_shift() != 0) {
1860 st->print_cr("\tdecode_klass_not_null rscratch1, rscratch1");
1861 }
1862 } else {
1863 st->print_cr("\tldr rscratch1, j_rarg0 + oopDesc::klass_offset_in_bytes()]\t# compressed klass");
1864 }
1865 st->print_cr("\tcmp r0, rscratch1\t # Inline cache check");
1866 st->print_cr("\tbne, SharedRuntime::_ic_miss_stub");
1867 }
1868 #endif
1869
1870 void MachUEPNode::emit(CodeBuffer& cbuf, PhaseRegAlloc* ra_) const
1871 {
1872 // This is the unverified entry point.
1873 MacroAssembler _masm(&cbuf);
1874
1875 __ cmp_klass(j_rarg0, rscratch2, rscratch1);
1876 Label skip;
1877 // TODO
1878 // can we avoid this skip and still use a reloc?
1879 __ br(Assembler::EQ, skip);
1880 __ far_jump(RuntimeAddress(SharedRuntime::get_ic_miss_stub()));
1881 __ bind(skip);
1882 }
1883
1884 uint MachUEPNode::size(PhaseRegAlloc* ra_) const
1885 {
1886 return MachNode::size(ra_);
1887 }
1888
1889 // REQUIRED EMIT CODE
1890
1891 //=============================================================================
1892
1893 // Emit exception handler code.
1894 int HandlerImpl::emit_exception_handler(CodeBuffer& cbuf)
1895 {
1896 // mov rscratch1 #exception_blob_entry_point
1897 // br rscratch1
1898 // Note that the code buffer's insts_mark is always relative to insts.
1899 // That's why we must use the macroassembler to generate a handler.
1900 MacroAssembler _masm(&cbuf);
1901 address base = __ start_a_stub(size_exception_handler());
1902 if (base == NULL) {
1903 ciEnv::current()->record_failure("CodeCache is full");
1904 return 0; // CodeBuffer::expand failed
1905 }
1906 int offset = __ offset();
1907 __ far_jump(RuntimeAddress(OptoRuntime::exception_blob()->entry_point()));
1908 assert(__ offset() - offset <= (int) size_exception_handler(), "overflow");
1909 __ end_a_stub();
1910 return offset;
1911 }
1912
1913 // Emit deopt handler code.
1914 int HandlerImpl::emit_deopt_handler(CodeBuffer& cbuf)
1915 {
1916 // Note that the code buffer's insts_mark is always relative to insts.
1917 // That's why we must use the macroassembler to generate a handler.
1918 MacroAssembler _masm(&cbuf);
1919 address base = __ start_a_stub(size_deopt_handler());
1920 if (base == NULL) {
1921 ciEnv::current()->record_failure("CodeCache is full");
1922 return 0; // CodeBuffer::expand failed
1923 }
1924 int offset = __ offset();
1925
1926 __ adr(lr, __ pc());
1927 __ far_jump(RuntimeAddress(SharedRuntime::deopt_blob()->unpack()));
1928
1929 assert(__ offset() - offset <= (int) size_deopt_handler(), "overflow");
1930 __ end_a_stub();
1931 return offset;
1932 }
1933
1934 // REQUIRED MATCHER CODE
1935
1936 //=============================================================================
1937
1938 const bool Matcher::match_rule_supported(int opcode) {
1939
1940 // TODO
1941 // identify extra cases that we might want to provide match rules for
1942 // e.g. Op_StrEquals and other intrinsics
1943 if (!has_match_rule(opcode)) {
1944 return false;
1945 }
1946
1947 return true; // Per default match rules are supported.
1948 }
1949
1950 int Matcher::regnum_to_fpu_offset(int regnum)
1951 {
1952 Unimplemented();
1953 return 0;
1954 }
1955
1956 // Is this branch offset short enough that a short branch can be used?
1957 //
1958 // NOTE: If the platform does not provide any short branch variants, then
1959 // this method should return false for offset 0.
1960 bool Matcher::is_short_branch_offset(int rule, int br_size, int offset) {
1961 // The passed offset is relative to address of the branch.
1962
1963 return (-32768 <= offset && offset < 32768);
1964 }
1965
1966 const bool Matcher::isSimpleConstant64(jlong value) {
1967 // Will one (StoreL ConL) be cheaper than two (StoreI ConI)?.
1968 // Probably always true, even if a temp register is required.
1969 return true;
1970 }
1971
1972 // true just means we have fast l2f conversion
1973 const bool Matcher::convL2FSupported(void) {
1974 return true;
1975 }
1976
1977 // Vector width in bytes.
1978 const int Matcher::vector_width_in_bytes(BasicType bt) {
1979 int size = MIN2(16,(int)MaxVectorSize);
1980 // Minimum 2 values in vector
1981 if (size < 2*type2aelembytes(bt)) size = 0;
1982 // But never < 4
1983 if (size < 4) size = 0;
1984 return size;
1985 }
1986
1987 // Limits on vector size (number of elements) loaded into vector.
1988 const int Matcher::max_vector_size(const BasicType bt) {
1989 return vector_width_in_bytes(bt)/type2aelembytes(bt);
1990 }
1991 const int Matcher::min_vector_size(const BasicType bt) {
1992 // For the moment limit the vector size to 8 bytes
1993 int size = 8 / type2aelembytes(bt);
1994 if (size < 2) size = 2;
1995 return size;
1996 }
1997
1998 // Vector ideal reg.
1999 const uint Matcher::vector_ideal_reg(int len) {
2000 switch(len) {
2001 case 8: return Op_VecD;
2002 case 16: return Op_VecX;
2003 }
2004 ShouldNotReachHere();
2005 return 0;
2006 }
2007
2008 const uint Matcher::vector_shift_count_ideal_reg(int size) {
2009 switch(size) {
2010 case 8: return Op_VecD;
2011 case 16: return Op_VecX;
2012 }
2013 ShouldNotReachHere();
2014 return 0;
2015 }
2016
2017 // AES support not yet implemented
2018 const bool Matcher::pass_original_key_for_aes() {
2019 return false;
2020 }
2021
2022 // x86 supports misaligned vectors store/load.
2023 const bool Matcher::misaligned_vectors_ok() {
2024 return !AlignVector; // can be changed by flag
2025 }
2026
2027 // false => size gets scaled to BytesPerLong, ok.
2028 const bool Matcher::init_array_count_is_in_bytes = false;
2029
2030 // Threshold size for cleararray.
2031 const int Matcher::init_array_short_size = 4 * BytesPerLong;
2032
2033 // Use conditional move (CMOVL)
2034 const int Matcher::long_cmove_cost() {
2035 // long cmoves are no more expensive than int cmoves
2036 return 0;
2037 }
2038
2039 const int Matcher::float_cmove_cost() {
2040 // float cmoves are no more expensive than int cmoves
2041 return 0;
2042 }
2043
2044 // Does the CPU require late expand (see block.cpp for description of late expand)?
2045 const bool Matcher::require_postalloc_expand = false;
2046
2047 // Should the Matcher clone shifts on addressing modes, expecting them
2048 // to be subsumed into complex addressing expressions or compute them
2049 // into registers? True for Intel but false for most RISCs
2050 const bool Matcher::clone_shift_expressions = false;
2051
2052 // Do we need to mask the count passed to shift instructions or does
2053 // the cpu only look at the lower 5/6 bits anyway?
2054 const bool Matcher::need_masked_shift_count = false;
2055
2056 // This affects two different things:
2057 // - how Decode nodes are matched
2058 // - how ImplicitNullCheck opportunities are recognized
2059 // If true, the matcher will try to remove all Decodes and match them
2060 // (as operands) into nodes. NullChecks are not prepared to deal with
2061 // Decodes by final_graph_reshaping().
2062 // If false, final_graph_reshaping() forces the decode behind the Cmp
2063 // for a NullCheck. The matcher matches the Decode node into a register.
2064 // Implicit_null_check optimization moves the Decode along with the
2065 // memory operation back up before the NullCheck.
2066 bool Matcher::narrow_oop_use_complex_address() {
2067 return Universe::narrow_oop_shift() == 0;
2068 }
2069
2070 bool Matcher::narrow_klass_use_complex_address() {
2071 // TODO
2072 // decide whether we need to set this to true
2073 return false;
2074 }
2075
2076 // Is it better to copy float constants, or load them directly from
2077 // memory? Intel can load a float constant from a direct address,
2078 // requiring no extra registers. Most RISCs will have to materialize
2079 // an address into a register first, so they would do better to copy
2080 // the constant from stack.
2081 const bool Matcher::rematerialize_float_constants = false;
2082
2083 // If CPU can load and store mis-aligned doubles directly then no
2084 // fixup is needed. Else we split the double into 2 integer pieces
2085 // and move it piece-by-piece. Only happens when passing doubles into
2086 // C code as the Java calling convention forces doubles to be aligned.
2087 const bool Matcher::misaligned_doubles_ok = true;
2088
2089 // No-op on amd64
2090 void Matcher::pd_implicit_null_fixup(MachNode *node, uint idx) {
2091 Unimplemented();
2092 }
2093
2094 // Advertise here if the CPU requires explicit rounding operations to
2095 // implement the UseStrictFP mode.
2096 const bool Matcher::strict_fp_requires_explicit_rounding = false;
2097
2098 // Are floats converted to double when stored to stack during
2099 // deoptimization?
2100 bool Matcher::float_in_double() { return true; }
2101
2102 // Do ints take an entire long register or just half?
2103 // The relevant question is how the int is callee-saved:
2104 // the whole long is written but de-opt'ing will have to extract
2105 // the relevant 32 bits.
2106 const bool Matcher::int_in_long = true;
2107
2108 // Return whether or not this register is ever used as an argument.
2109 // This function is used on startup to build the trampoline stubs in
2110 // generateOptoStub. Registers not mentioned will be killed by the VM
2111 // call in the trampoline, and arguments in those registers not be
2112 // available to the callee.
2113 bool Matcher::can_be_java_arg(int reg)
2114 {
2115 return
2116 reg == R0_num || reg == R0_H_num ||
2117 reg == R1_num || reg == R1_H_num ||
2118 reg == R2_num || reg == R2_H_num ||
2119 reg == R3_num || reg == R3_H_num ||
2120 reg == R4_num || reg == R4_H_num ||
2121 reg == R5_num || reg == R5_H_num ||
2122 reg == R6_num || reg == R6_H_num ||
2123 reg == R7_num || reg == R7_H_num ||
2124 reg == V0_num || reg == V0_H_num ||
2125 reg == V1_num || reg == V1_H_num ||
2126 reg == V2_num || reg == V2_H_num ||
2127 reg == V3_num || reg == V3_H_num ||
2128 reg == V4_num || reg == V4_H_num ||
2129 reg == V5_num || reg == V5_H_num ||
2130 reg == V6_num || reg == V6_H_num ||
2131 reg == V7_num || reg == V7_H_num;
2132 }
2133
2134 bool Matcher::is_spillable_arg(int reg)
2135 {
2136 return can_be_java_arg(reg);
2137 }
2138
2139 bool Matcher::use_asm_for_ldiv_by_con(jlong divisor) {
2140 return false;
2141 }
2142
2143 RegMask Matcher::divI_proj_mask() {
2144 ShouldNotReachHere();
2145 return RegMask();
2146 }
2147
2148 // Register for MODI projection of divmodI.
2149 RegMask Matcher::modI_proj_mask() {
2150 ShouldNotReachHere();
2151 return RegMask();
2152 }
2153
2154 // Register for DIVL projection of divmodL.
2155 RegMask Matcher::divL_proj_mask() {
2156 ShouldNotReachHere();
2157 return RegMask();
2158 }
2159
2160 // Register for MODL projection of divmodL.
2161 RegMask Matcher::modL_proj_mask() {
2162 ShouldNotReachHere();
2163 return RegMask();
2164 }
2165
2166 const RegMask Matcher::method_handle_invoke_SP_save_mask() {
2167 return FP_REG_mask();
2168 }
2169
2170 bool size_fits_all_mem_uses(AddPNode* addp, int shift) {
2171 for (DUIterator_Fast imax, i = addp->fast_outs(imax); i < imax; i++) {
2172 Node* u = addp->fast_out(i);
2173 if (u->is_LoadStore()) {
2174 // On AArch64, LoadStoreNodes (i.e. compare and swap
2175 // instructions) only take register indirect as an operand, so
2176 // any attempt to use an AddPNode as an input to a LoadStoreNode
2177 // must fail.
2178 return false;
2179 }
2180 if (u->is_Mem()) {
2181 int opsize = u->as_Mem()->memory_size();
2182 assert(opsize > 0, "unexpected memory operand size");
2183 if (u->as_Mem()->memory_size() != (1<<shift)) {
2184 return false;
2185 }
2186 }
2187 }
2188 return true;
2189 }
2190
2191 #define MOV_VOLATILE(REG, BASE, INDEX, SCALE, DISP, SCRATCH, INSN) \
2192 MacroAssembler _masm(&cbuf); \
2193 { \
2194 guarantee(INDEX == -1, "mode not permitted for volatile"); \
2195 guarantee(DISP == 0, "mode not permitted for volatile"); \
2196 guarantee(SCALE == 0, "mode not permitted for volatile"); \
2197 __ INSN(REG, as_Register(BASE)); \
2198 }
2199
2200 typedef void (MacroAssembler::* mem_insn)(Register Rt, const Address &adr);
2201 typedef void (MacroAssembler::* mem_float_insn)(FloatRegister Rt, const Address &adr);
2202 typedef void (MacroAssembler::* mem_vector_insn)(FloatRegister Rt,
2203 MacroAssembler::SIMD_RegVariant T, const Address &adr);
2204
2205 // Used for all non-volatile memory accesses. The use of
2206 // $mem->opcode() to discover whether this pattern uses sign-extended
2207 // offsets is something of a kludge.
2208 static void loadStore(MacroAssembler masm, mem_insn insn,
2209 Register reg, int opcode,
2210 Register base, int index, int size, int disp)
2211 {
2212 Address::extend scale;
2213
2214 // Hooboy, this is fugly. We need a way to communicate to the
2215 // encoder that the index needs to be sign extended, so we have to
2216 // enumerate all the cases.
2217 switch (opcode) {
2218 case INDINDEXSCALEDOFFSETI2L:
2219 case INDINDEXSCALEDI2L:
2220 case INDINDEXSCALEDOFFSETI2LN:
2221 case INDINDEXSCALEDI2LN:
2222 case INDINDEXOFFSETI2L:
2223 case INDINDEXOFFSETI2LN:
2224 scale = Address::sxtw(size);
2225 break;
2226 default:
2227 scale = Address::lsl(size);
2228 }
2229
2230 if (index == -1) {
2231 (masm.*insn)(reg, Address(base, disp));
2232 } else {
2233 if (disp == 0) {
2234 (masm.*insn)(reg, Address(base, as_Register(index), scale));
2235 } else {
2236 masm.lea(rscratch1, Address(base, disp));
2237 (masm.*insn)(reg, Address(rscratch1, as_Register(index), scale));
2238 }
2239 }
2240 }
2241
2242 static void loadStore(MacroAssembler masm, mem_float_insn insn,
2243 FloatRegister reg, int opcode,
2244 Register base, int index, int size, int disp)
2245 {
2246 Address::extend scale;
2247
2248 switch (opcode) {
2249 case INDINDEXSCALEDOFFSETI2L:
2250 case INDINDEXSCALEDI2L:
2251 case INDINDEXSCALEDOFFSETI2LN:
2252 case INDINDEXSCALEDI2LN:
2253 scale = Address::sxtw(size);
2254 break;
2255 default:
2256 scale = Address::lsl(size);
2257 }
2258
2259 if (index == -1) {
2260 (masm.*insn)(reg, Address(base, disp));
2261 } else {
2262 if (disp == 0) {
2263 (masm.*insn)(reg, Address(base, as_Register(index), scale));
2264 } else {
2265 masm.lea(rscratch1, Address(base, disp));
2266 (masm.*insn)(reg, Address(rscratch1, as_Register(index), scale));
2267 }
2268 }
2269 }
2270
2271 static void loadStore(MacroAssembler masm, mem_vector_insn insn,
2272 FloatRegister reg, MacroAssembler::SIMD_RegVariant T,
2273 int opcode, Register base, int index, int size, int disp)
2274 {
2275 if (index == -1) {
2276 (masm.*insn)(reg, T, Address(base, disp));
2277 } else {
2278 assert(disp == 0, "unsupported address mode");
2279 (masm.*insn)(reg, T, Address(base, as_Register(index), Address::lsl(size)));
2280 }
2281 }
2282
2283 %}
2284
2285
2286
2287 //----------ENCODING BLOCK-----------------------------------------------------
2288 // This block specifies the encoding classes used by the compiler to
2289 // output byte streams. Encoding classes are parameterized macros
2290 // used by Machine Instruction Nodes in order to generate the bit
2291 // encoding of the instruction. Operands specify their base encoding
2292 // interface with the interface keyword. There are currently
2293 // supported four interfaces, REG_INTER, CONST_INTER, MEMORY_INTER, &
2294 // COND_INTER. REG_INTER causes an operand to generate a function
2295 // which returns its register number when queried. CONST_INTER causes
2296 // an operand to generate a function which returns the value of the
2297 // constant when queried. MEMORY_INTER causes an operand to generate
2298 // four functions which return the Base Register, the Index Register,
2299 // the Scale Value, and the Offset Value of the operand when queried.
2300 // COND_INTER causes an operand to generate six functions which return
2301 // the encoding code (ie - encoding bits for the instruction)
2302 // associated with each basic boolean condition for a conditional
2303 // instruction.
2304 //
2305 // Instructions specify two basic values for encoding. Again, a
2306 // function is available to check if the constant displacement is an
2307 // oop. They use the ins_encode keyword to specify their encoding
2308 // classes (which must be a sequence of enc_class names, and their
2309 // parameters, specified in the encoding block), and they use the
2310 // opcode keyword to specify, in order, their primary, secondary, and
2311 // tertiary opcode. Only the opcode sections which a particular
2312 // instruction needs for encoding need to be specified.
2313 encode %{
2314 // Build emit functions for each basic byte or larger field in the
2315 // intel encoding scheme (opcode, rm, sib, immediate), and call them
2316 // from C++ code in the enc_class source block. Emit functions will
2317 // live in the main source block for now. In future, we can
2318 // generalize this by adding a syntax that specifies the sizes of
2319 // fields in an order, so that the adlc can build the emit functions
2320 // automagically
2321
2322 // catch all for unimplemented encodings
2323 enc_class enc_unimplemented %{
2324 MacroAssembler _masm(&cbuf);
2325 __ unimplemented("C2 catch all");
2326 %}
2327
2328 // BEGIN Non-volatile memory access
2329
2330 enc_class aarch64_enc_ldrsbw(iRegI dst, memory mem) %{
2331 Register dst_reg = as_Register($dst$$reg);
2332 loadStore(MacroAssembler(&cbuf), &MacroAssembler::ldrsbw, dst_reg, $mem->opcode(),
2333 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
2334 %}
2335
2336 enc_class aarch64_enc_ldrsb(iRegI dst, memory mem) %{
2337 Register dst_reg = as_Register($dst$$reg);
2338 loadStore(MacroAssembler(&cbuf), &MacroAssembler::ldrsb, dst_reg, $mem->opcode(),
2339 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
2340 %}
2341
2342 enc_class aarch64_enc_ldrb(iRegI dst, memory mem) %{
2343 Register dst_reg = as_Register($dst$$reg);
2344 loadStore(MacroAssembler(&cbuf), &MacroAssembler::ldrb, dst_reg, $mem->opcode(),
2345 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
2346 %}
2347
2348 enc_class aarch64_enc_ldrb(iRegL dst, memory mem) %{
2349 Register dst_reg = as_Register($dst$$reg);
2350 loadStore(MacroAssembler(&cbuf), &MacroAssembler::ldrb, dst_reg, $mem->opcode(),
2351 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
2352 %}
2353
2354 enc_class aarch64_enc_ldrshw(iRegI dst, memory mem) %{
2355 Register dst_reg = as_Register($dst$$reg);
2356 loadStore(MacroAssembler(&cbuf), &MacroAssembler::ldrshw, dst_reg, $mem->opcode(),
2357 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
2358 %}
2359
2360 enc_class aarch64_enc_ldrsh(iRegI dst, memory mem) %{
2361 Register dst_reg = as_Register($dst$$reg);
2362 loadStore(MacroAssembler(&cbuf), &MacroAssembler::ldrsh, dst_reg, $mem->opcode(),
2363 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
2364 %}
2365
2366 enc_class aarch64_enc_ldrh(iRegI dst, memory mem) %{
2367 Register dst_reg = as_Register($dst$$reg);
2368 loadStore(MacroAssembler(&cbuf), &MacroAssembler::ldrh, dst_reg, $mem->opcode(),
2369 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
2370 %}
2371
2372 enc_class aarch64_enc_ldrh(iRegL dst, memory mem) %{
2373 Register dst_reg = as_Register($dst$$reg);
2374 loadStore(MacroAssembler(&cbuf), &MacroAssembler::ldrh, dst_reg, $mem->opcode(),
2375 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
2376 %}
2377
2378 enc_class aarch64_enc_ldrw(iRegI dst, memory mem) %{
2379 Register dst_reg = as_Register($dst$$reg);
2380 loadStore(MacroAssembler(&cbuf), &MacroAssembler::ldrw, dst_reg, $mem->opcode(),
2381 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
2382 %}
2383
2384 enc_class aarch64_enc_ldrw(iRegL dst, memory mem) %{
2385 Register dst_reg = as_Register($dst$$reg);
2386 loadStore(MacroAssembler(&cbuf), &MacroAssembler::ldrw, dst_reg, $mem->opcode(),
2387 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
2388 %}
2389
2390 enc_class aarch64_enc_ldrsw(iRegL dst, memory mem) %{
2391 Register dst_reg = as_Register($dst$$reg);
2392 loadStore(MacroAssembler(&cbuf), &MacroAssembler::ldrsw, dst_reg, $mem->opcode(),
2393 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
2394 %}
2395
2396 enc_class aarch64_enc_ldr(iRegL dst, memory mem) %{
2397 Register dst_reg = as_Register($dst$$reg);
2398 loadStore(MacroAssembler(&cbuf), &MacroAssembler::ldr, dst_reg, $mem->opcode(),
2399 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
2400 %}
2401
2402 enc_class aarch64_enc_ldrs(vRegF dst, memory mem) %{
2403 FloatRegister dst_reg = as_FloatRegister($dst$$reg);
2404 loadStore(MacroAssembler(&cbuf), &MacroAssembler::ldrs, dst_reg, $mem->opcode(),
2405 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
2406 %}
2407
2408 enc_class aarch64_enc_ldrd(vRegD dst, memory mem) %{
2409 FloatRegister dst_reg = as_FloatRegister($dst$$reg);
2410 loadStore(MacroAssembler(&cbuf), &MacroAssembler::ldrd, dst_reg, $mem->opcode(),
2411 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
2412 %}
2413
2414 enc_class aarch64_enc_ldrvS(vecD dst, memory mem) %{
2415 FloatRegister dst_reg = as_FloatRegister($dst$$reg);
2416 loadStore(MacroAssembler(&cbuf), &MacroAssembler::ldr, dst_reg, MacroAssembler::S,
2417 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
2418 %}
2419
2420 enc_class aarch64_enc_ldrvD(vecD dst, memory mem) %{
2421 FloatRegister dst_reg = as_FloatRegister($dst$$reg);
2422 loadStore(MacroAssembler(&cbuf), &MacroAssembler::ldr, dst_reg, MacroAssembler::D,
2423 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
2424 %}
2425
2426 enc_class aarch64_enc_ldrvQ(vecX dst, memory mem) %{
2427 FloatRegister dst_reg = as_FloatRegister($dst$$reg);
2428 loadStore(MacroAssembler(&cbuf), &MacroAssembler::ldr, dst_reg, MacroAssembler::Q,
2429 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
2430 %}
2431
2432 enc_class aarch64_enc_strb(iRegI src, memory mem) %{
2433 Register src_reg = as_Register($src$$reg);
2434 loadStore(MacroAssembler(&cbuf), &MacroAssembler::strb, src_reg, $mem->opcode(),
2435 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
2436 %}
2437
2438 enc_class aarch64_enc_strb0(memory mem) %{
2439 MacroAssembler _masm(&cbuf);
2440 loadStore(_masm, &MacroAssembler::strb, zr, $mem->opcode(),
2441 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
2442 %}
2443
2444 enc_class aarch64_enc_strb0_ordered(memory mem) %{
2445 MacroAssembler _masm(&cbuf);
2446 __ membar(Assembler::StoreStore);
2447 loadStore(_masm, &MacroAssembler::strb, zr, $mem->opcode(),
2448 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
2449 %}
2450
2451 enc_class aarch64_enc_strh(iRegI src, memory mem) %{
2452 Register src_reg = as_Register($src$$reg);
2453 loadStore(MacroAssembler(&cbuf), &MacroAssembler::strh, src_reg, $mem->opcode(),
2454 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
2455 %}
2456
2457 enc_class aarch64_enc_strh0(memory mem) %{
2458 MacroAssembler _masm(&cbuf);
2459 loadStore(_masm, &MacroAssembler::strh, zr, $mem->opcode(),
2460 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
2461 %}
2462
2463 enc_class aarch64_enc_strw(iRegI src, memory mem) %{
2464 Register src_reg = as_Register($src$$reg);
2465 loadStore(MacroAssembler(&cbuf), &MacroAssembler::strw, src_reg, $mem->opcode(),
2466 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
2467 %}
2468
2469 enc_class aarch64_enc_strw0(memory mem) %{
2470 MacroAssembler _masm(&cbuf);
2471 loadStore(_masm, &MacroAssembler::strw, zr, $mem->opcode(),
2472 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
2473 %}
2474
2475 enc_class aarch64_enc_str(iRegL src, memory mem) %{
2476 Register src_reg = as_Register($src$$reg);
2477 // we sometimes get asked to store the stack pointer into the
2478 // current thread -- we cannot do that directly on AArch64
2479 if (src_reg == r31_sp) {
2480 MacroAssembler _masm(&cbuf);
2481 assert(as_Register($mem$$base) == rthread, "unexpected store for sp");
2482 __ mov(rscratch2, sp);
2483 src_reg = rscratch2;
2484 }
2485 loadStore(MacroAssembler(&cbuf), &MacroAssembler::str, src_reg, $mem->opcode(),
2486 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
2487 %}
2488
2489 enc_class aarch64_enc_str0(memory mem) %{
2490 MacroAssembler _masm(&cbuf);
2491 loadStore(_masm, &MacroAssembler::str, zr, $mem->opcode(),
2492 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
2493 %}
2494
2495 enc_class aarch64_enc_strs(vRegF src, memory mem) %{
2496 FloatRegister src_reg = as_FloatRegister($src$$reg);
2497 loadStore(MacroAssembler(&cbuf), &MacroAssembler::strs, src_reg, $mem->opcode(),
2498 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
2499 %}
2500
2501 enc_class aarch64_enc_strd(vRegD src, memory mem) %{
2502 FloatRegister src_reg = as_FloatRegister($src$$reg);
2503 loadStore(MacroAssembler(&cbuf), &MacroAssembler::strd, src_reg, $mem->opcode(),
2504 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
2505 %}
2506
2507 enc_class aarch64_enc_strvS(vecD src, memory mem) %{
2508 FloatRegister src_reg = as_FloatRegister($src$$reg);
2509 loadStore(MacroAssembler(&cbuf), &MacroAssembler::str, src_reg, MacroAssembler::S,
2510 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
2511 %}
2512
2513 enc_class aarch64_enc_strvD(vecD src, memory mem) %{
2514 FloatRegister src_reg = as_FloatRegister($src$$reg);
2515 loadStore(MacroAssembler(&cbuf), &MacroAssembler::str, src_reg, MacroAssembler::D,
2516 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
2517 %}
2518
2519 enc_class aarch64_enc_strvQ(vecX src, memory mem) %{
2520 FloatRegister src_reg = as_FloatRegister($src$$reg);
2521 loadStore(MacroAssembler(&cbuf), &MacroAssembler::str, src_reg, MacroAssembler::Q,
2522 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
2523 %}
2524
2525 // END Non-volatile memory access
2526
2527 // this encoding writes the address of the first instruction in the
2528 // call sequence for the runtime call into the anchor pc slot. this
2529 // address allows the runtime to i) locate the code buffer for the
2530 // caller (any address in the buffer would do) and ii) find the oop
2531 // map associated with the call (has to address the instruction
2532 // following the call). note that we have to store the address which
2533 // follows the actual call.
2534 //
2535 // the offset from the current pc can be computed by considering
2536 // what gets generated between this point up to and including the
2537 // call. it looks like this
2538 //
2539 // movz xscratch1 0xnnnn <-- current pc is here
2540 // movk xscratch1 0xnnnn
2541 // movk xscratch1 0xnnnn
2542 // str xscratch1, [xthread,#anchor_pc_off]
2543 // mov xscratch2, sp
2544 // str xscratch2, [xthread,#anchor_sp_off
2545 // mov x0, x1
2546 // . . .
2547 // mov xn-1, xn
2548 // mov xn, thread <-- always passed
2549 // mov xn+1, rfp <-- optional iff primary == 1
2550 // movz xscratch1 0xnnnn
2551 // movk xscratch1 0xnnnn
2552 // movk xscratch1 0xnnnn
2553 // blr xscratch1
2554 // . . .
2555 //
2556 // where the called routine has n args (including the thread and,
2557 // possibly the stub's caller return address currently in rfp). we
2558 // can compute n by looking at the number of args passed into the
2559 // stub. we assert that nargs is < 7.
2560 //
2561 // so the offset we need to add to the pc (in 32-bit words) is
2562 // 3 + <-- load 48-bit constant return pc
2563 // 1 + <-- write anchor pc
2564 // 1 + <-- copy sp
2565 // 1 + <-- write anchor sp
2566 // nargs + <-- java stub arg count
2567 // 1 + <-- extra thread arg
2568 // [ 1 + ] <-- optional ret address of stub caller
2569 // 3 + <-- load 64 bit call target address
2570 // 1 <-- blr instruction
2571 //
2572 // i.e we need to add (nargs + 11) * 4 bytes or (nargs + 12) * 4 bytes
2573 //
2574
2575 enc_class aarch64_enc_save_pc() %{
2576 Compile* C = ra_->C;
2577 int nargs = C->tf()->domain()->cnt() - TypeFunc::Parms;
2578 if ($primary) { nargs++; }
2579 assert(nargs <= 8, "opto runtime stub has more than 8 args!");
2580 MacroAssembler _masm(&cbuf);
2581 address pc = __ pc();
2582 int call_offset = (nargs + 11) * 4;
2583 int field_offset = in_bytes(JavaThread::frame_anchor_offset()) +
2584 in_bytes(JavaFrameAnchor::last_Java_pc_offset());
2585 __ lea(rscratch1, InternalAddress(pc + call_offset));
2586 __ str(rscratch1, Address(rthread, field_offset));
2587 %}
2588
2589 // volatile loads and stores
2590
2591 enc_class aarch64_enc_stlrb(iRegI src, memory mem) %{
2592 MOV_VOLATILE(as_Register($src$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
2593 rscratch1, stlrb);
2594 if (VM_Version::cpu_cpuFeatures() & VM_Version::CPU_DMB_ATOMICS)
2595 __ dmb(__ ISH);
2596 %}
2597
2598 enc_class aarch64_enc_stlrh(iRegI src, memory mem) %{
2599 MOV_VOLATILE(as_Register($src$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
2600 rscratch1, stlrh);
2601 if (VM_Version::cpu_cpuFeatures() & VM_Version::CPU_DMB_ATOMICS)
2602 __ dmb(__ ISH);
2603 %}
2604
2605 enc_class aarch64_enc_stlrw(iRegI src, memory mem) %{
2606 MOV_VOLATILE(as_Register($src$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
2607 rscratch1, stlrw);
2608 if (VM_Version::cpu_cpuFeatures() & VM_Version::CPU_DMB_ATOMICS)
2609 __ dmb(__ ISH);
2610 %}
2611
2612
2613 enc_class aarch64_enc_ldarsbw(iRegI dst, memory mem) %{
2614 Register dst_reg = as_Register($dst$$reg);
2615 MOV_VOLATILE(dst_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
2616 rscratch1, ldarb);
2617 __ sxtbw(dst_reg, dst_reg);
2618 %}
2619
2620 enc_class aarch64_enc_ldarsb(iRegL dst, memory mem) %{
2621 Register dst_reg = as_Register($dst$$reg);
2622 MOV_VOLATILE(dst_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
2623 rscratch1, ldarb);
2624 __ sxtb(dst_reg, dst_reg);
2625 %}
2626
2627 enc_class aarch64_enc_ldarbw(iRegI dst, memory mem) %{
2628 MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
2629 rscratch1, ldarb);
2630 %}
2631
2632 enc_class aarch64_enc_ldarb(iRegL dst, memory mem) %{
2633 MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
2634 rscratch1, ldarb);
2635 %}
2636
2637 enc_class aarch64_enc_ldarshw(iRegI dst, memory mem) %{
2638 Register dst_reg = as_Register($dst$$reg);
2639 MOV_VOLATILE(dst_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
2640 rscratch1, ldarh);
2641 __ sxthw(dst_reg, dst_reg);
2642 %}
2643
2644 enc_class aarch64_enc_ldarsh(iRegL dst, memory mem) %{
2645 Register dst_reg = as_Register($dst$$reg);
2646 MOV_VOLATILE(dst_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
2647 rscratch1, ldarh);
2648 __ sxth(dst_reg, dst_reg);
2649 %}
2650
2651 enc_class aarch64_enc_ldarhw(iRegI dst, memory mem) %{
2652 MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
2653 rscratch1, ldarh);
2654 %}
2655
2656 enc_class aarch64_enc_ldarh(iRegL dst, memory mem) %{
2657 MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
2658 rscratch1, ldarh);
2659 %}
2660
2661 enc_class aarch64_enc_ldarw(iRegI dst, memory mem) %{
2662 MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
2663 rscratch1, ldarw);
2664 %}
2665
2666 enc_class aarch64_enc_ldarw(iRegL dst, memory mem) %{
2667 MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
2668 rscratch1, ldarw);
2669 %}
2670
2671 enc_class aarch64_enc_ldar(iRegL dst, memory mem) %{
2672 MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
2673 rscratch1, ldar);
2674 %}
2675
2676 enc_class aarch64_enc_fldars(vRegF dst, memory mem) %{
2677 MOV_VOLATILE(rscratch1, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
2678 rscratch1, ldarw);
2679 __ fmovs(as_FloatRegister($dst$$reg), rscratch1);
2680 %}
2681
2682 enc_class aarch64_enc_fldard(vRegD dst, memory mem) %{
2683 MOV_VOLATILE(rscratch1, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
2684 rscratch1, ldar);
2685 __ fmovd(as_FloatRegister($dst$$reg), rscratch1);
2686 %}
2687
2688 enc_class aarch64_enc_stlr(iRegL src, memory mem) %{
2689 Register src_reg = as_Register($src$$reg);
2690 // we sometimes get asked to store the stack pointer into the
2691 // current thread -- we cannot do that directly on AArch64
2692 if (src_reg == r31_sp) {
2693 MacroAssembler _masm(&cbuf);
2694 assert(as_Register($mem$$base) == rthread, "unexpected store for sp");
2695 __ mov(rscratch2, sp);
2696 src_reg = rscratch2;
2697 }
2698 MOV_VOLATILE(src_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
2699 rscratch1, stlr);
2700 if (VM_Version::cpu_cpuFeatures() & VM_Version::CPU_DMB_ATOMICS)
2701 __ dmb(__ ISH);
2702 %}
2703
2704 enc_class aarch64_enc_fstlrs(vRegF src, memory mem) %{
2705 {
2706 MacroAssembler _masm(&cbuf);
2707 FloatRegister src_reg = as_FloatRegister($src$$reg);
2708 __ fmovs(rscratch2, src_reg);
2709 }
2710 MOV_VOLATILE(rscratch2, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
2711 rscratch1, stlrw);
2712 if (VM_Version::cpu_cpuFeatures() & VM_Version::CPU_DMB_ATOMICS)
2713 __ dmb(__ ISH);
2714 %}
2715
2716 enc_class aarch64_enc_fstlrd(vRegD src, memory mem) %{
2717 {
2718 MacroAssembler _masm(&cbuf);
2719 FloatRegister src_reg = as_FloatRegister($src$$reg);
2720 __ fmovd(rscratch2, src_reg);
2721 }
2722 MOV_VOLATILE(rscratch2, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
2723 rscratch1, stlr);
2724 if (VM_Version::cpu_cpuFeatures() & VM_Version::CPU_DMB_ATOMICS)
2725 __ dmb(__ ISH);
2726 %}
2727
2728 // synchronized read/update encodings
2729
2730 enc_class aarch64_enc_ldaxr(iRegL dst, memory mem) %{
2731 MacroAssembler _masm(&cbuf);
2732 Register dst_reg = as_Register($dst$$reg);
2733 Register base = as_Register($mem$$base);
2734 int index = $mem$$index;
2735 int scale = $mem$$scale;
2736 int disp = $mem$$disp;
2737 if (index == -1) {
2738 if (disp != 0) {
2739 __ lea(rscratch1, Address(base, disp));
2740 __ ldaxr(dst_reg, rscratch1);
2741 } else {
2742 // TODO
2743 // should we ever get anything other than this case?
2744 __ ldaxr(dst_reg, base);
2745 }
2746 } else {
2747 Register index_reg = as_Register(index);
2748 if (disp == 0) {
2749 __ lea(rscratch1, Address(base, index_reg, Address::lsl(scale)));
2750 __ ldaxr(dst_reg, rscratch1);
2751 } else {
2752 __ lea(rscratch1, Address(base, disp));
2753 __ lea(rscratch1, Address(rscratch1, index_reg, Address::lsl(scale)));
2754 __ ldaxr(dst_reg, rscratch1);
2755 }
2756 }
2757 %}
2758
2759 enc_class aarch64_enc_stlxr(iRegLNoSp src, memory mem) %{
2760 MacroAssembler _masm(&cbuf);
2761 Register src_reg = as_Register($src$$reg);
2762 Register base = as_Register($mem$$base);
2763 int index = $mem$$index;
2764 int scale = $mem$$scale;
2765 int disp = $mem$$disp;
2766 if (index == -1) {
2767 if (disp != 0) {
2768 __ lea(rscratch2, Address(base, disp));
2769 __ stlxr(rscratch1, src_reg, rscratch2);
2770 } else {
2771 // TODO
2772 // should we ever get anything other than this case?
2773 __ stlxr(rscratch1, src_reg, base);
2774 }
2775 } else {
2776 Register index_reg = as_Register(index);
2777 if (disp == 0) {
2778 __ lea(rscratch2, Address(base, index_reg, Address::lsl(scale)));
2779 __ stlxr(rscratch1, src_reg, rscratch2);
2780 } else {
2781 __ lea(rscratch2, Address(base, disp));
2782 __ lea(rscratch2, Address(rscratch2, index_reg, Address::lsl(scale)));
2783 __ stlxr(rscratch1, src_reg, rscratch2);
2784 }
2785 }
2786 __ cmpw(rscratch1, zr);
2787 %}
2788
2789 enc_class aarch64_enc_cmpxchg(memory mem, iRegLNoSp oldval, iRegLNoSp newval) %{
2790 MacroAssembler _masm(&cbuf);
2791 guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
2792 __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
2793 Assembler::xword, /*acquire*/ false, /*release*/ true);
2794 %}
2795
2796 enc_class aarch64_enc_cmpxchgw(memory mem, iRegINoSp oldval, iRegINoSp newval) %{
2797 MacroAssembler _masm(&cbuf);
2798 guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
2799 __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
2800 Assembler::word, /*acquire*/ false, /*release*/ true);
2801 %}
2802
2803
2804 enc_class aarch64_enc_cmpxchg_oop_shenandoah(memory mem, iRegP oldval, iRegP newval, iRegPNoSp tmp, iRegINoSp res) %{
2805 MacroAssembler _masm(&cbuf);
2806 guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
2807 Register tmp = $tmp$$Register;
2808 __ mov(tmp, $oldval$$Register); // Must not clobber oldval.
2809 ShenandoahBarrierSetAssembler::bsasm()->cmpxchg_oop(&_masm, $mem$$Register, tmp, $newval$$Register,
2810 /*acquire*/ false, /*release*/ true, /*weak*/ false, /*is_cae*/ false, $res$$Register);
2811 %}
2812
2813 // The only difference between aarch64_enc_cmpxchg and
2814 // aarch64_enc_cmpxchg_acq is that we use load-acquire in the
2815 // CompareAndSwap sequence to serve as a barrier on acquiring a
2816 // lock.
2817 enc_class aarch64_enc_cmpxchg_acq(memory mem, iRegLNoSp oldval, iRegLNoSp newval) %{
2818 MacroAssembler _masm(&cbuf);
2819 guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
2820 __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
2821 Assembler::xword, /*acquire*/ true, /*release*/ true);
2822 %}
2823
2824 enc_class aarch64_enc_cmpxchgw_acq(memory mem, iRegINoSp oldval, iRegINoSp newval) %{
2825 MacroAssembler _masm(&cbuf);
2826 guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
2827 __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
2828 Assembler::word, /*acquire*/ true, /*release*/ true);
2829 %}
2830
2831 enc_class aarch64_enc_cmpxchg_acq_oop_shenandoah(memory mem, iRegP oldval, iRegP newval, iRegPNoSp tmp, iRegINoSp res) %{
2832 MacroAssembler _masm(&cbuf);
2833 guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
2834 Register tmp = $tmp$$Register;
2835 __ mov(tmp, $oldval$$Register); // Must not clobber oldval.
2836 ShenandoahBarrierSetAssembler::bsasm()->cmpxchg_oop(&_masm, $mem$$Register, tmp, $newval$$Register,
2837 /*acquire*/ true, /*release*/ true, /*weak*/ false, /*is_cae*/ false,
2838 $res$$Register);
2839 %}
2840
2841 // auxiliary used for CompareAndSwapX to set result register
2842 enc_class aarch64_enc_cset_eq(iRegINoSp res) %{
2843 MacroAssembler _masm(&cbuf);
2844 Register res_reg = as_Register($res$$reg);
2845 __ cset(res_reg, Assembler::EQ);
2846 %}
2847
2848 // prefetch encodings
2849
2850 enc_class aarch64_enc_prefetchr(memory mem) %{
2851 MacroAssembler _masm(&cbuf);
2852 Register base = as_Register($mem$$base);
2853 int index = $mem$$index;
2854 int scale = $mem$$scale;
2855 int disp = $mem$$disp;
2856 if (index == -1) {
2857 __ prfm(Address(base, disp), PLDL1KEEP);
2858 } else {
2859 Register index_reg = as_Register(index);
2860 if (disp == 0) {
2861 __ prfm(Address(base, index_reg, Address::lsl(scale)), PLDL1KEEP);
2862 } else {
2863 __ lea(rscratch1, Address(base, disp));
2864 __ prfm(Address(rscratch1, index_reg, Address::lsl(scale)), PLDL1KEEP);
2865 }
2866 }
2867 %}
2868
2869 enc_class aarch64_enc_prefetchw(memory mem) %{
2870 MacroAssembler _masm(&cbuf);
2871 Register base = as_Register($mem$$base);
2872 int index = $mem$$index;
2873 int scale = $mem$$scale;
2874 int disp = $mem$$disp;
2875 if (index == -1) {
2876 __ prfm(Address(base, disp), PSTL1KEEP);
2877 } else {
2878 Register index_reg = as_Register(index);
2879 if (disp == 0) {
2880 __ prfm(Address(base, index_reg, Address::lsl(scale)), PSTL1KEEP);
2881 } else {
2882 __ lea(rscratch1, Address(base, disp));
2883 __ prfm(Address(rscratch1, index_reg, Address::lsl(scale)), PSTL1KEEP);
2884 }
2885 }
2886 %}
2887
2888 enc_class aarch64_enc_prefetchnta(memory mem) %{
2889 MacroAssembler _masm(&cbuf);
2890 Register base = as_Register($mem$$base);
2891 int index = $mem$$index;
2892 int scale = $mem$$scale;
2893 int disp = $mem$$disp;
2894 if (index == -1) {
2895 __ prfm(Address(base, disp), PSTL1STRM);
2896 } else {
2897 Register index_reg = as_Register(index);
2898 if (disp == 0) {
2899 __ prfm(Address(base, index_reg, Address::lsl(scale)), PSTL1STRM);
2900 __ nop();
2901 } else {
2902 __ lea(rscratch1, Address(base, disp));
2903 __ prfm(Address(rscratch1, index_reg, Address::lsl(scale)), PSTL1STRM);
2904 }
2905 }
2906 %}
2907
2908 /// mov envcodings
2909
2910 enc_class aarch64_enc_movw_imm(iRegI dst, immI src) %{
2911 MacroAssembler _masm(&cbuf);
2912 u_int32_t con = (u_int32_t)$src$$constant;
2913 Register dst_reg = as_Register($dst$$reg);
2914 if (con == 0) {
2915 __ movw(dst_reg, zr);
2916 } else {
2917 __ movw(dst_reg, con);
2918 }
2919 %}
2920
2921 enc_class aarch64_enc_mov_imm(iRegL dst, immL src) %{
2922 MacroAssembler _masm(&cbuf);
2923 Register dst_reg = as_Register($dst$$reg);
2924 u_int64_t con = (u_int64_t)$src$$constant;
2925 if (con == 0) {
2926 __ mov(dst_reg, zr);
2927 } else {
2928 __ mov(dst_reg, con);
2929 }
2930 %}
2931
2932 enc_class aarch64_enc_mov_p(iRegP dst, immP src) %{
2933 MacroAssembler _masm(&cbuf);
2934 Register dst_reg = as_Register($dst$$reg);
2935 address con = (address)$src$$constant;
2936 if (con == NULL || con == (address)1) {
2937 ShouldNotReachHere();
2938 } else {
2939 relocInfo::relocType rtype = $src->constant_reloc();
2940 if (rtype == relocInfo::oop_type) {
2941 __ movoop(dst_reg, (jobject)con, /*immediate*/true);
2942 } else if (rtype == relocInfo::metadata_type) {
2943 __ mov_metadata(dst_reg, (Metadata*)con);
2944 } else {
2945 assert(rtype == relocInfo::none, "unexpected reloc type");
2946 if (con < (address)(uintptr_t)os::vm_page_size()) {
2947 __ mov(dst_reg, con);
2948 } else {
2949 unsigned long offset;
2950 __ adrp(dst_reg, con, offset);
2951 __ add(dst_reg, dst_reg, offset);
2952 }
2953 }
2954 }
2955 %}
2956
2957 enc_class aarch64_enc_mov_p0(iRegP dst, immP0 src) %{
2958 MacroAssembler _masm(&cbuf);
2959 Register dst_reg = as_Register($dst$$reg);
2960 __ mov(dst_reg, zr);
2961 %}
2962
2963 enc_class aarch64_enc_mov_p1(iRegP dst, immP_1 src) %{
2964 MacroAssembler _masm(&cbuf);
2965 Register dst_reg = as_Register($dst$$reg);
2966 __ mov(dst_reg, (u_int64_t)1);
2967 %}
2968
2969 enc_class aarch64_enc_mov_poll_page(iRegP dst, immPollPage src) %{
2970 MacroAssembler _masm(&cbuf);
2971 address page = (address)$src$$constant;
2972 Register dst_reg = as_Register($dst$$reg);
2973 unsigned long off;
2974 __ adrp(dst_reg, Address(page, relocInfo::poll_type), off);
2975 assert(off == 0, "assumed offset == 0");
2976 %}
2977
2978 enc_class aarch64_enc_mov_byte_map_base(iRegP dst, immByteMapBase src) %{
2979 MacroAssembler _masm(&cbuf);
2980 __ load_byte_map_base($dst$$Register);
2981 %}
2982
2983 enc_class aarch64_enc_mov_n(iRegN dst, immN src) %{
2984 MacroAssembler _masm(&cbuf);
2985 Register dst_reg = as_Register($dst$$reg);
2986 address con = (address)$src$$constant;
2987 if (con == NULL) {
2988 ShouldNotReachHere();
2989 } else {
2990 relocInfo::relocType rtype = $src->constant_reloc();
2991 assert(rtype == relocInfo::oop_type, "unexpected reloc type");
2992 __ set_narrow_oop(dst_reg, (jobject)con);
2993 }
2994 %}
2995
2996 enc_class aarch64_enc_mov_n0(iRegN dst, immN0 src) %{
2997 MacroAssembler _masm(&cbuf);
2998 Register dst_reg = as_Register($dst$$reg);
2999 __ mov(dst_reg, zr);
3000 %}
3001
3002 enc_class aarch64_enc_mov_nk(iRegN dst, immNKlass src) %{
3003 MacroAssembler _masm(&cbuf);
3004 Register dst_reg = as_Register($dst$$reg);
3005 address con = (address)$src$$constant;
3006 if (con == NULL) {
3007 ShouldNotReachHere();
3008 } else {
3009 relocInfo::relocType rtype = $src->constant_reloc();
3010 assert(rtype == relocInfo::metadata_type, "unexpected reloc type");
3011 __ set_narrow_klass(dst_reg, (Klass *)con);
3012 }
3013 %}
3014
3015 // arithmetic encodings
3016
3017 enc_class aarch64_enc_addsubw_imm(iRegI dst, iRegI src1, immIAddSub src2) %{
3018 MacroAssembler _masm(&cbuf);
3019 Register dst_reg = as_Register($dst$$reg);
3020 Register src_reg = as_Register($src1$$reg);
3021 int32_t con = (int32_t)$src2$$constant;
3022 // add has primary == 0, subtract has primary == 1
3023 if ($primary) { con = -con; }
3024 if (con < 0) {
3025 __ subw(dst_reg, src_reg, -con);
3026 } else {
3027 __ addw(dst_reg, src_reg, con);
3028 }
3029 %}
3030
3031 enc_class aarch64_enc_addsub_imm(iRegL dst, iRegL src1, immLAddSub src2) %{
3032 MacroAssembler _masm(&cbuf);
3033 Register dst_reg = as_Register($dst$$reg);
3034 Register src_reg = as_Register($src1$$reg);
3035 int32_t con = (int32_t)$src2$$constant;
3036 // add has primary == 0, subtract has primary == 1
3037 if ($primary) { con = -con; }
3038 if (con < 0) {
3039 __ sub(dst_reg, src_reg, -con);
3040 } else {
3041 __ add(dst_reg, src_reg, con);
3042 }
3043 %}
3044
3045 enc_class aarch64_enc_divw(iRegI dst, iRegI src1, iRegI src2) %{
3046 MacroAssembler _masm(&cbuf);
3047 Register dst_reg = as_Register($dst$$reg);
3048 Register src1_reg = as_Register($src1$$reg);
3049 Register src2_reg = as_Register($src2$$reg);
3050 __ corrected_idivl(dst_reg, src1_reg, src2_reg, false, rscratch1);
3051 %}
3052
3053 enc_class aarch64_enc_div(iRegI dst, iRegI src1, iRegI src2) %{
3054 MacroAssembler _masm(&cbuf);
3055 Register dst_reg = as_Register($dst$$reg);
3056 Register src1_reg = as_Register($src1$$reg);
3057 Register src2_reg = as_Register($src2$$reg);
3058 __ corrected_idivq(dst_reg, src1_reg, src2_reg, false, rscratch1);
3059 %}
3060
3061 enc_class aarch64_enc_modw(iRegI dst, iRegI src1, iRegI src2) %{
3062 MacroAssembler _masm(&cbuf);
3063 Register dst_reg = as_Register($dst$$reg);
3064 Register src1_reg = as_Register($src1$$reg);
3065 Register src2_reg = as_Register($src2$$reg);
3066 __ corrected_idivl(dst_reg, src1_reg, src2_reg, true, rscratch1);
3067 %}
3068
3069 enc_class aarch64_enc_mod(iRegI dst, iRegI src1, iRegI src2) %{
3070 MacroAssembler _masm(&cbuf);
3071 Register dst_reg = as_Register($dst$$reg);
3072 Register src1_reg = as_Register($src1$$reg);
3073 Register src2_reg = as_Register($src2$$reg);
3074 __ corrected_idivq(dst_reg, src1_reg, src2_reg, true, rscratch1);
3075 %}
3076
3077 // compare instruction encodings
3078
3079 enc_class aarch64_enc_cmpw(iRegI src1, iRegI src2) %{
3080 MacroAssembler _masm(&cbuf);
3081 Register reg1 = as_Register($src1$$reg);
3082 Register reg2 = as_Register($src2$$reg);
3083 __ cmpw(reg1, reg2);
3084 %}
3085
3086 enc_class aarch64_enc_cmpw_imm_addsub(iRegI src1, immIAddSub src2) %{
3087 MacroAssembler _masm(&cbuf);
3088 Register reg = as_Register($src1$$reg);
3089 int32_t val = $src2$$constant;
3090 if (val >= 0) {
3091 __ subsw(zr, reg, val);
3092 } else {
3093 __ addsw(zr, reg, -val);
3094 }
3095 %}
3096
3097 enc_class aarch64_enc_cmpw_imm(iRegI src1, immI src2) %{
3098 MacroAssembler _masm(&cbuf);
3099 Register reg1 = as_Register($src1$$reg);
3100 u_int32_t val = (u_int32_t)$src2$$constant;
3101 __ movw(rscratch1, val);
3102 __ cmpw(reg1, rscratch1);
3103 %}
3104
3105 enc_class aarch64_enc_cmp(iRegL src1, iRegL src2) %{
3106 MacroAssembler _masm(&cbuf);
3107 Register reg1 = as_Register($src1$$reg);
3108 Register reg2 = as_Register($src2$$reg);
3109 __ cmp(reg1, reg2);
3110 %}
3111
3112 enc_class aarch64_enc_cmp_imm_addsub(iRegL src1, immL12 src2) %{
3113 MacroAssembler _masm(&cbuf);
3114 Register reg = as_Register($src1$$reg);
3115 int64_t val = $src2$$constant;
3116 if (val >= 0) {
3117 __ subs(zr, reg, val);
3118 } else if (val != -val) {
3119 __ adds(zr, reg, -val);
3120 } else {
3121 // aargh, Long.MIN_VALUE is a special case
3122 __ orr(rscratch1, zr, (u_int64_t)val);
3123 __ subs(zr, reg, rscratch1);
3124 }
3125 %}
3126
3127 enc_class aarch64_enc_cmp_imm(iRegL src1, immL src2) %{
3128 MacroAssembler _masm(&cbuf);
3129 Register reg1 = as_Register($src1$$reg);
3130 u_int64_t val = (u_int64_t)$src2$$constant;
3131 __ mov(rscratch1, val);
3132 __ cmp(reg1, rscratch1);
3133 %}
3134
3135 enc_class aarch64_enc_cmpp(iRegP src1, iRegP src2) %{
3136 MacroAssembler _masm(&cbuf);
3137 Register reg1 = as_Register($src1$$reg);
3138 Register reg2 = as_Register($src2$$reg);
3139 __ cmp(reg1, reg2);
3140 %}
3141
3142 enc_class aarch64_enc_cmpn(iRegN src1, iRegN src2) %{
3143 MacroAssembler _masm(&cbuf);
3144 Register reg1 = as_Register($src1$$reg);
3145 Register reg2 = as_Register($src2$$reg);
3146 __ cmpw(reg1, reg2);
3147 %}
3148
3149 enc_class aarch64_enc_testp(iRegP src) %{
3150 MacroAssembler _masm(&cbuf);
3151 Register reg = as_Register($src$$reg);
3152 __ cmp(reg, zr);
3153 %}
3154
3155 enc_class aarch64_enc_testn(iRegN src) %{
3156 MacroAssembler _masm(&cbuf);
3157 Register reg = as_Register($src$$reg);
3158 __ cmpw(reg, zr);
3159 %}
3160
3161 enc_class aarch64_enc_b(label lbl) %{
3162 MacroAssembler _masm(&cbuf);
3163 Label *L = $lbl$$label;
3164 __ b(*L);
3165 %}
3166
3167 enc_class aarch64_enc_br_con(cmpOp cmp, label lbl) %{
3168 MacroAssembler _masm(&cbuf);
3169 Label *L = $lbl$$label;
3170 __ br ((Assembler::Condition)$cmp$$cmpcode, *L);
3171 %}
3172
3173 enc_class aarch64_enc_br_conU(cmpOpU cmp, label lbl) %{
3174 MacroAssembler _masm(&cbuf);
3175 Label *L = $lbl$$label;
3176 __ br ((Assembler::Condition)$cmp$$cmpcode, *L);
3177 %}
3178
3179 enc_class aarch64_enc_partial_subtype_check(iRegP sub, iRegP super, iRegP temp, iRegP result)
3180 %{
3181 Register sub_reg = as_Register($sub$$reg);
3182 Register super_reg = as_Register($super$$reg);
3183 Register temp_reg = as_Register($temp$$reg);
3184 Register result_reg = as_Register($result$$reg);
3185
3186 Label miss;
3187 MacroAssembler _masm(&cbuf);
3188 __ check_klass_subtype_slow_path(sub_reg, super_reg, temp_reg, result_reg,
3189 NULL, &miss,
3190 /*set_cond_codes:*/ true);
3191 if ($primary) {
3192 __ mov(result_reg, zr);
3193 }
3194 __ bind(miss);
3195 %}
3196
3197 enc_class aarch64_enc_java_static_call(method meth) %{
3198 MacroAssembler _masm(&cbuf);
3199
3200 address mark = __ pc();
3201 address addr = (address)$meth$$method;
3202 address call;
3203 if (!_method) {
3204 // A call to a runtime wrapper, e.g. new, new_typeArray_Java, uncommon_trap.
3205 call = __ trampoline_call(Address(addr, relocInfo::runtime_call_type), &cbuf);
3206 } else if (_optimized_virtual) {
3207 call = __ trampoline_call(Address(addr, relocInfo::opt_virtual_call_type), &cbuf);
3208 } else {
3209 call = __ trampoline_call(Address(addr, relocInfo::static_call_type), &cbuf);
3210 }
3211 if (call == NULL) {
3212 ciEnv::current()->record_failure("CodeCache is full");
3213 return;
3214 }
3215
3216 if (_method) {
3217 // Emit stub for static call
3218 address stub = CompiledStaticCall::emit_to_interp_stub(cbuf, mark);
3219 if (stub == NULL) {
3220 ciEnv::current()->record_failure("CodeCache is full");
3221 return;
3222 }
3223 }
3224 %}
3225
3226 enc_class aarch64_enc_java_handle_call(method meth) %{
3227 MacroAssembler _masm(&cbuf);
3228 relocInfo::relocType reloc;
3229
3230 // RFP is preserved across all calls, even compiled calls.
3231 // Use it to preserve SP.
3232 __ mov(rfp, sp);
3233
3234 address mark = __ pc();
3235 address addr = (address)$meth$$method;
3236 address call;
3237 if (!_method) {
3238 // A call to a runtime wrapper, e.g. new, new_typeArray_Java, uncommon_trap.
3239 call = __ trampoline_call(Address(addr, relocInfo::runtime_call_type), &cbuf);
3240 } else if (_optimized_virtual) {
3241 call = __ trampoline_call(Address(addr, relocInfo::opt_virtual_call_type), &cbuf);
3242 } else {
3243 call = __ trampoline_call(Address(addr, relocInfo::static_call_type), &cbuf);
3244 }
3245 if (call == NULL) {
3246 ciEnv::current()->record_failure("CodeCache is full");
3247 return;
3248 }
3249
3250 if (_method) {
3251 // Emit stub for static call
3252 address stub = CompiledStaticCall::emit_to_interp_stub(cbuf, mark);
3253 if (stub == NULL) {
3254 ciEnv::current()->record_failure("CodeCache is full");
3255 return;
3256 }
3257 }
3258
3259 // now restore sp
3260 __ mov(sp, rfp);
3261 %}
3262
3263 enc_class aarch64_enc_java_dynamic_call(method meth) %{
3264 MacroAssembler _masm(&cbuf);
3265 address call = __ ic_call((address)$meth$$method);
3266 if (call == NULL) {
3267 ciEnv::current()->record_failure("CodeCache is full");
3268 return;
3269 }
3270 %}
3271
3272 enc_class aarch64_enc_call_epilog() %{
3273 MacroAssembler _masm(&cbuf);
3274 if (VerifyStackAtCalls) {
3275 // Check that stack depth is unchanged: find majik cookie on stack
3276 __ call_Unimplemented();
3277 }
3278 %}
3279
3280 enc_class aarch64_enc_java_to_runtime(method meth) %{
3281 MacroAssembler _masm(&cbuf);
3282
3283 // some calls to generated routines (arraycopy code) are scheduled
3284 // by C2 as runtime calls. if so we can call them using a br (they
3285 // will be in a reachable segment) otherwise we have to use a blr
3286 // which loads the absolute address into a register.
3287 address entry = (address)$meth$$method;
3288 CodeBlob *cb = CodeCache::find_blob(entry);
3289 if (cb) {
3290 address call = __ trampoline_call(Address(entry, relocInfo::runtime_call_type));
3291 if (call == NULL) {
3292 ciEnv::current()->record_failure("CodeCache is full");
3293 return;
3294 }
3295 } else {
3296 Label retaddr;
3297 __ adr(rscratch2, retaddr);
3298 __ lea(rscratch1, RuntimeAddress(entry));
3299 // Leave a breadcrumb for JavaFrameAnchor::capture_last_Java_pc()
3300 __ stp(zr, rscratch2, Address(__ pre(sp, -2 * wordSize)));
3301 __ blr(rscratch1);
3302 __ bind(retaddr);
3303 __ add(sp, sp, 2 * wordSize);
3304 }
3305 %}
3306
3307 enc_class aarch64_enc_rethrow() %{
3308 MacroAssembler _masm(&cbuf);
3309 __ far_jump(RuntimeAddress(OptoRuntime::rethrow_stub()));
3310 %}
3311
3312 enc_class aarch64_enc_ret() %{
3313 MacroAssembler _masm(&cbuf);
3314 __ ret(lr);
3315 %}
3316
3317 enc_class aarch64_enc_tail_call(iRegP jump_target) %{
3318 MacroAssembler _masm(&cbuf);
3319 Register target_reg = as_Register($jump_target$$reg);
3320 __ br(target_reg);
3321 %}
3322
3323 enc_class aarch64_enc_tail_jmp(iRegP jump_target) %{
3324 MacroAssembler _masm(&cbuf);
3325 Register target_reg = as_Register($jump_target$$reg);
3326 // exception oop should be in r0
3327 // ret addr has been popped into lr
3328 // callee expects it in r3
3329 __ mov(r3, lr);
3330 __ br(target_reg);
3331 %}
3332
3333 enc_class aarch64_enc_fast_lock(iRegP object, iRegP box, iRegP tmp, iRegP tmp2) %{
3334 MacroAssembler _masm(&cbuf);
3335 Register oop = as_Register($object$$reg);
3336 Register box = as_Register($box$$reg);
3337 Register disp_hdr = as_Register($tmp$$reg);
3338 Register tmp = as_Register($tmp2$$reg);
3339 Label cont;
3340 Label object_has_monitor;
3341 Label cas_failed;
3342
3343 assert_different_registers(oop, box, tmp, disp_hdr);
3344
3345 // Load markOop from object into displaced_header.
3346 __ ldr(disp_hdr, Address(oop, oopDesc::mark_offset_in_bytes()));
3347
3348 // Always do locking in runtime.
3349 if (EmitSync & 0x01) {
3350 __ cmp(oop, zr);
3351 return;
3352 }
3353
3354 if (UseBiasedLocking && !UseOptoBiasInlining) {
3355 __ biased_locking_enter(box, oop, disp_hdr, tmp, true, cont);
3356 }
3357
3358 // Handle existing monitor
3359 if ((EmitSync & 0x02) == 0) {
3360 __ tbnz(disp_hdr, exact_log2(markOopDesc::monitor_value), object_has_monitor);
3361 }
3362
3363 // Set tmp to be (markOop of object | UNLOCK_VALUE).
3364 __ orr(tmp, disp_hdr, markOopDesc::unlocked_value);
3365
3366 // Load Compare Value application register.
3367
3368 // Initialize the box. (Must happen before we update the object mark!)
3369 __ str(tmp, Address(box, BasicLock::displaced_header_offset_in_bytes()));
3370
3371 // Compare object markOop with an unlocked value (tmp) and if
3372 // equal exchange the stack address of our box with object markOop.
3373 // On failure disp_hdr contains the possibly locked markOop.
3374 if (UseLSE) {
3375 __ mov(disp_hdr, tmp);
3376 __ casal(Assembler::xword, disp_hdr, box, oop); // Updates disp_hdr
3377 __ cmp(tmp, disp_hdr);
3378 __ br(Assembler::EQ, cont);
3379 } else {
3380 Label retry_load;
3381 if ((VM_Version::cpu_cpuFeatures() & VM_Version::CPU_STXR_PREFETCH))
3382 __ prfm(Address(oop), PSTL1STRM);
3383 __ bind(retry_load);
3384 __ ldaxr(disp_hdr, oop);
3385 __ cmp(tmp, disp_hdr);
3386 __ br(Assembler::NE, cas_failed);
3387 // use stlxr to ensure update is immediately visible
3388 __ stlxr(disp_hdr, box, oop);
3389 __ cbzw(disp_hdr, cont);
3390 __ b(retry_load);
3391 }
3392
3393 assert(oopDesc::mark_offset_in_bytes() == 0, "offset of _mark is not 0");
3394
3395 // If the compare-and-exchange succeeded, then we found an unlocked
3396 // object, will have now locked it will continue at label cont
3397
3398 __ bind(cas_failed);
3399 // We did not see an unlocked object so try the fast recursive case.
3400
3401 // Check if the owner is self by comparing the value in the
3402 // markOop of object (disp_hdr) with the stack pointer.
3403 __ mov(rscratch1, sp);
3404 __ sub(disp_hdr, disp_hdr, rscratch1);
3405 __ mov(tmp, (address) (~(os::vm_page_size()-1) | (uintptr_t)markOopDesc::lock_mask_in_place));
3406 // If condition is true we are cont and hence we can store 0 as the
3407 // displaced header in the box, which indicates that it is a recursive lock.
3408 __ ands(tmp/*==0?*/, disp_hdr, tmp);
3409 __ str(tmp/*==0, perhaps*/, Address(box, BasicLock::displaced_header_offset_in_bytes()));
3410
3411 // Handle existing monitor.
3412 if ((EmitSync & 0x02) == 0) {
3413 __ b(cont);
3414
3415 __ bind(object_has_monitor);
3416 // The object's monitor m is unlocked iff m->owner == NULL,
3417 // otherwise m->owner may contain a thread or a stack address.
3418 //
3419 // Try to CAS m->owner from NULL to current thread.
3420 __ add(tmp, disp_hdr, (ObjectMonitor::owner_offset_in_bytes()-markOopDesc::monitor_value));
3421 __ mov(disp_hdr, zr);
3422
3423 if (UseLSE) {
3424 __ mov(rscratch1, disp_hdr);
3425 __ casal(Assembler::xword, rscratch1, rthread, tmp);
3426 __ cmp(rscratch1, disp_hdr);
3427 } else {
3428 Label retry_load, fail;
3429 if ((VM_Version::cpu_cpuFeatures() & VM_Version::CPU_STXR_PREFETCH))
3430 __ prfm(Address(tmp), PSTL1STRM);
3431 __ bind(retry_load);
3432 __ ldaxr(rscratch1, tmp);
3433 __ cmp(disp_hdr, rscratch1);
3434 __ br(Assembler::NE, fail);
3435 // use stlxr to ensure update is immediately visible
3436 __ stlxr(rscratch1, rthread, tmp);
3437 __ cbnzw(rscratch1, retry_load);
3438 __ bind(fail);
3439 }
3440
3441 // Store a non-null value into the box to avoid looking like a re-entrant
3442 // lock. The fast-path monitor unlock code checks for
3443 // markOopDesc::monitor_value so use markOopDesc::unused_mark which has the
3444 // relevant bit set, and also matches ObjectSynchronizer::slow_enter.
3445 __ mov(tmp, (address)markOopDesc::unused_mark());
3446 __ str(tmp, Address(box, BasicLock::displaced_header_offset_in_bytes()));
3447 }
3448
3449 __ bind(cont);
3450 // flag == EQ indicates success
3451 // flag == NE indicates failure
3452 %}
3453
3454 enc_class aarch64_enc_fast_unlock(iRegP object, iRegP box, iRegP tmp, iRegP tmp2) %{
3455 MacroAssembler _masm(&cbuf);
3456 Register oop = as_Register($object$$reg);
3457 Register box = as_Register($box$$reg);
3458 Register disp_hdr = as_Register($tmp$$reg);
3459 Register tmp = as_Register($tmp2$$reg);
3460 Label cont;
3461 Label object_has_monitor;
3462
3463 assert_different_registers(oop, box, tmp, disp_hdr);
3464
3465 // Always do locking in runtime.
3466 if (EmitSync & 0x01) {
3467 __ cmp(oop, zr); // Oop can't be 0 here => always false.
3468 return;
3469 }
3470
3471 if (UseBiasedLocking && !UseOptoBiasInlining) {
3472 __ biased_locking_exit(oop, tmp, cont);
3473 }
3474
3475 // Find the lock address and load the displaced header from the stack.
3476 __ ldr(disp_hdr, Address(box, BasicLock::displaced_header_offset_in_bytes()));
3477
3478 // If the displaced header is 0, we have a recursive unlock.
3479 __ cmp(disp_hdr, zr);
3480 __ br(Assembler::EQ, cont);
3481
3482 // Handle existing monitor.
3483 if ((EmitSync & 0x02) == 0) {
3484 __ ldr(tmp, Address(oop, oopDesc::mark_offset_in_bytes()));
3485 __ tbnz(disp_hdr, exact_log2(markOopDesc::monitor_value), object_has_monitor);
3486 }
3487
3488 // Check if it is still a light weight lock, this is is true if we
3489 // see the stack address of the basicLock in the markOop of the
3490 // object.
3491
3492 if (UseLSE) {
3493 __ mov(tmp, box);
3494 __ casl(Assembler::xword, tmp, disp_hdr, oop);
3495 __ cmp(tmp, box);
3496 __ b(cont);
3497 } else {
3498 Label retry_load;
3499 if ((VM_Version::cpu_cpuFeatures() & VM_Version::CPU_STXR_PREFETCH))
3500 __ prfm(Address(oop), PSTL1STRM);
3501 __ bind(retry_load);
3502 __ ldxr(tmp, oop);
3503 __ cmp(box, tmp);
3504 __ br(Assembler::NE, cont);
3505 // use stlxr to ensure update is immediately visible
3506 __ stlxr(tmp, disp_hdr, oop);
3507 __ cbzw(tmp, cont);
3508 __ b(retry_load);
3509 }
3510
3511 assert(oopDesc::mark_offset_in_bytes() == 0, "offset of _mark is not 0");
3512
3513 // Handle existing monitor.
3514 if ((EmitSync & 0x02) == 0) {
3515 __ bind(object_has_monitor);
3516 __ add(tmp, tmp, -markOopDesc::monitor_value); // monitor
3517 __ ldr(rscratch1, Address(tmp, ObjectMonitor::owner_offset_in_bytes()));
3518 __ ldr(disp_hdr, Address(tmp, ObjectMonitor::recursions_offset_in_bytes()));
3519 __ eor(rscratch1, rscratch1, rthread); // Will be 0 if we are the owner.
3520 __ orr(rscratch1, rscratch1, disp_hdr); // Will be 0 if there are 0 recursions
3521 __ cmp(rscratch1, zr);
3522 __ br(Assembler::NE, cont);
3523
3524 __ ldr(rscratch1, Address(tmp, ObjectMonitor::EntryList_offset_in_bytes()));
3525 __ ldr(disp_hdr, Address(tmp, ObjectMonitor::cxq_offset_in_bytes()));
3526 __ orr(rscratch1, rscratch1, disp_hdr); // Will be 0 if both are 0.
3527 __ cmp(rscratch1, zr);
3528 __ br(Assembler::NE, cont);
3529 // need a release store here
3530 __ lea(tmp, Address(tmp, ObjectMonitor::owner_offset_in_bytes()));
3531 __ stlr(zr, tmp); // set unowned
3532 }
3533
3534 __ bind(cont);
3535 // flag == EQ indicates success
3536 // flag == NE indicates failure
3537 %}
3538
3539 %}
3540
3541 //----------FRAME--------------------------------------------------------------
3542 // Definition of frame structure and management information.
3543 //
3544 // S T A C K L A Y O U T Allocators stack-slot number
3545 // | (to get allocators register number
3546 // G Owned by | | v add OptoReg::stack0())
3547 // r CALLER | |
3548 // o | +--------+ pad to even-align allocators stack-slot
3549 // w V | pad0 | numbers; owned by CALLER
3550 // t -----------+--------+----> Matcher::_in_arg_limit, unaligned
3551 // h ^ | in | 5
3552 // | | args | 4 Holes in incoming args owned by SELF
3553 // | | | | 3
3554 // | | +--------+
3555 // V | | old out| Empty on Intel, window on Sparc
3556 // | old |preserve| Must be even aligned.
3557 // | SP-+--------+----> Matcher::_old_SP, even aligned
3558 // | | in | 3 area for Intel ret address
3559 // Owned by |preserve| Empty on Sparc.
3560 // SELF +--------+
3561 // | | pad2 | 2 pad to align old SP
3562 // | +--------+ 1
3563 // | | locks | 0
3564 // | +--------+----> OptoReg::stack0(), even aligned
3565 // | | pad1 | 11 pad to align new SP
3566 // | +--------+
3567 // | | | 10
3568 // | | spills | 9 spills
3569 // V | | 8 (pad0 slot for callee)
3570 // -----------+--------+----> Matcher::_out_arg_limit, unaligned
3571 // ^ | out | 7
3572 // | | args | 6 Holes in outgoing args owned by CALLEE
3573 // Owned by +--------+
3574 // CALLEE | new out| 6 Empty on Intel, window on Sparc
3575 // | new |preserve| Must be even-aligned.
3576 // | SP-+--------+----> Matcher::_new_SP, even aligned
3577 // | | |
3578 //
3579 // Note 1: Only region 8-11 is determined by the allocator. Region 0-5 is
3580 // known from SELF's arguments and the Java calling convention.
3581 // Region 6-7 is determined per call site.
3582 // Note 2: If the calling convention leaves holes in the incoming argument
3583 // area, those holes are owned by SELF. Holes in the outgoing area
3584 // are owned by the CALLEE. Holes should not be nessecary in the
3585 // incoming area, as the Java calling convention is completely under
3586 // the control of the AD file. Doubles can be sorted and packed to
3587 // avoid holes. Holes in the outgoing arguments may be nessecary for
3588 // varargs C calling conventions.
3589 // Note 3: Region 0-3 is even aligned, with pad2 as needed. Region 3-5 is
3590 // even aligned with pad0 as needed.
3591 // Region 6 is even aligned. Region 6-7 is NOT even aligned;
3592 // (the latter is true on Intel but is it false on AArch64?)
3593 // region 6-11 is even aligned; it may be padded out more so that
3594 // the region from SP to FP meets the minimum stack alignment.
3595 // Note 4: For I2C adapters, the incoming FP may not meet the minimum stack
3596 // alignment. Region 11, pad1, may be dynamically extended so that
3597 // SP meets the minimum alignment.
3598
3599 frame %{
3600 // What direction does stack grow in (assumed to be same for C & Java)
3601 stack_direction(TOWARDS_LOW);
3602
3603 // These three registers define part of the calling convention
3604 // between compiled code and the interpreter.
3605
3606 // Inline Cache Register or methodOop for I2C.
3607 inline_cache_reg(R12);
3608
3609 // Method Oop Register when calling interpreter.
3610 interpreter_method_oop_reg(R12);
3611
3612 // Number of stack slots consumed by locking an object
3613 sync_stack_slots(2);
3614
3615 // Compiled code's Frame Pointer
3616 frame_pointer(R31);
3617
3618 // Interpreter stores its frame pointer in a register which is
3619 // stored to the stack by I2CAdaptors.
3620 // I2CAdaptors convert from interpreted java to compiled java.
3621 interpreter_frame_pointer(R29);
3622
3623 // Stack alignment requirement
3624 stack_alignment(StackAlignmentInBytes); // Alignment size in bytes (128-bit -> 16 bytes)
3625
3626 // Number of stack slots between incoming argument block and the start of
3627 // a new frame. The PROLOG must add this many slots to the stack. The
3628 // EPILOG must remove this many slots. aarch64 needs two slots for
3629 // return address and fp.
3630 // TODO think this is correct but check
3631 in_preserve_stack_slots(4);
3632
3633 // Number of outgoing stack slots killed above the out_preserve_stack_slots
3634 // for calls to C. Supports the var-args backing area for register parms.
3635 varargs_C_out_slots_killed(frame::arg_reg_save_area_bytes/BytesPerInt);
3636
3637 // The after-PROLOG location of the return address. Location of
3638 // return address specifies a type (REG or STACK) and a number
3639 // representing the register number (i.e. - use a register name) or
3640 // stack slot.
3641 // Ret Addr is on stack in slot 0 if no locks or verification or alignment.
3642 // Otherwise, it is above the locks and verification slot and alignment word
3643 // TODO this may well be correct but need to check why that - 2 is there
3644 // ppc port uses 0 but we definitely need to allow for fixed_slots
3645 // which folds in the space used for monitors
3646 return_addr(STACK - 2 +
3647 round_to((Compile::current()->in_preserve_stack_slots() +
3648 Compile::current()->fixed_slots()),
3649 stack_alignment_in_slots()));
3650
3651 // Body of function which returns an integer array locating
3652 // arguments either in registers or in stack slots. Passed an array
3653 // of ideal registers called "sig" and a "length" count. Stack-slot
3654 // offsets are based on outgoing arguments, i.e. a CALLER setting up
3655 // arguments for a CALLEE. Incoming stack arguments are
3656 // automatically biased by the preserve_stack_slots field above.
3657
3658 calling_convention
3659 %{
3660 // No difference between ingoing/outgoing just pass false
3661 SharedRuntime::java_calling_convention(sig_bt, regs, length, false);
3662 %}
3663
3664 c_calling_convention
3665 %{
3666 // This is obviously always outgoing
3667 (void) SharedRuntime::c_calling_convention(sig_bt, regs, NULL, length);
3668 %}
3669
3670 // Location of compiled Java return values. Same as C for now.
3671 return_value
3672 %{
3673 // TODO do we allow ideal_reg == Op_RegN???
3674 assert(ideal_reg >= Op_RegI && ideal_reg <= Op_RegL,
3675 "only return normal values");
3676
3677 static const int lo[Op_RegL + 1] = { // enum name
3678 0, // Op_Node
3679 0, // Op_Set
3680 R0_num, // Op_RegN
3681 R0_num, // Op_RegI
3682 R0_num, // Op_RegP
3683 V0_num, // Op_RegF
3684 V0_num, // Op_RegD
3685 R0_num // Op_RegL
3686 };
3687
3688 static const int hi[Op_RegL + 1] = { // enum name
3689 0, // Op_Node
3690 0, // Op_Set
3691 OptoReg::Bad, // Op_RegN
3692 OptoReg::Bad, // Op_RegI
3693 R0_H_num, // Op_RegP
3694 OptoReg::Bad, // Op_RegF
3695 V0_H_num, // Op_RegD
3696 R0_H_num // Op_RegL
3697 };
3698
3699 return OptoRegPair(hi[ideal_reg], lo[ideal_reg]);
3700 %}
3701 %}
3702
3703 //----------ATTRIBUTES---------------------------------------------------------
3704 //----------Operand Attributes-------------------------------------------------
3705 op_attrib op_cost(1); // Required cost attribute
3706
3707 //----------Instruction Attributes---------------------------------------------
3708 ins_attrib ins_cost(INSN_COST); // Required cost attribute
3709 ins_attrib ins_size(32); // Required size attribute (in bits)
3710 ins_attrib ins_short_branch(0); // Required flag: is this instruction
3711 // a non-matching short branch variant
3712 // of some long branch?
3713 ins_attrib ins_alignment(4); // Required alignment attribute (must
3714 // be a power of 2) specifies the
3715 // alignment that some part of the
3716 // instruction (not necessarily the
3717 // start) requires. If > 1, a
3718 // compute_padding() function must be
3719 // provided for the instruction
3720
3721 //----------OPERANDS-----------------------------------------------------------
3722 // Operand definitions must precede instruction definitions for correct parsing
3723 // in the ADLC because operands constitute user defined types which are used in
3724 // instruction definitions.
3725
3726 //----------Simple Operands----------------------------------------------------
3727
3728 // Integer operands 32 bit
3729 // 32 bit immediate
3730 operand immI()
3731 %{
3732 match(ConI);
3733
3734 op_cost(0);
3735 format %{ %}
3736 interface(CONST_INTER);
3737 %}
3738
3739 // 32 bit zero
3740 operand immI0()
3741 %{
3742 predicate(n->get_int() == 0);
3743 match(ConI);
3744
3745 op_cost(0);
3746 format %{ %}
3747 interface(CONST_INTER);
3748 %}
3749
3750 // 32 bit unit increment
3751 operand immI_1()
3752 %{
3753 predicate(n->get_int() == 1);
3754 match(ConI);
3755
3756 op_cost(0);
3757 format %{ %}
3758 interface(CONST_INTER);
3759 %}
3760
3761 // 32 bit unit decrement
3762 operand immI_M1()
3763 %{
3764 predicate(n->get_int() == -1);
3765 match(ConI);
3766
3767 op_cost(0);
3768 format %{ %}
3769 interface(CONST_INTER);
3770 %}
3771
3772 operand immI_le_4()
3773 %{
3774 predicate(n->get_int() <= 4);
3775 match(ConI);
3776
3777 op_cost(0);
3778 format %{ %}
3779 interface(CONST_INTER);
3780 %}
3781
3782 operand immI_31()
3783 %{
3784 predicate(n->get_int() == 31);
3785 match(ConI);
3786
3787 op_cost(0);
3788 format %{ %}
3789 interface(CONST_INTER);
3790 %}
3791
3792 operand immI_8()
3793 %{
3794 predicate(n->get_int() == 8);
3795 match(ConI);
3796
3797 op_cost(0);
3798 format %{ %}
3799 interface(CONST_INTER);
3800 %}
3801
3802 operand immI_16()
3803 %{
3804 predicate(n->get_int() == 16);
3805 match(ConI);
3806
3807 op_cost(0);
3808 format %{ %}
3809 interface(CONST_INTER);
3810 %}
3811
3812 operand immI_24()
3813 %{
3814 predicate(n->get_int() == 24);
3815 match(ConI);
3816
3817 op_cost(0);
3818 format %{ %}
3819 interface(CONST_INTER);
3820 %}
3821
3822 operand immI_32()
3823 %{
3824 predicate(n->get_int() == 32);
3825 match(ConI);
3826
3827 op_cost(0);
3828 format %{ %}
3829 interface(CONST_INTER);
3830 %}
3831
3832 operand immI_48()
3833 %{
3834 predicate(n->get_int() == 48);
3835 match(ConI);
3836
3837 op_cost(0);
3838 format %{ %}
3839 interface(CONST_INTER);
3840 %}
3841
3842 operand immI_56()
3843 %{
3844 predicate(n->get_int() == 56);
3845 match(ConI);
3846
3847 op_cost(0);
3848 format %{ %}
3849 interface(CONST_INTER);
3850 %}
3851
3852 operand immI_64()
3853 %{
3854 predicate(n->get_int() == 64);
3855 match(ConI);
3856
3857 op_cost(0);
3858 format %{ %}
3859 interface(CONST_INTER);
3860 %}
3861
3862 operand immI_255()
3863 %{
3864 predicate(n->get_int() == 255);
3865 match(ConI);
3866
3867 op_cost(0);
3868 format %{ %}
3869 interface(CONST_INTER);
3870 %}
3871
3872 operand immI_65535()
3873 %{
3874 predicate(n->get_int() == 65535);
3875 match(ConI);
3876
3877 op_cost(0);
3878 format %{ %}
3879 interface(CONST_INTER);
3880 %}
3881
3882 operand immL_63()
3883 %{
3884 predicate(n->get_int() == 63);
3885 match(ConI);
3886
3887 op_cost(0);
3888 format %{ %}
3889 interface(CONST_INTER);
3890 %}
3891
3892 operand immL_255()
3893 %{
3894 predicate(n->get_int() == 255);
3895 match(ConI);
3896
3897 op_cost(0);
3898 format %{ %}
3899 interface(CONST_INTER);
3900 %}
3901
3902 operand immL_65535()
3903 %{
3904 predicate(n->get_long() == 65535L);
3905 match(ConL);
3906
3907 op_cost(0);
3908 format %{ %}
3909 interface(CONST_INTER);
3910 %}
3911
3912 operand immL_4294967295()
3913 %{
3914 predicate(n->get_long() == 4294967295L);
3915 match(ConL);
3916
3917 op_cost(0);
3918 format %{ %}
3919 interface(CONST_INTER);
3920 %}
3921
3922 operand immL_bitmask()
3923 %{
3924 predicate((n->get_long() != 0)
3925 && ((n->get_long() & 0xc000000000000000l) == 0)
3926 && is_power_of_2(n->get_long() + 1));
3927 match(ConL);
3928
3929 op_cost(0);
3930 format %{ %}
3931 interface(CONST_INTER);
3932 %}
3933
3934 operand immI_bitmask()
3935 %{
3936 predicate((n->get_int() != 0)
3937 && ((n->get_int() & 0xc0000000) == 0)
3938 && is_power_of_2(n->get_int() + 1));
3939 match(ConI);
3940
3941 op_cost(0);
3942 format %{ %}
3943 interface(CONST_INTER);
3944 %}
3945
3946 // Scale values for scaled offset addressing modes (up to long but not quad)
3947 operand immIScale()
3948 %{
3949 predicate(0 <= n->get_int() && (n->get_int() <= 3));
3950 match(ConI);
3951
3952 op_cost(0);
3953 format %{ %}
3954 interface(CONST_INTER);
3955 %}
3956
3957 // 26 bit signed offset -- for pc-relative branches
3958 operand immI26()
3959 %{
3960 predicate(((-(1 << 25)) <= n->get_int()) && (n->get_int() < (1 << 25)));
3961 match(ConI);
3962
3963 op_cost(0);
3964 format %{ %}
3965 interface(CONST_INTER);
3966 %}
3967
3968 // 19 bit signed offset -- for pc-relative loads
3969 operand immI19()
3970 %{
3971 predicate(((-(1 << 18)) <= n->get_int()) && (n->get_int() < (1 << 18)));
3972 match(ConI);
3973
3974 op_cost(0);
3975 format %{ %}
3976 interface(CONST_INTER);
3977 %}
3978
3979 // 12 bit unsigned offset -- for base plus immediate loads
3980 operand immIU12()
3981 %{
3982 predicate((0 <= n->get_int()) && (n->get_int() < (1 << 12)));
3983 match(ConI);
3984
3985 op_cost(0);
3986 format %{ %}
3987 interface(CONST_INTER);
3988 %}
3989
3990 operand immLU12()
3991 %{
3992 predicate((0 <= n->get_long()) && (n->get_long() < (1 << 12)));
3993 match(ConL);
3994
3995 op_cost(0);
3996 format %{ %}
3997 interface(CONST_INTER);
3998 %}
3999
4000 // Offset for scaled or unscaled immediate loads and stores
4001 operand immIOffset()
4002 %{
4003 predicate(Address::offset_ok_for_immed(n->get_int()));
4004 match(ConI);
4005
4006 op_cost(0);
4007 format %{ %}
4008 interface(CONST_INTER);
4009 %}
4010
4011 operand immIOffset4()
4012 %{
4013 predicate(Address::offset_ok_for_immed(n->get_int(), 2));
4014 match(ConI);
4015
4016 op_cost(0);
4017 format %{ %}
4018 interface(CONST_INTER);
4019 %}
4020
4021 operand immIOffset8()
4022 %{
4023 predicate(Address::offset_ok_for_immed(n->get_int(), 3));
4024 match(ConI);
4025
4026 op_cost(0);
4027 format %{ %}
4028 interface(CONST_INTER);
4029 %}
4030
4031 operand immIOffset16()
4032 %{
4033 predicate(Address::offset_ok_for_immed(n->get_int(), 4));
4034 match(ConI);
4035
4036 op_cost(0);
4037 format %{ %}
4038 interface(CONST_INTER);
4039 %}
4040
4041 operand immLoffset()
4042 %{
4043 predicate(Address::offset_ok_for_immed(n->get_long()));
4044 match(ConL);
4045
4046 op_cost(0);
4047 format %{ %}
4048 interface(CONST_INTER);
4049 %}
4050
4051 operand immLoffset4()
4052 %{
4053 predicate(Address::offset_ok_for_immed(n->get_long(), 2));
4054 match(ConL);
4055
4056 op_cost(0);
4057 format %{ %}
4058 interface(CONST_INTER);
4059 %}
4060
4061 operand immLoffset8()
4062 %{
4063 predicate(Address::offset_ok_for_immed(n->get_long(), 3));
4064 match(ConL);
4065
4066 op_cost(0);
4067 format %{ %}
4068 interface(CONST_INTER);
4069 %}
4070
4071 operand immLoffset16()
4072 %{
4073 predicate(Address::offset_ok_for_immed(n->get_long(), 4));
4074 match(ConL);
4075
4076 op_cost(0);
4077 format %{ %}
4078 interface(CONST_INTER);
4079 %}
4080
4081 // 32 bit integer valid for add sub immediate
4082 operand immIAddSub()
4083 %{
4084 predicate(Assembler::operand_valid_for_add_sub_immediate((long)n->get_int()));
4085 match(ConI);
4086 op_cost(0);
4087 format %{ %}
4088 interface(CONST_INTER);
4089 %}
4090
4091 // 32 bit unsigned integer valid for logical immediate
4092 // TODO -- check this is right when e.g the mask is 0x80000000
4093 operand immILog()
4094 %{
4095 predicate(Assembler::operand_valid_for_logical_immediate(/*is32*/true, (unsigned long)n->get_int()));
4096 match(ConI);
4097
4098 op_cost(0);
4099 format %{ %}
4100 interface(CONST_INTER);
4101 %}
4102
4103 // Integer operands 64 bit
4104 // 64 bit immediate
4105 operand immL()
4106 %{
4107 match(ConL);
4108
4109 op_cost(0);
4110 format %{ %}
4111 interface(CONST_INTER);
4112 %}
4113
4114 // 64 bit zero
4115 operand immL0()
4116 %{
4117 predicate(n->get_long() == 0);
4118 match(ConL);
4119
4120 op_cost(0);
4121 format %{ %}
4122 interface(CONST_INTER);
4123 %}
4124
4125 // 64 bit unit increment
4126 operand immL_1()
4127 %{
4128 predicate(n->get_long() == 1);
4129 match(ConL);
4130
4131 op_cost(0);
4132 format %{ %}
4133 interface(CONST_INTER);
4134 %}
4135
4136 // 64 bit unit decrement
4137 operand immL_M1()
4138 %{
4139 predicate(n->get_long() == -1);
4140 match(ConL);
4141
4142 op_cost(0);
4143 format %{ %}
4144 interface(CONST_INTER);
4145 %}
4146
4147 // 32 bit offset of pc in thread anchor
4148
4149 operand immL_pc_off()
4150 %{
4151 predicate(n->get_long() == in_bytes(JavaThread::frame_anchor_offset()) +
4152 in_bytes(JavaFrameAnchor::last_Java_pc_offset()));
4153 match(ConL);
4154
4155 op_cost(0);
4156 format %{ %}
4157 interface(CONST_INTER);
4158 %}
4159
4160 // 64 bit integer valid for add sub immediate
4161 operand immLAddSub()
4162 %{
4163 predicate(Assembler::operand_valid_for_add_sub_immediate(n->get_long()));
4164 match(ConL);
4165 op_cost(0);
4166 format %{ %}
4167 interface(CONST_INTER);
4168 %}
4169
4170 // 64 bit integer valid for logical immediate
4171 operand immLLog()
4172 %{
4173 predicate(Assembler::operand_valid_for_logical_immediate(/*is32*/false, (unsigned long)n->get_long()));
4174 match(ConL);
4175 op_cost(0);
4176 format %{ %}
4177 interface(CONST_INTER);
4178 %}
4179
4180 // Long Immediate: low 32-bit mask
4181 operand immL_32bits()
4182 %{
4183 predicate(n->get_long() == 0xFFFFFFFFL);
4184 match(ConL);
4185 op_cost(0);
4186 format %{ %}
4187 interface(CONST_INTER);
4188 %}
4189
4190 // Pointer operands
4191 // Pointer Immediate
4192 operand immP()
4193 %{
4194 match(ConP);
4195
4196 op_cost(0);
4197 format %{ %}
4198 interface(CONST_INTER);
4199 %}
4200
4201 // NULL Pointer Immediate
4202 operand immP0()
4203 %{
4204 predicate(n->get_ptr() == 0);
4205 match(ConP);
4206
4207 op_cost(0);
4208 format %{ %}
4209 interface(CONST_INTER);
4210 %}
4211
4212 // Pointer Immediate One
4213 // this is used in object initialization (initial object header)
4214 operand immP_1()
4215 %{
4216 predicate(n->get_ptr() == 1);
4217 match(ConP);
4218
4219 op_cost(0);
4220 format %{ %}
4221 interface(CONST_INTER);
4222 %}
4223
4224 // Polling Page Pointer Immediate
4225 operand immPollPage()
4226 %{
4227 predicate((address)n->get_ptr() == os::get_polling_page());
4228 match(ConP);
4229
4230 op_cost(0);
4231 format %{ %}
4232 interface(CONST_INTER);
4233 %}
4234
4235 // Card Table Byte Map Base
4236 operand immByteMapBase()
4237 %{
4238 // Get base of card map
4239 predicate(!UseShenandoahGC && // TODO: Should really check for BS::is_a, see JDK-8193193
4240 (jbyte*)n->get_ptr() == ((CardTableModRefBS*)(Universe::heap()->barrier_set()))->byte_map_base);
4241 match(ConP);
4242
4243 op_cost(0);
4244 format %{ %}
4245 interface(CONST_INTER);
4246 %}
4247
4248 // Pointer Immediate Minus One
4249 // this is used when we want to write the current PC to the thread anchor
4250 operand immP_M1()
4251 %{
4252 predicate(n->get_ptr() == -1);
4253 match(ConP);
4254
4255 op_cost(0);
4256 format %{ %}
4257 interface(CONST_INTER);
4258 %}
4259
4260 // Pointer Immediate Minus Two
4261 // this is used when we want to write the current PC to the thread anchor
4262 operand immP_M2()
4263 %{
4264 predicate(n->get_ptr() == -2);
4265 match(ConP);
4266
4267 op_cost(0);
4268 format %{ %}
4269 interface(CONST_INTER);
4270 %}
4271
4272 // Float and Double operands
4273 // Double Immediate
4274 operand immD()
4275 %{
4276 match(ConD);
4277 op_cost(0);
4278 format %{ %}
4279 interface(CONST_INTER);
4280 %}
4281
4282 // constant 'double +0.0'.
4283 operand immD0()
4284 %{
4285 predicate((n->getd() == 0) &&
4286 (fpclassify(n->getd()) == FP_ZERO) && (signbit(n->getd()) == 0));
4287 match(ConD);
4288 op_cost(0);
4289 format %{ %}
4290 interface(CONST_INTER);
4291 %}
4292
4293 // constant 'double +0.0'.
4294 operand immDPacked()
4295 %{
4296 predicate(Assembler::operand_valid_for_float_immediate(n->getd()));
4297 match(ConD);
4298 op_cost(0);
4299 format %{ %}
4300 interface(CONST_INTER);
4301 %}
4302
4303 // Float Immediate
4304 operand immF()
4305 %{
4306 match(ConF);
4307 op_cost(0);
4308 format %{ %}
4309 interface(CONST_INTER);
4310 %}
4311
4312 // constant 'float +0.0'.
4313 operand immF0()
4314 %{
4315 predicate((n->getf() == 0) &&
4316 (fpclassify(n->getf()) == FP_ZERO) && (signbit(n->getf()) == 0));
4317 match(ConF);
4318 op_cost(0);
4319 format %{ %}
4320 interface(CONST_INTER);
4321 %}
4322
4323 //
4324 operand immFPacked()
4325 %{
4326 predicate(Assembler::operand_valid_for_float_immediate((double)n->getf()));
4327 match(ConF);
4328 op_cost(0);
4329 format %{ %}
4330 interface(CONST_INTER);
4331 %}
4332
4333 // Narrow pointer operands
4334 // Narrow Pointer Immediate
4335 operand immN()
4336 %{
4337 match(ConN);
4338
4339 op_cost(0);
4340 format %{ %}
4341 interface(CONST_INTER);
4342 %}
4343
4344 // Narrow NULL Pointer Immediate
4345 operand immN0()
4346 %{
4347 predicate(n->get_narrowcon() == 0);
4348 match(ConN);
4349
4350 op_cost(0);
4351 format %{ %}
4352 interface(CONST_INTER);
4353 %}
4354
4355 operand immNKlass()
4356 %{
4357 match(ConNKlass);
4358
4359 op_cost(0);
4360 format %{ %}
4361 interface(CONST_INTER);
4362 %}
4363
4364 // Integer 32 bit Register Operands
4365 // Integer 32 bitRegister (excludes SP)
4366 operand iRegI()
4367 %{
4368 constraint(ALLOC_IN_RC(any_reg32));
4369 match(RegI);
4370 match(iRegINoSp);
4371 op_cost(0);
4372 format %{ %}
4373 interface(REG_INTER);
4374 %}
4375
4376 // Integer 32 bit Register not Special
4377 operand iRegINoSp()
4378 %{
4379 constraint(ALLOC_IN_RC(no_special_reg32));
4380 match(RegI);
4381 op_cost(0);
4382 format %{ %}
4383 interface(REG_INTER);
4384 %}
4385
4386 // Integer 64 bit Register Operands
4387 // Integer 64 bit Register (includes SP)
4388 operand iRegL()
4389 %{
4390 constraint(ALLOC_IN_RC(any_reg));
4391 match(RegL);
4392 match(iRegLNoSp);
4393 op_cost(0);
4394 format %{ %}
4395 interface(REG_INTER);
4396 %}
4397
4398 // Integer 64 bit Register not Special
4399 operand iRegLNoSp()
4400 %{
4401 constraint(ALLOC_IN_RC(no_special_reg));
4402 match(RegL);
4403 format %{ %}
4404 interface(REG_INTER);
4405 %}
4406
4407 // Pointer Register Operands
4408 // Pointer Register
4409 operand iRegP()
4410 %{
4411 constraint(ALLOC_IN_RC(ptr_reg));
4412 match(RegP);
4413 match(iRegPNoSp);
4414 match(iRegP_R0);
4415 //match(iRegP_R2);
4416 //match(iRegP_R4);
4417 //match(iRegP_R5);
4418 match(thread_RegP);
4419 op_cost(0);
4420 format %{ %}
4421 interface(REG_INTER);
4422 %}
4423
4424 // Pointer 64 bit Register not Special
4425 operand iRegPNoSp()
4426 %{
4427 constraint(ALLOC_IN_RC(no_special_ptr_reg));
4428 match(RegP);
4429 // match(iRegP);
4430 // match(iRegP_R0);
4431 // match(iRegP_R2);
4432 // match(iRegP_R4);
4433 // match(iRegP_R5);
4434 // match(thread_RegP);
4435 op_cost(0);
4436 format %{ %}
4437 interface(REG_INTER);
4438 %}
4439
4440 // Pointer 64 bit Register R0 only
4441 operand iRegP_R0()
4442 %{
4443 constraint(ALLOC_IN_RC(r0_reg));
4444 match(RegP);
4445 // match(iRegP);
4446 match(iRegPNoSp);
4447 op_cost(0);
4448 format %{ %}
4449 interface(REG_INTER);
4450 %}
4451
4452 // Pointer 64 bit Register R1 only
4453 operand iRegP_R1()
4454 %{
4455 constraint(ALLOC_IN_RC(r1_reg));
4456 match(RegP);
4457 // match(iRegP);
4458 match(iRegPNoSp);
4459 op_cost(0);
4460 format %{ %}
4461 interface(REG_INTER);
4462 %}
4463
4464 // Pointer 64 bit Register R2 only
4465 operand iRegP_R2()
4466 %{
4467 constraint(ALLOC_IN_RC(r2_reg));
4468 match(RegP);
4469 // match(iRegP);
4470 match(iRegPNoSp);
4471 op_cost(0);
4472 format %{ %}
4473 interface(REG_INTER);
4474 %}
4475
4476 // Pointer 64 bit Register R3 only
4477 operand iRegP_R3()
4478 %{
4479 constraint(ALLOC_IN_RC(r3_reg));
4480 match(RegP);
4481 // match(iRegP);
4482 match(iRegPNoSp);
4483 op_cost(0);
4484 format %{ %}
4485 interface(REG_INTER);
4486 %}
4487
4488 // Pointer 64 bit Register R4 only
4489 operand iRegP_R4()
4490 %{
4491 constraint(ALLOC_IN_RC(r4_reg));
4492 match(RegP);
4493 // match(iRegP);
4494 match(iRegPNoSp);
4495 op_cost(0);
4496 format %{ %}
4497 interface(REG_INTER);
4498 %}
4499
4500 // Pointer 64 bit Register R5 only
4501 operand iRegP_R5()
4502 %{
4503 constraint(ALLOC_IN_RC(r5_reg));
4504 match(RegP);
4505 // match(iRegP);
4506 match(iRegPNoSp);
4507 op_cost(0);
4508 format %{ %}
4509 interface(REG_INTER);
4510 %}
4511
4512 // Pointer 64 bit Register R10 only
4513 operand iRegP_R10()
4514 %{
4515 constraint(ALLOC_IN_RC(r10_reg));
4516 match(RegP);
4517 // match(iRegP);
4518 match(iRegPNoSp);
4519 op_cost(0);
4520 format %{ %}
4521 interface(REG_INTER);
4522 %}
4523
4524 // Long 64 bit Register R11 only
4525 operand iRegL_R11()
4526 %{
4527 constraint(ALLOC_IN_RC(r11_reg));
4528 match(RegL);
4529 match(iRegLNoSp);
4530 op_cost(0);
4531 format %{ %}
4532 interface(REG_INTER);
4533 %}
4534
4535 // Pointer 64 bit Register FP only
4536 operand iRegP_FP()
4537 %{
4538 constraint(ALLOC_IN_RC(fp_reg));
4539 match(RegP);
4540 // match(iRegP);
4541 op_cost(0);
4542 format %{ %}
4543 interface(REG_INTER);
4544 %}
4545
4546 // Register R0 only
4547 operand iRegI_R0()
4548 %{
4549 constraint(ALLOC_IN_RC(int_r0_reg));
4550 match(RegI);
4551 match(iRegINoSp);
4552 op_cost(0);
4553 format %{ %}
4554 interface(REG_INTER);
4555 %}
4556
4557 // Register R2 only
4558 operand iRegI_R2()
4559 %{
4560 constraint(ALLOC_IN_RC(int_r2_reg));
4561 match(RegI);
4562 match(iRegINoSp);
4563 op_cost(0);
4564 format %{ %}
4565 interface(REG_INTER);
4566 %}
4567
4568 // Register R3 only
4569 operand iRegI_R3()
4570 %{
4571 constraint(ALLOC_IN_RC(int_r3_reg));
4572 match(RegI);
4573 match(iRegINoSp);
4574 op_cost(0);
4575 format %{ %}
4576 interface(REG_INTER);
4577 %}
4578
4579
4580 // Register R2 only
4581 operand iRegI_R4()
4582 %{
4583 constraint(ALLOC_IN_RC(int_r4_reg));
4584 match(RegI);
4585 match(iRegINoSp);
4586 op_cost(0);
4587 format %{ %}
4588 interface(REG_INTER);
4589 %}
4590
4591
4592 // Pointer Register Operands
4593 // Narrow Pointer Register
4594 operand iRegN()
4595 %{
4596 constraint(ALLOC_IN_RC(any_reg32));
4597 match(RegN);
4598 match(iRegNNoSp);
4599 op_cost(0);
4600 format %{ %}
4601 interface(REG_INTER);
4602 %}
4603
4604 // Integer 64 bit Register not Special
4605 operand iRegNNoSp()
4606 %{
4607 constraint(ALLOC_IN_RC(no_special_reg32));
4608 match(RegN);
4609 op_cost(0);
4610 format %{ %}
4611 interface(REG_INTER);
4612 %}
4613
4614 // heap base register -- used for encoding immN0
4615
4616 operand iRegIHeapbase()
4617 %{
4618 constraint(ALLOC_IN_RC(heapbase_reg));
4619 match(RegI);
4620 op_cost(0);
4621 format %{ %}
4622 interface(REG_INTER);
4623 %}
4624
4625 // Float Register
4626 // Float register operands
4627 operand vRegF()
4628 %{
4629 constraint(ALLOC_IN_RC(float_reg));
4630 match(RegF);
4631
4632 op_cost(0);
4633 format %{ %}
4634 interface(REG_INTER);
4635 %}
4636
4637 // Double Register
4638 // Double register operands
4639 operand vRegD()
4640 %{
4641 constraint(ALLOC_IN_RC(double_reg));
4642 match(RegD);
4643
4644 op_cost(0);
4645 format %{ %}
4646 interface(REG_INTER);
4647 %}
4648
4649 operand vecD()
4650 %{
4651 constraint(ALLOC_IN_RC(vectord_reg));
4652 match(VecD);
4653
4654 op_cost(0);
4655 format %{ %}
4656 interface(REG_INTER);
4657 %}
4658
4659 operand vecX()
4660 %{
4661 constraint(ALLOC_IN_RC(vectorx_reg));
4662 match(VecX);
4663
4664 op_cost(0);
4665 format %{ %}
4666 interface(REG_INTER);
4667 %}
4668
4669 operand vRegD_V0()
4670 %{
4671 constraint(ALLOC_IN_RC(v0_reg));
4672 match(RegD);
4673 op_cost(0);
4674 format %{ %}
4675 interface(REG_INTER);
4676 %}
4677
4678 operand vRegD_V1()
4679 %{
4680 constraint(ALLOC_IN_RC(v1_reg));
4681 match(RegD);
4682 op_cost(0);
4683 format %{ %}
4684 interface(REG_INTER);
4685 %}
4686
4687 operand vRegD_V2()
4688 %{
4689 constraint(ALLOC_IN_RC(v2_reg));
4690 match(RegD);
4691 op_cost(0);
4692 format %{ %}
4693 interface(REG_INTER);
4694 %}
4695
4696 operand vRegD_V3()
4697 %{
4698 constraint(ALLOC_IN_RC(v3_reg));
4699 match(RegD);
4700 op_cost(0);
4701 format %{ %}
4702 interface(REG_INTER);
4703 %}
4704
4705 // Flags register, used as output of signed compare instructions
4706
4707 // note that on AArch64 we also use this register as the output for
4708 // for floating point compare instructions (CmpF CmpD). this ensures
4709 // that ordered inequality tests use GT, GE, LT or LE none of which
4710 // pass through cases where the result is unordered i.e. one or both
4711 // inputs to the compare is a NaN. this means that the ideal code can
4712 // replace e.g. a GT with an LE and not end up capturing the NaN case
4713 // (where the comparison should always fail). EQ and NE tests are
4714 // always generated in ideal code so that unordered folds into the NE
4715 // case, matching the behaviour of AArch64 NE.
4716 //
4717 // This differs from x86 where the outputs of FP compares use a
4718 // special FP flags registers and where compares based on this
4719 // register are distinguished into ordered inequalities (cmpOpUCF) and
4720 // EQ/NEQ tests (cmpOpUCF2). x86 has to special case the latter tests
4721 // to explicitly handle the unordered case in branches. x86 also has
4722 // to include extra CMoveX rules to accept a cmpOpUCF input.
4723
4724 operand rFlagsReg()
4725 %{
4726 constraint(ALLOC_IN_RC(int_flags));
4727 match(RegFlags);
4728
4729 op_cost(0);
4730 format %{ "RFLAGS" %}
4731 interface(REG_INTER);
4732 %}
4733
4734 // Flags register, used as output of unsigned compare instructions
4735 operand rFlagsRegU()
4736 %{
4737 constraint(ALLOC_IN_RC(int_flags));
4738 match(RegFlags);
4739
4740 op_cost(0);
4741 format %{ "RFLAGSU" %}
4742 interface(REG_INTER);
4743 %}
4744
4745 // Special Registers
4746
4747 // Method Register
4748 operand inline_cache_RegP(iRegP reg)
4749 %{
4750 constraint(ALLOC_IN_RC(method_reg)); // inline_cache_reg
4751 match(reg);
4752 match(iRegPNoSp);
4753 op_cost(0);
4754 format %{ %}
4755 interface(REG_INTER);
4756 %}
4757
4758 operand interpreter_method_oop_RegP(iRegP reg)
4759 %{
4760 constraint(ALLOC_IN_RC(method_reg)); // interpreter_method_oop_reg
4761 match(reg);
4762 match(iRegPNoSp);
4763 op_cost(0);
4764 format %{ %}
4765 interface(REG_INTER);
4766 %}
4767
4768 // Thread Register
4769 operand thread_RegP(iRegP reg)
4770 %{
4771 constraint(ALLOC_IN_RC(thread_reg)); // link_reg
4772 match(reg);
4773 op_cost(0);
4774 format %{ %}
4775 interface(REG_INTER);
4776 %}
4777
4778 operand lr_RegP(iRegP reg)
4779 %{
4780 constraint(ALLOC_IN_RC(lr_reg)); // link_reg
4781 match(reg);
4782 op_cost(0);
4783 format %{ %}
4784 interface(REG_INTER);
4785 %}
4786
4787 //----------Memory Operands----------------------------------------------------
4788
4789 operand indirect(iRegP reg)
4790 %{
4791 constraint(ALLOC_IN_RC(ptr_reg));
4792 match(reg);
4793 op_cost(0);
4794 format %{ "[$reg]" %}
4795 interface(MEMORY_INTER) %{
4796 base($reg);
4797 index(0xffffffff);
4798 scale(0x0);
4799 disp(0x0);
4800 %}
4801 %}
4802
4803 operand indIndexScaledOffsetI(iRegP reg, iRegL lreg, immIScale scale, immIU12 off)
4804 %{
4805 predicate(size_fits_all_mem_uses(n->as_AddP(),
4806 n->in(AddPNode::Address)->in(AddPNode::Offset)->in(2)->get_int()));
4807 constraint(ALLOC_IN_RC(ptr_reg));
4808 match(AddP (AddP reg (LShiftL lreg scale)) off);
4809 op_cost(INSN_COST);
4810 format %{ "$reg, $lreg lsl($scale), $off" %}
4811 interface(MEMORY_INTER) %{
4812 base($reg);
4813 index($lreg);
4814 scale($scale);
4815 disp($off);
4816 %}
4817 %}
4818
4819 operand indIndexScaledOffsetL(iRegP reg, iRegL lreg, immIScale scale, immLU12 off)
4820 %{
4821 predicate(size_fits_all_mem_uses(n->as_AddP(),
4822 n->in(AddPNode::Address)->in(AddPNode::Offset)->in(2)->get_int()));
4823 constraint(ALLOC_IN_RC(ptr_reg));
4824 match(AddP (AddP reg (LShiftL lreg scale)) off);
4825 op_cost(INSN_COST);
4826 format %{ "$reg, $lreg lsl($scale), $off" %}
4827 interface(MEMORY_INTER) %{
4828 base($reg);
4829 index($lreg);
4830 scale($scale);
4831 disp($off);
4832 %}
4833 %}
4834
4835 operand indIndexOffsetI2L(iRegP reg, iRegI ireg, immLU12 off)
4836 %{
4837 constraint(ALLOC_IN_RC(ptr_reg));
4838 match(AddP (AddP reg (ConvI2L ireg)) off);
4839 op_cost(INSN_COST);
4840 format %{ "$reg, $ireg, $off I2L" %}
4841 interface(MEMORY_INTER) %{
4842 base($reg);
4843 index($ireg);
4844 scale(0x0);
4845 disp($off);
4846 %}
4847 %}
4848
4849 operand indIndexScaledOffsetI2L(iRegP reg, iRegI ireg, immIScale scale, immLU12 off)
4850 %{
4851 predicate(size_fits_all_mem_uses(n->as_AddP(),
4852 n->in(AddPNode::Address)->in(AddPNode::Offset)->in(2)->get_int()));
4853 constraint(ALLOC_IN_RC(ptr_reg));
4854 match(AddP (AddP reg (LShiftL (ConvI2L ireg) scale)) off);
4855 op_cost(INSN_COST);
4856 format %{ "$reg, $ireg sxtw($scale), $off I2L" %}
4857 interface(MEMORY_INTER) %{
4858 base($reg);
4859 index($ireg);
4860 scale($scale);
4861 disp($off);
4862 %}
4863 %}
4864
4865 operand indIndexScaledI2L(iRegP reg, iRegI ireg, immIScale scale)
4866 %{
4867 predicate(size_fits_all_mem_uses(n->as_AddP(),
4868 n->in(AddPNode::Offset)->in(2)->get_int()));
4869 constraint(ALLOC_IN_RC(ptr_reg));
4870 match(AddP reg (LShiftL (ConvI2L ireg) scale));
4871 op_cost(0);
4872 format %{ "$reg, $ireg sxtw($scale), 0, I2L" %}
4873 interface(MEMORY_INTER) %{
4874 base($reg);
4875 index($ireg);
4876 scale($scale);
4877 disp(0x0);
4878 %}
4879 %}
4880
4881 operand indIndexScaled(iRegP reg, iRegL lreg, immIScale scale)
4882 %{
4883 predicate(size_fits_all_mem_uses(n->as_AddP(),
4884 n->in(AddPNode::Offset)->in(2)->get_int()));
4885 constraint(ALLOC_IN_RC(ptr_reg));
4886 match(AddP reg (LShiftL lreg scale));
4887 op_cost(0);
4888 format %{ "$reg, $lreg lsl($scale)" %}
4889 interface(MEMORY_INTER) %{
4890 base($reg);
4891 index($lreg);
4892 scale($scale);
4893 disp(0x0);
4894 %}
4895 %}
4896
4897 operand indIndex(iRegP reg, iRegL lreg)
4898 %{
4899 constraint(ALLOC_IN_RC(ptr_reg));
4900 match(AddP reg lreg);
4901 op_cost(0);
4902 format %{ "$reg, $lreg" %}
4903 interface(MEMORY_INTER) %{
4904 base($reg);
4905 index($lreg);
4906 scale(0x0);
4907 disp(0x0);
4908 %}
4909 %}
4910
4911 operand indOffI(iRegP reg, immIOffset off)
4912 %{
4913 constraint(ALLOC_IN_RC(ptr_reg));
4914 match(AddP reg off);
4915 op_cost(0);
4916 format %{ "[$reg, $off]" %}
4917 interface(MEMORY_INTER) %{
4918 base($reg);
4919 index(0xffffffff);
4920 scale(0x0);
4921 disp($off);
4922 %}
4923 %}
4924
4925 operand indOffI4(iRegP reg, immIOffset4 off)
4926 %{
4927 constraint(ALLOC_IN_RC(ptr_reg));
4928 match(AddP reg off);
4929 op_cost(0);
4930 format %{ "[$reg, $off]" %}
4931 interface(MEMORY_INTER) %{
4932 base($reg);
4933 index(0xffffffff);
4934 scale(0x0);
4935 disp($off);
4936 %}
4937 %}
4938
4939 operand indOffI8(iRegP reg, immIOffset8 off)
4940 %{
4941 constraint(ALLOC_IN_RC(ptr_reg));
4942 match(AddP reg off);
4943 op_cost(0);
4944 format %{ "[$reg, $off]" %}
4945 interface(MEMORY_INTER) %{
4946 base($reg);
4947 index(0xffffffff);
4948 scale(0x0);
4949 disp($off);
4950 %}
4951 %}
4952
4953 operand indOffI16(iRegP reg, immIOffset16 off)
4954 %{
4955 constraint(ALLOC_IN_RC(ptr_reg));
4956 match(AddP reg off);
4957 op_cost(0);
4958 format %{ "[$reg, $off]" %}
4959 interface(MEMORY_INTER) %{
4960 base($reg);
4961 index(0xffffffff);
4962 scale(0x0);
4963 disp($off);
4964 %}
4965 %}
4966
4967 operand indOffL(iRegP reg, immLoffset off)
4968 %{
4969 constraint(ALLOC_IN_RC(ptr_reg));
4970 match(AddP reg off);
4971 op_cost(0);
4972 format %{ "[$reg, $off]" %}
4973 interface(MEMORY_INTER) %{
4974 base($reg);
4975 index(0xffffffff);
4976 scale(0x0);
4977 disp($off);
4978 %}
4979 %}
4980
4981 operand indOffL4(iRegP reg, immLoffset4 off)
4982 %{
4983 constraint(ALLOC_IN_RC(ptr_reg));
4984 match(AddP reg off);
4985 op_cost(0);
4986 format %{ "[$reg, $off]" %}
4987 interface(MEMORY_INTER) %{
4988 base($reg);
4989 index(0xffffffff);
4990 scale(0x0);
4991 disp($off);
4992 %}
4993 %}
4994
4995 operand indOffL8(iRegP reg, immLoffset8 off)
4996 %{
4997 constraint(ALLOC_IN_RC(ptr_reg));
4998 match(AddP reg off);
4999 op_cost(0);
5000 format %{ "[$reg, $off]" %}
5001 interface(MEMORY_INTER) %{
5002 base($reg);
5003 index(0xffffffff);
5004 scale(0x0);
5005 disp($off);
5006 %}
5007 %}
5008
5009 operand indOffL16(iRegP reg, immLoffset16 off)
5010 %{
5011 constraint(ALLOC_IN_RC(ptr_reg));
5012 match(AddP reg off);
5013 op_cost(0);
5014 format %{ "[$reg, $off]" %}
5015 interface(MEMORY_INTER) %{
5016 base($reg);
5017 index(0xffffffff);
5018 scale(0x0);
5019 disp($off);
5020 %}
5021 %}
5022
5023 operand indirectN(iRegN reg)
5024 %{
5025 predicate(Universe::narrow_oop_shift() == 0);
5026 constraint(ALLOC_IN_RC(ptr_reg));
5027 match(DecodeN reg);
5028 op_cost(0);
5029 format %{ "[$reg]\t# narrow" %}
5030 interface(MEMORY_INTER) %{
5031 base($reg);
5032 index(0xffffffff);
5033 scale(0x0);
5034 disp(0x0);
5035 %}
5036 %}
5037
5038 operand indIndexScaledOffsetIN(iRegN reg, iRegL lreg, immIScale scale, immIU12 off)
5039 %{
5040 predicate(Universe::narrow_oop_shift() == 0 &&
5041 size_fits_all_mem_uses(n->as_AddP(),
5042 n->in(AddPNode::Address)->in(AddPNode::Offset)->in(2)->get_int()));
5043 constraint(ALLOC_IN_RC(ptr_reg));
5044 match(AddP (AddP (DecodeN reg) (LShiftL lreg scale)) off);
5045 op_cost(0);
5046 format %{ "$reg, $lreg lsl($scale), $off\t# narrow" %}
5047 interface(MEMORY_INTER) %{
5048 base($reg);
5049 index($lreg);
5050 scale($scale);
5051 disp($off);
5052 %}
5053 %}
5054
5055 operand indIndexScaledOffsetLN(iRegN reg, iRegL lreg, immIScale scale, immLU12 off)
5056 %{
5057 predicate(Universe::narrow_oop_shift() == 0 &&
5058 size_fits_all_mem_uses(n->as_AddP(),
5059 n->in(AddPNode::Address)->in(AddPNode::Offset)->in(2)->get_int()));
5060 constraint(ALLOC_IN_RC(ptr_reg));
5061 match(AddP (AddP (DecodeN reg) (LShiftL lreg scale)) off);
5062 op_cost(INSN_COST);
5063 format %{ "$reg, $lreg lsl($scale), $off\t# narrow" %}
5064 interface(MEMORY_INTER) %{
5065 base($reg);
5066 index($lreg);
5067 scale($scale);
5068 disp($off);
5069 %}
5070 %}
5071
5072 operand indIndexOffsetI2LN(iRegN reg, iRegI ireg, immLU12 off)
5073 %{
5074 predicate(Universe::narrow_oop_shift() == 0);
5075 constraint(ALLOC_IN_RC(ptr_reg));
5076 match(AddP (AddP (DecodeN reg) (ConvI2L ireg)) off);
5077 op_cost(INSN_COST);
5078 format %{ "$reg, $ireg, $off I2L\t# narrow" %}
5079 interface(MEMORY_INTER) %{
5080 base($reg);
5081 index($ireg);
5082 scale(0x0);
5083 disp($off);
5084 %}
5085 %}
5086
5087 operand indIndexScaledOffsetI2LN(iRegN reg, iRegI ireg, immIScale scale, immLU12 off)
5088 %{
5089 predicate(Universe::narrow_oop_shift() == 0 &&
5090 size_fits_all_mem_uses(n->as_AddP(),
5091 n->in(AddPNode::Address)->in(AddPNode::Offset)->in(2)->get_int()));
5092 constraint(ALLOC_IN_RC(ptr_reg));
5093 match(AddP (AddP (DecodeN reg) (LShiftL (ConvI2L ireg) scale)) off);
5094 op_cost(INSN_COST);
5095 format %{ "$reg, $ireg sxtw($scale), $off I2L\t# narrow" %}
5096 interface(MEMORY_INTER) %{
5097 base($reg);
5098 index($ireg);
5099 scale($scale);
5100 disp($off);
5101 %}
5102 %}
5103
5104 operand indIndexScaledI2LN(iRegN reg, iRegI ireg, immIScale scale)
5105 %{
5106 predicate(Universe::narrow_oop_shift() == 0 &&
5107 size_fits_all_mem_uses(n->as_AddP(),
5108 n->in(AddPNode::Offset)->in(2)->get_int()));
5109 constraint(ALLOC_IN_RC(ptr_reg));
5110 match(AddP (DecodeN reg) (LShiftL (ConvI2L ireg) scale));
5111 op_cost(0);
5112 format %{ "$reg, $ireg sxtw($scale), 0, I2L\t# narrow" %}
5113 interface(MEMORY_INTER) %{
5114 base($reg);
5115 index($ireg);
5116 scale($scale);
5117 disp(0x0);
5118 %}
5119 %}
5120
5121 operand indIndexScaledN(iRegN reg, iRegL lreg, immIScale scale)
5122 %{
5123 predicate(Universe::narrow_oop_shift() == 0 &&
5124 size_fits_all_mem_uses(n->as_AddP(),
5125 n->in(AddPNode::Offset)->in(2)->get_int()));
5126 constraint(ALLOC_IN_RC(ptr_reg));
5127 match(AddP (DecodeN reg) (LShiftL lreg scale));
5128 op_cost(0);
5129 format %{ "$reg, $lreg lsl($scale)\t# narrow" %}
5130 interface(MEMORY_INTER) %{
5131 base($reg);
5132 index($lreg);
5133 scale($scale);
5134 disp(0x0);
5135 %}
5136 %}
5137
5138 operand indIndexN(iRegN reg, iRegL lreg)
5139 %{
5140 predicate(Universe::narrow_oop_shift() == 0);
5141 constraint(ALLOC_IN_RC(ptr_reg));
5142 match(AddP (DecodeN reg) lreg);
5143 op_cost(0);
5144 format %{ "$reg, $lreg\t# narrow" %}
5145 interface(MEMORY_INTER) %{
5146 base($reg);
5147 index($lreg);
5148 scale(0x0);
5149 disp(0x0);
5150 %}
5151 %}
5152
5153 operand indOffIN(iRegN reg, immIOffset off)
5154 %{
5155 predicate(Universe::narrow_oop_shift() == 0);
5156 constraint(ALLOC_IN_RC(ptr_reg));
5157 match(AddP (DecodeN reg) off);
5158 op_cost(0);
5159 format %{ "[$reg, $off]\t# narrow" %}
5160 interface(MEMORY_INTER) %{
5161 base($reg);
5162 index(0xffffffff);
5163 scale(0x0);
5164 disp($off);
5165 %}
5166 %}
5167
5168 operand indOffLN(iRegN reg, immLoffset off)
5169 %{
5170 predicate(Universe::narrow_oop_shift() == 0);
5171 constraint(ALLOC_IN_RC(ptr_reg));
5172 match(AddP (DecodeN reg) off);
5173 op_cost(0);
5174 format %{ "[$reg, $off]\t# narrow" %}
5175 interface(MEMORY_INTER) %{
5176 base($reg);
5177 index(0xffffffff);
5178 scale(0x0);
5179 disp($off);
5180 %}
5181 %}
5182
5183
5184
5185 // AArch64 opto stubs need to write to the pc slot in the thread anchor
5186 operand thread_anchor_pc(thread_RegP reg, immL_pc_off off)
5187 %{
5188 constraint(ALLOC_IN_RC(ptr_reg));
5189 match(AddP reg off);
5190 op_cost(0);
5191 format %{ "[$reg, $off]" %}
5192 interface(MEMORY_INTER) %{
5193 base($reg);
5194 index(0xffffffff);
5195 scale(0x0);
5196 disp($off);
5197 %}
5198 %}
5199
5200 //----------Special Memory Operands--------------------------------------------
5201 // Stack Slot Operand - This operand is used for loading and storing temporary
5202 // values on the stack where a match requires a value to
5203 // flow through memory.
5204 operand stackSlotP(sRegP reg)
5205 %{
5206 constraint(ALLOC_IN_RC(stack_slots));
5207 op_cost(100);
5208 // No match rule because this operand is only generated in matching
5209 // match(RegP);
5210 format %{ "[$reg]" %}
5211 interface(MEMORY_INTER) %{
5212 base(0x1e); // RSP
5213 index(0x0); // No Index
5214 scale(0x0); // No Scale
5215 disp($reg); // Stack Offset
5216 %}
5217 %}
5218
5219 operand stackSlotI(sRegI reg)
5220 %{
5221 constraint(ALLOC_IN_RC(stack_slots));
5222 // No match rule because this operand is only generated in matching
5223 // match(RegI);
5224 format %{ "[$reg]" %}
5225 interface(MEMORY_INTER) %{
5226 base(0x1e); // RSP
5227 index(0x0); // No Index
5228 scale(0x0); // No Scale
5229 disp($reg); // Stack Offset
5230 %}
5231 %}
5232
5233 operand stackSlotF(sRegF reg)
5234 %{
5235 constraint(ALLOC_IN_RC(stack_slots));
5236 // No match rule because this operand is only generated in matching
5237 // match(RegF);
5238 format %{ "[$reg]" %}
5239 interface(MEMORY_INTER) %{
5240 base(0x1e); // RSP
5241 index(0x0); // No Index
5242 scale(0x0); // No Scale
5243 disp($reg); // Stack Offset
5244 %}
5245 %}
5246
5247 operand stackSlotD(sRegD reg)
5248 %{
5249 constraint(ALLOC_IN_RC(stack_slots));
5250 // No match rule because this operand is only generated in matching
5251 // match(RegD);
5252 format %{ "[$reg]" %}
5253 interface(MEMORY_INTER) %{
5254 base(0x1e); // RSP
5255 index(0x0); // No Index
5256 scale(0x0); // No Scale
5257 disp($reg); // Stack Offset
5258 %}
5259 %}
5260
5261 operand stackSlotL(sRegL reg)
5262 %{
5263 constraint(ALLOC_IN_RC(stack_slots));
5264 // No match rule because this operand is only generated in matching
5265 // match(RegL);
5266 format %{ "[$reg]" %}
5267 interface(MEMORY_INTER) %{
5268 base(0x1e); // RSP
5269 index(0x0); // No Index
5270 scale(0x0); // No Scale
5271 disp($reg); // Stack Offset
5272 %}
5273 %}
5274
5275 // Operands for expressing Control Flow
5276 // NOTE: Label is a predefined operand which should not be redefined in
5277 // the AD file. It is generically handled within the ADLC.
5278
5279 //----------Conditional Branch Operands----------------------------------------
5280 // Comparison Op - This is the operation of the comparison, and is limited to
5281 // the following set of codes:
5282 // L (<), LE (<=), G (>), GE (>=), E (==), NE (!=)
5283 //
5284 // Other attributes of the comparison, such as unsignedness, are specified
5285 // by the comparison instruction that sets a condition code flags register.
5286 // That result is represented by a flags operand whose subtype is appropriate
5287 // to the unsignedness (etc.) of the comparison.
5288 //
5289 // Later, the instruction which matches both the Comparison Op (a Bool) and
5290 // the flags (produced by the Cmp) specifies the coding of the comparison op
5291 // by matching a specific subtype of Bool operand below, such as cmpOpU.
5292
5293 // used for signed integral comparisons and fp comparisons
5294
5295 operand cmpOp()
5296 %{
5297 match(Bool);
5298
5299 format %{ "" %}
5300 interface(COND_INTER) %{
5301 equal(0x0, "eq");
5302 not_equal(0x1, "ne");
5303 less(0xb, "lt");
5304 greater_equal(0xa, "ge");
5305 less_equal(0xd, "le");
5306 greater(0xc, "gt");
5307 overflow(0x6, "vs");
5308 no_overflow(0x7, "vc");
5309 %}
5310 %}
5311
5312 // used for unsigned integral comparisons
5313
5314 operand cmpOpU()
5315 %{
5316 match(Bool);
5317
5318 format %{ "" %}
5319 interface(COND_INTER) %{
5320 equal(0x0, "eq");
5321 not_equal(0x1, "ne");
5322 less(0x3, "lo");
5323 greater_equal(0x2, "hs");
5324 less_equal(0x9, "ls");
5325 greater(0x8, "hi");
5326 overflow(0x6, "vs");
5327 no_overflow(0x7, "vc");
5328 %}
5329 %}
5330
5331 // Special operand allowing long args to int ops to be truncated for free
5332
5333 operand iRegL2I(iRegL reg) %{
5334
5335 op_cost(0);
5336
5337 match(ConvL2I reg);
5338
5339 format %{ "l2i($reg)" %}
5340
5341 interface(REG_INTER)
5342 %}
5343
5344 opclass vmem4(indirect, indIndex, indOffI4, indOffL4);
5345 opclass vmem8(indirect, indIndex, indOffI8, indOffL8);
5346 opclass vmem16(indirect, indIndex, indOffI16, indOffL16);
5347
5348 //----------OPERAND CLASSES----------------------------------------------------
5349 // Operand Classes are groups of operands that are used as to simplify
5350 // instruction definitions by not requiring the AD writer to specify
5351 // separate instructions for every form of operand when the
5352 // instruction accepts multiple operand types with the same basic
5353 // encoding and format. The classic case of this is memory operands.
5354
5355 // memory is used to define read/write location for load/store
5356 // instruction defs. we can turn a memory op into an Address
5357
5358 opclass memory(indirect, indIndexScaledOffsetI, indIndexScaledOffsetL, indIndexOffsetI2L, indIndexScaledOffsetI2L, indIndexScaled, indIndexScaledI2L, indIndex, indOffI, indOffL,
5359 indirectN, indIndexScaledOffsetIN, indIndexScaledOffsetLN, indIndexOffsetI2LN, indIndexScaledOffsetI2LN, indIndexScaledN, indIndexScaledI2LN, indIndexN, indOffIN, indOffLN);
5360
5361 // iRegIorL2I is used for src inputs in rules for 32 bit int (I)
5362
5363
5364 // iRegIorL2I is used for src inputs in rules for 32 bit int (I)
5365 // operations. it allows the src to be either an iRegI or a (ConvL2I
5366 // iRegL). in the latter case the l2i normally planted for a ConvL2I
5367 // can be elided because the 32-bit instruction will just employ the
5368 // lower 32 bits anyway.
5369 //
5370 // n.b. this does not elide all L2I conversions. if the truncated
5371 // value is consumed by more than one operation then the ConvL2I
5372 // cannot be bundled into the consuming nodes so an l2i gets planted
5373 // (actually a movw $dst $src) and the downstream instructions consume
5374 // the result of the l2i as an iRegI input. That's a shame since the
5375 // movw is actually redundant but its not too costly.
5376
5377 opclass iRegIorL2I(iRegI, iRegL2I);
5378
5379 //----------PIPELINE-----------------------------------------------------------
5380 // Rules which define the behavior of the target architectures pipeline.
5381
5382 // For specific pipelines, eg A53, define the stages of that pipeline
5383 //pipe_desc(ISS, EX1, EX2, WR);
5384 #define ISS S0
5385 #define EX1 S1
5386 #define EX2 S2
5387 #define WR S3
5388
5389 // Integer ALU reg operation
5390 pipeline %{
5391
5392 attributes %{
5393 // ARM instructions are of fixed length
5394 fixed_size_instructions; // Fixed size instructions TODO does
5395 max_instructions_per_bundle = 2; // A53 = 2, A57 = 4
5396 // ARM instructions come in 32-bit word units
5397 instruction_unit_size = 4; // An instruction is 4 bytes long
5398 instruction_fetch_unit_size = 64; // The processor fetches one line
5399 instruction_fetch_units = 1; // of 64 bytes
5400
5401 // List of nop instructions
5402 nops( MachNop );
5403 %}
5404
5405 // We don't use an actual pipeline model so don't care about resources
5406 // or description. we do use pipeline classes to introduce fixed
5407 // latencies
5408
5409 //----------RESOURCES----------------------------------------------------------
5410 // Resources are the functional units available to the machine
5411
5412 resources( INS0, INS1, INS01 = INS0 | INS1,
5413 ALU0, ALU1, ALU = ALU0 | ALU1,
5414 MAC,
5415 DIV,
5416 BRANCH,
5417 LDST,
5418 NEON_FP);
5419
5420 //----------PIPELINE DESCRIPTION-----------------------------------------------
5421 // Pipeline Description specifies the stages in the machine's pipeline
5422
5423 // Define the pipeline as a generic 6 stage pipeline
5424 pipe_desc(S0, S1, S2, S3, S4, S5);
5425
5426 //----------PIPELINE CLASSES---------------------------------------------------
5427 // Pipeline Classes describe the stages in which input and output are
5428 // referenced by the hardware pipeline.
5429
5430 pipe_class fp_dop_reg_reg_s(vRegF dst, vRegF src1, vRegF src2)
5431 %{
5432 single_instruction;
5433 src1 : S1(read);
5434 src2 : S2(read);
5435 dst : S5(write);
5436 INS01 : ISS;
5437 NEON_FP : S5;
5438 %}
5439
5440 pipe_class fp_dop_reg_reg_d(vRegD dst, vRegD src1, vRegD src2)
5441 %{
5442 single_instruction;
5443 src1 : S1(read);
5444 src2 : S2(read);
5445 dst : S5(write);
5446 INS01 : ISS;
5447 NEON_FP : S5;
5448 %}
5449
5450 pipe_class fp_uop_s(vRegF dst, vRegF src)
5451 %{
5452 single_instruction;
5453 src : S1(read);
5454 dst : S5(write);
5455 INS01 : ISS;
5456 NEON_FP : S5;
5457 %}
5458
5459 pipe_class fp_uop_d(vRegD dst, vRegD src)
5460 %{
5461 single_instruction;
5462 src : S1(read);
5463 dst : S5(write);
5464 INS01 : ISS;
5465 NEON_FP : S5;
5466 %}
5467
5468 pipe_class fp_d2f(vRegF dst, vRegD src)
5469 %{
5470 single_instruction;
5471 src : S1(read);
5472 dst : S5(write);
5473 INS01 : ISS;
5474 NEON_FP : S5;
5475 %}
5476
5477 pipe_class fp_f2d(vRegD dst, vRegF src)
5478 %{
5479 single_instruction;
5480 src : S1(read);
5481 dst : S5(write);
5482 INS01 : ISS;
5483 NEON_FP : S5;
5484 %}
5485
5486 pipe_class fp_f2i(iRegINoSp dst, vRegF src)
5487 %{
5488 single_instruction;
5489 src : S1(read);
5490 dst : S5(write);
5491 INS01 : ISS;
5492 NEON_FP : S5;
5493 %}
5494
5495 pipe_class fp_f2l(iRegLNoSp dst, vRegF src)
5496 %{
5497 single_instruction;
5498 src : S1(read);
5499 dst : S5(write);
5500 INS01 : ISS;
5501 NEON_FP : S5;
5502 %}
5503
5504 pipe_class fp_i2f(vRegF dst, iRegIorL2I src)
5505 %{
5506 single_instruction;
5507 src : S1(read);
5508 dst : S5(write);
5509 INS01 : ISS;
5510 NEON_FP : S5;
5511 %}
5512
5513 pipe_class fp_l2f(vRegF dst, iRegL src)
5514 %{
5515 single_instruction;
5516 src : S1(read);
5517 dst : S5(write);
5518 INS01 : ISS;
5519 NEON_FP : S5;
5520 %}
5521
5522 pipe_class fp_d2i(iRegINoSp dst, vRegD src)
5523 %{
5524 single_instruction;
5525 src : S1(read);
5526 dst : S5(write);
5527 INS01 : ISS;
5528 NEON_FP : S5;
5529 %}
5530
5531 pipe_class fp_d2l(iRegLNoSp dst, vRegD src)
5532 %{
5533 single_instruction;
5534 src : S1(read);
5535 dst : S5(write);
5536 INS01 : ISS;
5537 NEON_FP : S5;
5538 %}
5539
5540 pipe_class fp_i2d(vRegD dst, iRegIorL2I src)
5541 %{
5542 single_instruction;
5543 src : S1(read);
5544 dst : S5(write);
5545 INS01 : ISS;
5546 NEON_FP : S5;
5547 %}
5548
5549 pipe_class fp_l2d(vRegD dst, iRegIorL2I src)
5550 %{
5551 single_instruction;
5552 src : S1(read);
5553 dst : S5(write);
5554 INS01 : ISS;
5555 NEON_FP : S5;
5556 %}
5557
5558 pipe_class fp_div_s(vRegF dst, vRegF src1, vRegF src2)
5559 %{
5560 single_instruction;
5561 src1 : S1(read);
5562 src2 : S2(read);
5563 dst : S5(write);
5564 INS0 : ISS;
5565 NEON_FP : S5;
5566 %}
5567
5568 pipe_class fp_div_d(vRegD dst, vRegD src1, vRegD src2)
5569 %{
5570 single_instruction;
5571 src1 : S1(read);
5572 src2 : S2(read);
5573 dst : S5(write);
5574 INS0 : ISS;
5575 NEON_FP : S5;
5576 %}
5577
5578 pipe_class fp_cond_reg_reg_s(vRegF dst, vRegF src1, vRegF src2, rFlagsReg cr)
5579 %{
5580 single_instruction;
5581 cr : S1(read);
5582 src1 : S1(read);
5583 src2 : S1(read);
5584 dst : S3(write);
5585 INS01 : ISS;
5586 NEON_FP : S3;
5587 %}
5588
5589 pipe_class fp_cond_reg_reg_d(vRegD dst, vRegD src1, vRegD src2, rFlagsReg cr)
5590 %{
5591 single_instruction;
5592 cr : S1(read);
5593 src1 : S1(read);
5594 src2 : S1(read);
5595 dst : S3(write);
5596 INS01 : ISS;
5597 NEON_FP : S3;
5598 %}
5599
5600 pipe_class fp_imm_s(vRegF dst)
5601 %{
5602 single_instruction;
5603 dst : S3(write);
5604 INS01 : ISS;
5605 NEON_FP : S3;
5606 %}
5607
5608 pipe_class fp_imm_d(vRegD dst)
5609 %{
5610 single_instruction;
5611 dst : S3(write);
5612 INS01 : ISS;
5613 NEON_FP : S3;
5614 %}
5615
5616 pipe_class fp_load_constant_s(vRegF dst)
5617 %{
5618 single_instruction;
5619 dst : S4(write);
5620 INS01 : ISS;
5621 NEON_FP : S4;
5622 %}
5623
5624 pipe_class fp_load_constant_d(vRegD dst)
5625 %{
5626 single_instruction;
5627 dst : S4(write);
5628 INS01 : ISS;
5629 NEON_FP : S4;
5630 %}
5631
5632 pipe_class vmul64(vecD dst, vecD src1, vecD src2)
5633 %{
5634 single_instruction;
5635 dst : S5(write);
5636 src1 : S1(read);
5637 src2 : S1(read);
5638 INS01 : ISS;
5639 NEON_FP : S5;
5640 %}
5641
5642 pipe_class vmul128(vecX dst, vecX src1, vecX src2)
5643 %{
5644 single_instruction;
5645 dst : S5(write);
5646 src1 : S1(read);
5647 src2 : S1(read);
5648 INS0 : ISS;
5649 NEON_FP : S5;
5650 %}
5651
5652 pipe_class vmla64(vecD dst, vecD src1, vecD src2)
5653 %{
5654 single_instruction;
5655 dst : S5(write);
5656 src1 : S1(read);
5657 src2 : S1(read);
5658 dst : S1(read);
5659 INS01 : ISS;
5660 NEON_FP : S5;
5661 %}
5662
5663 pipe_class vmla128(vecX dst, vecX src1, vecX src2)
5664 %{
5665 single_instruction;
5666 dst : S5(write);
5667 src1 : S1(read);
5668 src2 : S1(read);
5669 dst : S1(read);
5670 INS0 : ISS;
5671 NEON_FP : S5;
5672 %}
5673
5674 pipe_class vdop64(vecD dst, vecD src1, vecD src2)
5675 %{
5676 single_instruction;
5677 dst : S4(write);
5678 src1 : S2(read);
5679 src2 : S2(read);
5680 INS01 : ISS;
5681 NEON_FP : S4;
5682 %}
5683
5684 pipe_class vdop128(vecX dst, vecX src1, vecX src2)
5685 %{
5686 single_instruction;
5687 dst : S4(write);
5688 src1 : S2(read);
5689 src2 : S2(read);
5690 INS0 : ISS;
5691 NEON_FP : S4;
5692 %}
5693
5694 pipe_class vlogical64(vecD dst, vecD src1, vecD src2)
5695 %{
5696 single_instruction;
5697 dst : S3(write);
5698 src1 : S2(read);
5699 src2 : S2(read);
5700 INS01 : ISS;
5701 NEON_FP : S3;
5702 %}
5703
5704 pipe_class vlogical128(vecX dst, vecX src1, vecX src2)
5705 %{
5706 single_instruction;
5707 dst : S3(write);
5708 src1 : S2(read);
5709 src2 : S2(read);
5710 INS0 : ISS;
5711 NEON_FP : S3;
5712 %}
5713
5714 pipe_class vshift64(vecD dst, vecD src, vecX shift)
5715 %{
5716 single_instruction;
5717 dst : S3(write);
5718 src : S1(read);
5719 shift : S1(read);
5720 INS01 : ISS;
5721 NEON_FP : S3;
5722 %}
5723
5724 pipe_class vshift128(vecX dst, vecX src, vecX shift)
5725 %{
5726 single_instruction;
5727 dst : S3(write);
5728 src : S1(read);
5729 shift : S1(read);
5730 INS0 : ISS;
5731 NEON_FP : S3;
5732 %}
5733
5734 pipe_class vshift64_imm(vecD dst, vecD src, immI shift)
5735 %{
5736 single_instruction;
5737 dst : S3(write);
5738 src : S1(read);
5739 INS01 : ISS;
5740 NEON_FP : S3;
5741 %}
5742
5743 pipe_class vshift128_imm(vecX dst, vecX src, immI shift)
5744 %{
5745 single_instruction;
5746 dst : S3(write);
5747 src : S1(read);
5748 INS0 : ISS;
5749 NEON_FP : S3;
5750 %}
5751
5752 pipe_class vdop_fp64(vecD dst, vecD src1, vecD src2)
5753 %{
5754 single_instruction;
5755 dst : S5(write);
5756 src1 : S1(read);
5757 src2 : S1(read);
5758 INS01 : ISS;
5759 NEON_FP : S5;
5760 %}
5761
5762 pipe_class vdop_fp128(vecX dst, vecX src1, vecX src2)
5763 %{
5764 single_instruction;
5765 dst : S5(write);
5766 src1 : S1(read);
5767 src2 : S1(read);
5768 INS0 : ISS;
5769 NEON_FP : S5;
5770 %}
5771
5772 pipe_class vmuldiv_fp64(vecD dst, vecD src1, vecD src2)
5773 %{
5774 single_instruction;
5775 dst : S5(write);
5776 src1 : S1(read);
5777 src2 : S1(read);
5778 INS0 : ISS;
5779 NEON_FP : S5;
5780 %}
5781
5782 pipe_class vmuldiv_fp128(vecX dst, vecX src1, vecX src2)
5783 %{
5784 single_instruction;
5785 dst : S5(write);
5786 src1 : S1(read);
5787 src2 : S1(read);
5788 INS0 : ISS;
5789 NEON_FP : S5;
5790 %}
5791
5792 pipe_class vsqrt_fp128(vecX dst, vecX src)
5793 %{
5794 single_instruction;
5795 dst : S5(write);
5796 src : S1(read);
5797 INS0 : ISS;
5798 NEON_FP : S5;
5799 %}
5800
5801 pipe_class vunop_fp64(vecD dst, vecD src)
5802 %{
5803 single_instruction;
5804 dst : S5(write);
5805 src : S1(read);
5806 INS01 : ISS;
5807 NEON_FP : S5;
5808 %}
5809
5810 pipe_class vunop_fp128(vecX dst, vecX src)
5811 %{
5812 single_instruction;
5813 dst : S5(write);
5814 src : S1(read);
5815 INS0 : ISS;
5816 NEON_FP : S5;
5817 %}
5818
5819 pipe_class vdup_reg_reg64(vecD dst, iRegI src)
5820 %{
5821 single_instruction;
5822 dst : S3(write);
5823 src : S1(read);
5824 INS01 : ISS;
5825 NEON_FP : S3;
5826 %}
5827
5828 pipe_class vdup_reg_reg128(vecX dst, iRegI src)
5829 %{
5830 single_instruction;
5831 dst : S3(write);
5832 src : S1(read);
5833 INS01 : ISS;
5834 NEON_FP : S3;
5835 %}
5836
5837 pipe_class vdup_reg_freg64(vecD dst, vRegF src)
5838 %{
5839 single_instruction;
5840 dst : S3(write);
5841 src : S1(read);
5842 INS01 : ISS;
5843 NEON_FP : S3;
5844 %}
5845
5846 pipe_class vdup_reg_freg128(vecX dst, vRegF src)
5847 %{
5848 single_instruction;
5849 dst : S3(write);
5850 src : S1(read);
5851 INS01 : ISS;
5852 NEON_FP : S3;
5853 %}
5854
5855 pipe_class vdup_reg_dreg128(vecX dst, vRegD src)
5856 %{
5857 single_instruction;
5858 dst : S3(write);
5859 src : S1(read);
5860 INS01 : ISS;
5861 NEON_FP : S3;
5862 %}
5863
5864 pipe_class vmovi_reg_imm64(vecD dst)
5865 %{
5866 single_instruction;
5867 dst : S3(write);
5868 INS01 : ISS;
5869 NEON_FP : S3;
5870 %}
5871
5872 pipe_class vmovi_reg_imm128(vecX dst)
5873 %{
5874 single_instruction;
5875 dst : S3(write);
5876 INS0 : ISS;
5877 NEON_FP : S3;
5878 %}
5879
5880 pipe_class vload_reg_mem64(vecD dst, vmem8 mem)
5881 %{
5882 single_instruction;
5883 dst : S5(write);
5884 mem : ISS(read);
5885 INS01 : ISS;
5886 NEON_FP : S3;
5887 %}
5888
5889 pipe_class vload_reg_mem128(vecX dst, vmem16 mem)
5890 %{
5891 single_instruction;
5892 dst : S5(write);
5893 mem : ISS(read);
5894 INS01 : ISS;
5895 NEON_FP : S3;
5896 %}
5897
5898 pipe_class vstore_reg_mem64(vecD src, vmem8 mem)
5899 %{
5900 single_instruction;
5901 mem : ISS(read);
5902 src : S2(read);
5903 INS01 : ISS;
5904 NEON_FP : S3;
5905 %}
5906
5907 pipe_class vstore_reg_mem128(vecD src, vmem16 mem)
5908 %{
5909 single_instruction;
5910 mem : ISS(read);
5911 src : S2(read);
5912 INS01 : ISS;
5913 NEON_FP : S3;
5914 %}
5915
5916 //------- Integer ALU operations --------------------------
5917
5918 // Integer ALU reg-reg operation
5919 // Operands needed in EX1, result generated in EX2
5920 // Eg. ADD x0, x1, x2
5921 pipe_class ialu_reg_reg(iRegI dst, iRegI src1, iRegI src2)
5922 %{
5923 single_instruction;
5924 dst : EX2(write);
5925 src1 : EX1(read);
5926 src2 : EX1(read);
5927 INS01 : ISS; // Dual issue as instruction 0 or 1
5928 ALU : EX2;
5929 %}
5930
5931 // Integer ALU reg-reg operation with constant shift
5932 // Shifted register must be available in LATE_ISS instead of EX1
5933 // Eg. ADD x0, x1, x2, LSL #2
5934 pipe_class ialu_reg_reg_shift(iRegI dst, iRegI src1, iRegI src2, immI shift)
5935 %{
5936 single_instruction;
5937 dst : EX2(write);
5938 src1 : EX1(read);
5939 src2 : ISS(read);
5940 INS01 : ISS;
5941 ALU : EX2;
5942 %}
5943
5944 // Integer ALU reg operation with constant shift
5945 // Eg. LSL x0, x1, #shift
5946 pipe_class ialu_reg_shift(iRegI dst, iRegI src1)
5947 %{
5948 single_instruction;
5949 dst : EX2(write);
5950 src1 : ISS(read);
5951 INS01 : ISS;
5952 ALU : EX2;
5953 %}
5954
5955 // Integer ALU reg-reg operation with variable shift
5956 // Both operands must be available in LATE_ISS instead of EX1
5957 // Result is available in EX1 instead of EX2
5958 // Eg. LSLV x0, x1, x2
5959 pipe_class ialu_reg_reg_vshift(iRegI dst, iRegI src1, iRegI src2)
5960 %{
5961 single_instruction;
5962 dst : EX1(write);
5963 src1 : ISS(read);
5964 src2 : ISS(read);
5965 INS01 : ISS;
5966 ALU : EX1;
5967 %}
5968
5969 // Integer ALU reg-reg operation with extract
5970 // As for _vshift above, but result generated in EX2
5971 // Eg. EXTR x0, x1, x2, #N
5972 pipe_class ialu_reg_reg_extr(iRegI dst, iRegI src1, iRegI src2)
5973 %{
5974 single_instruction;
5975 dst : EX2(write);
5976 src1 : ISS(read);
5977 src2 : ISS(read);
5978 INS1 : ISS; // Can only dual issue as Instruction 1
5979 ALU : EX1;
5980 %}
5981
5982 // Integer ALU reg operation
5983 // Eg. NEG x0, x1
5984 pipe_class ialu_reg(iRegI dst, iRegI src)
5985 %{
5986 single_instruction;
5987 dst : EX2(write);
5988 src : EX1(read);
5989 INS01 : ISS;
5990 ALU : EX2;
5991 %}
5992
5993 // Integer ALU reg mmediate operation
5994 // Eg. ADD x0, x1, #N
5995 pipe_class ialu_reg_imm(iRegI dst, iRegI src1)
5996 %{
5997 single_instruction;
5998 dst : EX2(write);
5999 src1 : EX1(read);
6000 INS01 : ISS;
6001 ALU : EX2;
6002 %}
6003
6004 // Integer ALU immediate operation (no source operands)
6005 // Eg. MOV x0, #N
6006 pipe_class ialu_imm(iRegI dst)
6007 %{
6008 single_instruction;
6009 dst : EX1(write);
6010 INS01 : ISS;
6011 ALU : EX1;
6012 %}
6013
6014 //------- Compare operation -------------------------------
6015
6016 // Compare reg-reg
6017 // Eg. CMP x0, x1
6018 pipe_class icmp_reg_reg(rFlagsReg cr, iRegI op1, iRegI op2)
6019 %{
6020 single_instruction;
6021 // fixed_latency(16);
6022 cr : EX2(write);
6023 op1 : EX1(read);
6024 op2 : EX1(read);
6025 INS01 : ISS;
6026 ALU : EX2;
6027 %}
6028
6029 // Compare reg-reg
6030 // Eg. CMP x0, #N
6031 pipe_class icmp_reg_imm(rFlagsReg cr, iRegI op1)
6032 %{
6033 single_instruction;
6034 // fixed_latency(16);
6035 cr : EX2(write);
6036 op1 : EX1(read);
6037 INS01 : ISS;
6038 ALU : EX2;
6039 %}
6040
6041 //------- Conditional instructions ------------------------
6042
6043 // Conditional no operands
6044 // Eg. CSINC x0, zr, zr, <cond>
6045 pipe_class icond_none(iRegI dst, rFlagsReg cr)
6046 %{
6047 single_instruction;
6048 cr : EX1(read);
6049 dst : EX2(write);
6050 INS01 : ISS;
6051 ALU : EX2;
6052 %}
6053
6054 // Conditional 2 operand
6055 // EG. CSEL X0, X1, X2, <cond>
6056 pipe_class icond_reg_reg(iRegI dst, iRegI src1, iRegI src2, rFlagsReg cr)
6057 %{
6058 single_instruction;
6059 cr : EX1(read);
6060 src1 : EX1(read);
6061 src2 : EX1(read);
6062 dst : EX2(write);
6063 INS01 : ISS;
6064 ALU : EX2;
6065 %}
6066
6067 // Conditional 2 operand
6068 // EG. CSEL X0, X1, X2, <cond>
6069 pipe_class icond_reg(iRegI dst, iRegI src, rFlagsReg cr)
6070 %{
6071 single_instruction;
6072 cr : EX1(read);
6073 src : EX1(read);
6074 dst : EX2(write);
6075 INS01 : ISS;
6076 ALU : EX2;
6077 %}
6078
6079 //------- Multiply pipeline operations --------------------
6080
6081 // Multiply reg-reg
6082 // Eg. MUL w0, w1, w2
6083 pipe_class imul_reg_reg(iRegI dst, iRegI src1, iRegI src2)
6084 %{
6085 single_instruction;
6086 dst : WR(write);
6087 src1 : ISS(read);
6088 src2 : ISS(read);
6089 INS01 : ISS;
6090 MAC : WR;
6091 %}
6092
6093 // Multiply accumulate
6094 // Eg. MADD w0, w1, w2, w3
6095 pipe_class imac_reg_reg(iRegI dst, iRegI src1, iRegI src2, iRegI src3)
6096 %{
6097 single_instruction;
6098 dst : WR(write);
6099 src1 : ISS(read);
6100 src2 : ISS(read);
6101 src3 : ISS(read);
6102 INS01 : ISS;
6103 MAC : WR;
6104 %}
6105
6106 // Eg. MUL w0, w1, w2
6107 pipe_class lmul_reg_reg(iRegI dst, iRegI src1, iRegI src2)
6108 %{
6109 single_instruction;
6110 fixed_latency(3); // Maximum latency for 64 bit mul
6111 dst : WR(write);
6112 src1 : ISS(read);
6113 src2 : ISS(read);
6114 INS01 : ISS;
6115 MAC : WR;
6116 %}
6117
6118 // Multiply accumulate
6119 // Eg. MADD w0, w1, w2, w3
6120 pipe_class lmac_reg_reg(iRegI dst, iRegI src1, iRegI src2, iRegI src3)
6121 %{
6122 single_instruction;
6123 fixed_latency(3); // Maximum latency for 64 bit mul
6124 dst : WR(write);
6125 src1 : ISS(read);
6126 src2 : ISS(read);
6127 src3 : ISS(read);
6128 INS01 : ISS;
6129 MAC : WR;
6130 %}
6131
6132 //------- Divide pipeline operations --------------------
6133
6134 // Eg. SDIV w0, w1, w2
6135 pipe_class idiv_reg_reg(iRegI dst, iRegI src1, iRegI src2)
6136 %{
6137 single_instruction;
6138 fixed_latency(8); // Maximum latency for 32 bit divide
6139 dst : WR(write);
6140 src1 : ISS(read);
6141 src2 : ISS(read);
6142 INS0 : ISS; // Can only dual issue as instruction 0
6143 DIV : WR;
6144 %}
6145
6146 // Eg. SDIV x0, x1, x2
6147 pipe_class ldiv_reg_reg(iRegI dst, iRegI src1, iRegI src2)
6148 %{
6149 single_instruction;
6150 fixed_latency(16); // Maximum latency for 64 bit divide
6151 dst : WR(write);
6152 src1 : ISS(read);
6153 src2 : ISS(read);
6154 INS0 : ISS; // Can only dual issue as instruction 0
6155 DIV : WR;
6156 %}
6157
6158 //------- Load pipeline operations ------------------------
6159
6160 // Load - prefetch
6161 // Eg. PFRM <mem>
6162 pipe_class iload_prefetch(memory mem)
6163 %{
6164 single_instruction;
6165 mem : ISS(read);
6166 INS01 : ISS;
6167 LDST : WR;
6168 %}
6169
6170 // Load - reg, mem
6171 // Eg. LDR x0, <mem>
6172 pipe_class iload_reg_mem(iRegI dst, memory mem)
6173 %{
6174 single_instruction;
6175 dst : WR(write);
6176 mem : ISS(read);
6177 INS01 : ISS;
6178 LDST : WR;
6179 %}
6180
6181 // Load - reg, reg
6182 // Eg. LDR x0, [sp, x1]
6183 pipe_class iload_reg_reg(iRegI dst, iRegI src)
6184 %{
6185 single_instruction;
6186 dst : WR(write);
6187 src : ISS(read);
6188 INS01 : ISS;
6189 LDST : WR;
6190 %}
6191
6192 //------- Store pipeline operations -----------------------
6193
6194 // Store - zr, mem
6195 // Eg. STR zr, <mem>
6196 pipe_class istore_mem(memory mem)
6197 %{
6198 single_instruction;
6199 mem : ISS(read);
6200 INS01 : ISS;
6201 LDST : WR;
6202 %}
6203
6204 // Store - reg, mem
6205 // Eg. STR x0, <mem>
6206 pipe_class istore_reg_mem(iRegI src, memory mem)
6207 %{
6208 single_instruction;
6209 mem : ISS(read);
6210 src : EX2(read);
6211 INS01 : ISS;
6212 LDST : WR;
6213 %}
6214
6215 // Store - reg, reg
6216 // Eg. STR x0, [sp, x1]
6217 pipe_class istore_reg_reg(iRegI dst, iRegI src)
6218 %{
6219 single_instruction;
6220 dst : ISS(read);
6221 src : EX2(read);
6222 INS01 : ISS;
6223 LDST : WR;
6224 %}
6225
6226 //------- Store pipeline operations -----------------------
6227
6228 // Branch
6229 pipe_class pipe_branch()
6230 %{
6231 single_instruction;
6232 INS01 : ISS;
6233 BRANCH : EX1;
6234 %}
6235
6236 // Conditional branch
6237 pipe_class pipe_branch_cond(rFlagsReg cr)
6238 %{
6239 single_instruction;
6240 cr : EX1(read);
6241 INS01 : ISS;
6242 BRANCH : EX1;
6243 %}
6244
6245 // Compare & Branch
6246 // EG. CBZ/CBNZ
6247 pipe_class pipe_cmp_branch(iRegI op1)
6248 %{
6249 single_instruction;
6250 op1 : EX1(read);
6251 INS01 : ISS;
6252 BRANCH : EX1;
6253 %}
6254
6255 //------- Synchronisation operations ----------------------
6256
6257 // Any operation requiring serialization.
6258 // EG. DMB/Atomic Ops/Load Acquire/Str Release
6259 pipe_class pipe_serial()
6260 %{
6261 single_instruction;
6262 force_serialization;
6263 fixed_latency(16);
6264 INS01 : ISS(2); // Cannot dual issue with any other instruction
6265 LDST : WR;
6266 %}
6267
6268 // Generic big/slow expanded idiom - also serialized
6269 pipe_class pipe_slow()
6270 %{
6271 instruction_count(10);
6272 multiple_bundles;
6273 force_serialization;
6274 fixed_latency(16);
6275 INS01 : ISS(2); // Cannot dual issue with any other instruction
6276 LDST : WR;
6277 %}
6278
6279 // Empty pipeline class
6280 pipe_class pipe_class_empty()
6281 %{
6282 single_instruction;
6283 fixed_latency(0);
6284 %}
6285
6286 // Default pipeline class.
6287 pipe_class pipe_class_default()
6288 %{
6289 single_instruction;
6290 fixed_latency(2);
6291 %}
6292
6293 // Pipeline class for compares.
6294 pipe_class pipe_class_compare()
6295 %{
6296 single_instruction;
6297 fixed_latency(16);
6298 %}
6299
6300 // Pipeline class for memory operations.
6301 pipe_class pipe_class_memory()
6302 %{
6303 single_instruction;
6304 fixed_latency(16);
6305 %}
6306
6307 // Pipeline class for call.
6308 pipe_class pipe_class_call()
6309 %{
6310 single_instruction;
6311 fixed_latency(100);
6312 %}
6313
6314 // Define the class for the Nop node.
6315 define %{
6316 MachNop = pipe_class_empty;
6317 %}
6318
6319 %}
6320 //----------INSTRUCTIONS-------------------------------------------------------
6321 //
6322 // match -- States which machine-independent subtree may be replaced
6323 // by this instruction.
6324 // ins_cost -- The estimated cost of this instruction is used by instruction
6325 // selection to identify a minimum cost tree of machine
6326 // instructions that matches a tree of machine-independent
6327 // instructions.
6328 // format -- A string providing the disassembly for this instruction.
6329 // The value of an instruction's operand may be inserted
6330 // by referring to it with a '$' prefix.
6331 // opcode -- Three instruction opcodes may be provided. These are referred
6332 // to within an encode class as $primary, $secondary, and $tertiary
6333 // rrspectively. The primary opcode is commonly used to
6334 // indicate the type of machine instruction, while secondary
6335 // and tertiary are often used for prefix options or addressing
6336 // modes.
6337 // ins_encode -- A list of encode classes with parameters. The encode class
6338 // name must have been defined in an 'enc_class' specification
6339 // in the encode section of the architecture description.
6340
6341 // ============================================================================
6342 // Memory (Load/Store) Instructions
6343
6344 // Load Instructions
6345
6346 // Load Byte (8 bit signed)
6347 instruct loadB(iRegINoSp dst, memory mem)
6348 %{
6349 match(Set dst (LoadB mem));
6350 predicate(!needs_acquiring_load(n));
6351
6352 ins_cost(4 * INSN_COST);
6353 format %{ "ldrsbw $dst, $mem\t# byte" %}
6354
6355 ins_encode(aarch64_enc_ldrsbw(dst, mem));
6356
6357 ins_pipe(iload_reg_mem);
6358 %}
6359
6360 // Load Byte (8 bit signed) into long
6361 instruct loadB2L(iRegLNoSp dst, memory mem)
6362 %{
6363 match(Set dst (ConvI2L (LoadB mem)));
6364 predicate(!needs_acquiring_load(n->in(1)));
6365
6366 ins_cost(4 * INSN_COST);
6367 format %{ "ldrsb $dst, $mem\t# byte" %}
6368
6369 ins_encode(aarch64_enc_ldrsb(dst, mem));
6370
6371 ins_pipe(iload_reg_mem);
6372 %}
6373
6374 // Load Byte (8 bit unsigned)
6375 instruct loadUB(iRegINoSp dst, memory mem)
6376 %{
6377 match(Set dst (LoadUB mem));
6378 predicate(!needs_acquiring_load(n));
6379
6380 ins_cost(4 * INSN_COST);
6381 format %{ "ldrbw $dst, $mem\t# byte" %}
6382
6383 ins_encode(aarch64_enc_ldrb(dst, mem));
6384
6385 ins_pipe(iload_reg_mem);
6386 %}
6387
6388 // Load Byte (8 bit unsigned) into long
6389 instruct loadUB2L(iRegLNoSp dst, memory mem)
6390 %{
6391 match(Set dst (ConvI2L (LoadUB mem)));
6392 predicate(!needs_acquiring_load(n->in(1)));
6393
6394 ins_cost(4 * INSN_COST);
6395 format %{ "ldrb $dst, $mem\t# byte" %}
6396
6397 ins_encode(aarch64_enc_ldrb(dst, mem));
6398
6399 ins_pipe(iload_reg_mem);
6400 %}
6401
6402 // Load Short (16 bit signed)
6403 instruct loadS(iRegINoSp dst, memory mem)
6404 %{
6405 match(Set dst (LoadS mem));
6406 predicate(!needs_acquiring_load(n));
6407
6408 ins_cost(4 * INSN_COST);
6409 format %{ "ldrshw $dst, $mem\t# short" %}
6410
6411 ins_encode(aarch64_enc_ldrshw(dst, mem));
6412
6413 ins_pipe(iload_reg_mem);
6414 %}
6415
6416 // Load Short (16 bit signed) into long
6417 instruct loadS2L(iRegLNoSp dst, memory mem)
6418 %{
6419 match(Set dst (ConvI2L (LoadS mem)));
6420 predicate(!needs_acquiring_load(n->in(1)));
6421
6422 ins_cost(4 * INSN_COST);
6423 format %{ "ldrsh $dst, $mem\t# short" %}
6424
6425 ins_encode(aarch64_enc_ldrsh(dst, mem));
6426
6427 ins_pipe(iload_reg_mem);
6428 %}
6429
6430 // Load Char (16 bit unsigned)
6431 instruct loadUS(iRegINoSp dst, memory mem)
6432 %{
6433 match(Set dst (LoadUS mem));
6434 predicate(!needs_acquiring_load(n));
6435
6436 ins_cost(4 * INSN_COST);
6437 format %{ "ldrh $dst, $mem\t# short" %}
6438
6439 ins_encode(aarch64_enc_ldrh(dst, mem));
6440
6441 ins_pipe(iload_reg_mem);
6442 %}
6443
6444 // Load Short/Char (16 bit unsigned) into long
6445 instruct loadUS2L(iRegLNoSp dst, memory mem)
6446 %{
6447 match(Set dst (ConvI2L (LoadUS mem)));
6448 predicate(!needs_acquiring_load(n->in(1)));
6449
6450 ins_cost(4 * INSN_COST);
6451 format %{ "ldrh $dst, $mem\t# short" %}
6452
6453 ins_encode(aarch64_enc_ldrh(dst, mem));
6454
6455 ins_pipe(iload_reg_mem);
6456 %}
6457
6458 // Load Integer (32 bit signed)
6459 instruct loadI(iRegINoSp dst, memory mem)
6460 %{
6461 match(Set dst (LoadI mem));
6462 predicate(!needs_acquiring_load(n));
6463
6464 ins_cost(4 * INSN_COST);
6465 format %{ "ldrw $dst, $mem\t# int" %}
6466
6467 ins_encode(aarch64_enc_ldrw(dst, mem));
6468
6469 ins_pipe(iload_reg_mem);
6470 %}
6471
6472 // Load Integer (32 bit signed) into long
6473 instruct loadI2L(iRegLNoSp dst, memory mem)
6474 %{
6475 match(Set dst (ConvI2L (LoadI mem)));
6476 predicate(!needs_acquiring_load(n->in(1)));
6477
6478 ins_cost(4 * INSN_COST);
6479 format %{ "ldrsw $dst, $mem\t# int" %}
6480
6481 ins_encode(aarch64_enc_ldrsw(dst, mem));
6482
6483 ins_pipe(iload_reg_mem);
6484 %}
6485
6486 // Load Integer (32 bit unsigned) into long
6487 instruct loadUI2L(iRegLNoSp dst, memory mem, immL_32bits mask)
6488 %{
6489 match(Set dst (AndL (ConvI2L (LoadI mem)) mask));
6490 predicate(!needs_acquiring_load(n->in(1)->in(1)->as_Load()));
6491
6492 ins_cost(4 * INSN_COST);
6493 format %{ "ldrw $dst, $mem\t# int" %}
6494
6495 ins_encode(aarch64_enc_ldrw(dst, mem));
6496
6497 ins_pipe(iload_reg_mem);
6498 %}
6499
6500 // Load Long (64 bit signed)
6501 instruct loadL(iRegLNoSp dst, memory mem)
6502 %{
6503 match(Set dst (LoadL mem));
6504 predicate(!needs_acquiring_load(n));
6505
6506 ins_cost(4 * INSN_COST);
6507 format %{ "ldr $dst, $mem\t# int" %}
6508
6509 ins_encode(aarch64_enc_ldr(dst, mem));
6510
6511 ins_pipe(iload_reg_mem);
6512 %}
6513
6514 // Load Range
6515 instruct loadRange(iRegINoSp dst, memory mem)
6516 %{
6517 match(Set dst (LoadRange mem));
6518
6519 ins_cost(4 * INSN_COST);
6520 format %{ "ldrw $dst, $mem\t# range" %}
6521
6522 ins_encode(aarch64_enc_ldrw(dst, mem));
6523
6524 ins_pipe(iload_reg_mem);
6525 %}
6526
6527 // Load Pointer
6528 instruct loadP(iRegPNoSp dst, memory mem)
6529 %{
6530 match(Set dst (LoadP mem));
6531 predicate(!needs_acquiring_load(n));
6532
6533 ins_cost(4 * INSN_COST);
6534 format %{ "ldr $dst, $mem\t# ptr" %}
6535
6536 ins_encode(aarch64_enc_ldr(dst, mem));
6537
6538 ins_pipe(iload_reg_mem);
6539 %}
6540
6541 // Load Compressed Pointer
6542 instruct loadN(iRegNNoSp dst, memory mem)
6543 %{
6544 match(Set dst (LoadN mem));
6545 predicate(!needs_acquiring_load(n));
6546
6547 ins_cost(4 * INSN_COST);
6548 format %{ "ldrw $dst, $mem\t# compressed ptr" %}
6549
6550 ins_encode(aarch64_enc_ldrw(dst, mem));
6551
6552 ins_pipe(iload_reg_mem);
6553 %}
6554
6555 // Load Klass Pointer
6556 instruct loadKlass(iRegPNoSp dst, memory mem)
6557 %{
6558 match(Set dst (LoadKlass mem));
6559 predicate(!needs_acquiring_load(n));
6560
6561 ins_cost(4 * INSN_COST);
6562 format %{ "ldr $dst, $mem\t# class" %}
6563
6564 ins_encode(aarch64_enc_ldr(dst, mem));
6565
6566 ins_pipe(iload_reg_mem);
6567 %}
6568
6569 // Load Narrow Klass Pointer
6570 instruct loadNKlass(iRegNNoSp dst, memory mem)
6571 %{
6572 match(Set dst (LoadNKlass mem));
6573 predicate(!needs_acquiring_load(n));
6574
6575 ins_cost(4 * INSN_COST);
6576 format %{ "ldrw $dst, $mem\t# compressed class ptr" %}
6577
6578 ins_encode(aarch64_enc_ldrw(dst, mem));
6579
6580 ins_pipe(iload_reg_mem);
6581 %}
6582
6583 // Load Float
6584 instruct loadF(vRegF dst, memory mem)
6585 %{
6586 match(Set dst (LoadF mem));
6587 predicate(!needs_acquiring_load(n));
6588
6589 ins_cost(4 * INSN_COST);
6590 format %{ "ldrs $dst, $mem\t# float" %}
6591
6592 ins_encode( aarch64_enc_ldrs(dst, mem) );
6593
6594 ins_pipe(pipe_class_memory);
6595 %}
6596
6597 // Load Double
6598 instruct loadD(vRegD dst, memory mem)
6599 %{
6600 match(Set dst (LoadD mem));
6601 predicate(!needs_acquiring_load(n));
6602
6603 ins_cost(4 * INSN_COST);
6604 format %{ "ldrd $dst, $mem\t# double" %}
6605
6606 ins_encode( aarch64_enc_ldrd(dst, mem) );
6607
6608 ins_pipe(pipe_class_memory);
6609 %}
6610
6611
6612 // Load Int Constant
6613 instruct loadConI(iRegINoSp dst, immI src)
6614 %{
6615 match(Set dst src);
6616
6617 ins_cost(INSN_COST);
6618 format %{ "mov $dst, $src\t# int" %}
6619
6620 ins_encode( aarch64_enc_movw_imm(dst, src) );
6621
6622 ins_pipe(ialu_imm);
6623 %}
6624
6625 // Load Long Constant
6626 instruct loadConL(iRegLNoSp dst, immL src)
6627 %{
6628 match(Set dst src);
6629
6630 ins_cost(INSN_COST);
6631 format %{ "mov $dst, $src\t# long" %}
6632
6633 ins_encode( aarch64_enc_mov_imm(dst, src) );
6634
6635 ins_pipe(ialu_imm);
6636 %}
6637
6638 // Load Pointer Constant
6639
6640 instruct loadConP(iRegPNoSp dst, immP con)
6641 %{
6642 match(Set dst con);
6643
6644 ins_cost(INSN_COST * 4);
6645 format %{
6646 "mov $dst, $con\t# ptr\n\t"
6647 %}
6648
6649 ins_encode(aarch64_enc_mov_p(dst, con));
6650
6651 ins_pipe(ialu_imm);
6652 %}
6653
6654 // Load Null Pointer Constant
6655
6656 instruct loadConP0(iRegPNoSp dst, immP0 con)
6657 %{
6658 match(Set dst con);
6659
6660 ins_cost(INSN_COST);
6661 format %{ "mov $dst, $con\t# NULL ptr" %}
6662
6663 ins_encode(aarch64_enc_mov_p0(dst, con));
6664
6665 ins_pipe(ialu_imm);
6666 %}
6667
6668 // Load Pointer Constant One
6669
6670 instruct loadConP1(iRegPNoSp dst, immP_1 con)
6671 %{
6672 match(Set dst con);
6673
6674 ins_cost(INSN_COST);
6675 format %{ "mov $dst, $con\t# NULL ptr" %}
6676
6677 ins_encode(aarch64_enc_mov_p1(dst, con));
6678
6679 ins_pipe(ialu_imm);
6680 %}
6681
6682 // Load Poll Page Constant
6683
6684 instruct loadConPollPage(iRegPNoSp dst, immPollPage con)
6685 %{
6686 match(Set dst con);
6687
6688 ins_cost(INSN_COST);
6689 format %{ "adr $dst, $con\t# Poll Page Ptr" %}
6690
6691 ins_encode(aarch64_enc_mov_poll_page(dst, con));
6692
6693 ins_pipe(ialu_imm);
6694 %}
6695
6696 // Load Byte Map Base Constant
6697
6698 instruct loadByteMapBase(iRegPNoSp dst, immByteMapBase con)
6699 %{
6700 match(Set dst con);
6701
6702 ins_cost(INSN_COST);
6703 format %{ "adr $dst, $con\t# Byte Map Base" %}
6704
6705 ins_encode(aarch64_enc_mov_byte_map_base(dst, con));
6706
6707 ins_pipe(ialu_imm);
6708 %}
6709
6710 // Load Narrow Pointer Constant
6711
6712 instruct loadConN(iRegNNoSp dst, immN con)
6713 %{
6714 match(Set dst con);
6715
6716 ins_cost(INSN_COST * 4);
6717 format %{ "mov $dst, $con\t# compressed ptr" %}
6718
6719 ins_encode(aarch64_enc_mov_n(dst, con));
6720
6721 ins_pipe(ialu_imm);
6722 %}
6723
6724 // Load Narrow Null Pointer Constant
6725
6726 instruct loadConN0(iRegNNoSp dst, immN0 con)
6727 %{
6728 match(Set dst con);
6729
6730 ins_cost(INSN_COST);
6731 format %{ "mov $dst, $con\t# compressed NULL ptr" %}
6732
6733 ins_encode(aarch64_enc_mov_n0(dst, con));
6734
6735 ins_pipe(ialu_imm);
6736 %}
6737
6738 // Load Narrow Klass Constant
6739
6740 instruct loadConNKlass(iRegNNoSp dst, immNKlass con)
6741 %{
6742 match(Set dst con);
6743
6744 ins_cost(INSN_COST);
6745 format %{ "mov $dst, $con\t# compressed klass ptr" %}
6746
6747 ins_encode(aarch64_enc_mov_nk(dst, con));
6748
6749 ins_pipe(ialu_imm);
6750 %}
6751
6752 // Load Packed Float Constant
6753
6754 instruct loadConF_packed(vRegF dst, immFPacked con) %{
6755 match(Set dst con);
6756 ins_cost(INSN_COST * 4);
6757 format %{ "fmovs $dst, $con"%}
6758 ins_encode %{
6759 __ fmovs(as_FloatRegister($dst$$reg), (double)$con$$constant);
6760 %}
6761
6762 ins_pipe(fp_imm_s);
6763 %}
6764
6765 // Load Float Constant
6766
6767 instruct loadConF(vRegF dst, immF con) %{
6768 match(Set dst con);
6769
6770 ins_cost(INSN_COST * 4);
6771
6772 format %{
6773 "ldrs $dst, [$constantaddress]\t# load from constant table: float=$con\n\t"
6774 %}
6775
6776 ins_encode %{
6777 __ ldrs(as_FloatRegister($dst$$reg), $constantaddress($con));
6778 %}
6779
6780 ins_pipe(fp_load_constant_s);
6781 %}
6782
6783 // Load Packed Double Constant
6784
6785 instruct loadConD_packed(vRegD dst, immDPacked con) %{
6786 match(Set dst con);
6787 ins_cost(INSN_COST);
6788 format %{ "fmovd $dst, $con"%}
6789 ins_encode %{
6790 __ fmovd(as_FloatRegister($dst$$reg), $con$$constant);
6791 %}
6792
6793 ins_pipe(fp_imm_d);
6794 %}
6795
6796 // Load Double Constant
6797
6798 instruct loadConD(vRegD dst, immD con) %{
6799 match(Set dst con);
6800
6801 ins_cost(INSN_COST * 5);
6802 format %{
6803 "ldrd $dst, [$constantaddress]\t# load from constant table: float=$con\n\t"
6804 %}
6805
6806 ins_encode %{
6807 __ ldrd(as_FloatRegister($dst$$reg), $constantaddress($con));
6808 %}
6809
6810 ins_pipe(fp_load_constant_d);
6811 %}
6812
6813 // Store Instructions
6814
6815 // Store CMS card-mark Immediate
6816 instruct storeimmCM0(immI0 zero, memory mem)
6817 %{
6818 match(Set mem (StoreCM mem zero));
6819 predicate(unnecessary_storestore(n));
6820
6821 ins_cost(INSN_COST);
6822 format %{ "strb zr, $mem\t# byte" %}
6823
6824 ins_encode(aarch64_enc_strb0(mem));
6825
6826 ins_pipe(istore_mem);
6827 %}
6828
6829 // Store CMS card-mark Immediate with intervening StoreStore
6830 // needed when using CMS with no conditional card marking
6831 instruct storeimmCM0_ordered(immI0 zero, memory mem)
6832 %{
6833 match(Set mem (StoreCM mem zero));
6834
6835 ins_cost(INSN_COST * 2);
6836 format %{ "dmb ishst"
6837 "\n\tstrb zr, $mem\t# byte" %}
6838
6839 ins_encode(aarch64_enc_strb0_ordered(mem));
6840
6841 ins_pipe(istore_mem);
6842 %}
6843
6844 // Store Byte
6845 instruct storeB(iRegIorL2I src, memory mem)
6846 %{
6847 match(Set mem (StoreB mem src));
6848 predicate(!needs_releasing_store(n));
6849
6850 ins_cost(INSN_COST);
6851 format %{ "strb $src, $mem\t# byte" %}
6852
6853 ins_encode(aarch64_enc_strb(src, mem));
6854
6855 ins_pipe(istore_reg_mem);
6856 %}
6857
6858
6859 instruct storeimmB0(immI0 zero, memory mem)
6860 %{
6861 match(Set mem (StoreB mem zero));
6862 predicate(!needs_releasing_store(n));
6863
6864 ins_cost(INSN_COST);
6865 format %{ "strb zr, $mem\t# byte" %}
6866
6867 ins_encode(aarch64_enc_strb0(mem));
6868
6869 ins_pipe(istore_mem);
6870 %}
6871
6872 // Store Char/Short
6873 instruct storeC(iRegIorL2I src, memory mem)
6874 %{
6875 match(Set mem (StoreC mem src));
6876 predicate(!needs_releasing_store(n));
6877
6878 ins_cost(INSN_COST);
6879 format %{ "strh $src, $mem\t# short" %}
6880
6881 ins_encode(aarch64_enc_strh(src, mem));
6882
6883 ins_pipe(istore_reg_mem);
6884 %}
6885
6886 instruct storeimmC0(immI0 zero, memory mem)
6887 %{
6888 match(Set mem (StoreC mem zero));
6889 predicate(!needs_releasing_store(n));
6890
6891 ins_cost(INSN_COST);
6892 format %{ "strh zr, $mem\t# short" %}
6893
6894 ins_encode(aarch64_enc_strh0(mem));
6895
6896 ins_pipe(istore_mem);
6897 %}
6898
6899 // Store Integer
6900
6901 instruct storeI(iRegIorL2I src, memory mem)
6902 %{
6903 match(Set mem(StoreI mem src));
6904 predicate(!needs_releasing_store(n));
6905
6906 ins_cost(INSN_COST);
6907 format %{ "strw $src, $mem\t# int" %}
6908
6909 ins_encode(aarch64_enc_strw(src, mem));
6910
6911 ins_pipe(istore_reg_mem);
6912 %}
6913
6914 instruct storeimmI0(immI0 zero, memory mem)
6915 %{
6916 match(Set mem(StoreI mem zero));
6917 predicate(!needs_releasing_store(n));
6918
6919 ins_cost(INSN_COST);
6920 format %{ "strw zr, $mem\t# int" %}
6921
6922 ins_encode(aarch64_enc_strw0(mem));
6923
6924 ins_pipe(istore_mem);
6925 %}
6926
6927 // Store Long (64 bit signed)
6928 instruct storeL(iRegL src, memory mem)
6929 %{
6930 match(Set mem (StoreL mem src));
6931 predicate(!needs_releasing_store(n));
6932
6933 ins_cost(INSN_COST);
6934 format %{ "str $src, $mem\t# int" %}
6935
6936 ins_encode(aarch64_enc_str(src, mem));
6937
6938 ins_pipe(istore_reg_mem);
6939 %}
6940
6941 // Store Long (64 bit signed)
6942 instruct storeimmL0(immL0 zero, memory mem)
6943 %{
6944 match(Set mem (StoreL mem zero));
6945 predicate(!needs_releasing_store(n));
6946
6947 ins_cost(INSN_COST);
6948 format %{ "str zr, $mem\t# int" %}
6949
6950 ins_encode(aarch64_enc_str0(mem));
6951
6952 ins_pipe(istore_mem);
6953 %}
6954
6955 // Store Pointer
6956 instruct storeP(iRegP src, memory mem)
6957 %{
6958 match(Set mem (StoreP mem src));
6959 predicate(!needs_releasing_store(n));
6960
6961 ins_cost(INSN_COST);
6962 format %{ "str $src, $mem\t# ptr" %}
6963
6964 ins_encode %{
6965 int opcode = $mem->opcode();
6966 Register base = as_Register($mem$$base);
6967 int index = $mem$$index;
6968 int size = $mem$$scale;
6969 int disp = $mem$$disp;
6970 Register reg = as_Register($src$$reg);
6971
6972 // we sometimes get asked to store the stack pointer into the
6973 // current thread -- we cannot do that directly on AArch64
6974 if (reg == r31_sp) {
6975 MacroAssembler _masm(&cbuf);
6976 assert(as_Register($mem$$base) == rthread, "unexpected store for sp");
6977 __ mov(rscratch2, sp);
6978 reg = rscratch2;
6979 }
6980 Address::extend scale;
6981
6982 // Hooboy, this is fugly. We need a way to communicate to the
6983 // encoder that the index needs to be sign extended, so we have to
6984 // enumerate all the cases.
6985 switch (opcode) {
6986 case INDINDEXSCALEDOFFSETI2L:
6987 case INDINDEXSCALEDI2L:
6988 case INDINDEXSCALEDOFFSETI2LN:
6989 case INDINDEXSCALEDI2LN:
6990 case INDINDEXOFFSETI2L:
6991 case INDINDEXOFFSETI2LN:
6992 scale = Address::sxtw(size);
6993 break;
6994 default:
6995 scale = Address::lsl(size);
6996 }
6997 Address adr;
6998 if (index == -1) {
6999 adr = Address(base, disp);
7000 } else {
7001 if (disp == 0) {
7002 adr = Address(base, as_Register(index), scale);
7003 } else {
7004 __ lea(rscratch1, Address(base, disp));
7005 adr = Address(rscratch1, as_Register(index), scale);
7006 }
7007 }
7008
7009 __ str(reg, adr);
7010 %}
7011
7012 ins_pipe(istore_reg_mem);
7013 %}
7014
7015 // Store Pointer
7016 instruct storeimmP0(immP0 zero, memory mem)
7017 %{
7018 match(Set mem (StoreP mem zero));
7019 predicate(!needs_releasing_store(n));
7020
7021 ins_cost(INSN_COST);
7022 format %{ "str zr, $mem\t# ptr" %}
7023
7024 ins_encode(aarch64_enc_str0(mem));
7025
7026 ins_pipe(istore_mem);
7027 %}
7028
7029 // Store Compressed Pointer
7030 instruct storeN(iRegN src, memory mem)
7031 %{
7032 match(Set mem (StoreN mem src));
7033 predicate(!needs_releasing_store(n));
7034
7035 ins_cost(INSN_COST);
7036 format %{ "strw $src, $mem\t# compressed ptr" %}
7037
7038 ins_encode(aarch64_enc_strw(src, mem));
7039
7040 ins_pipe(istore_reg_mem);
7041 %}
7042
7043 instruct storeImmN0(iRegIHeapbase heapbase, immN0 zero, memory mem)
7044 %{
7045 match(Set mem (StoreN mem zero));
7046 predicate(Universe::narrow_oop_base() == NULL &&
7047 Universe::narrow_klass_base() == NULL &&
7048 (!needs_releasing_store(n)));
7049
7050 ins_cost(INSN_COST);
7051 format %{ "strw rheapbase, $mem\t# compressed ptr (rheapbase==0)" %}
7052
7053 ins_encode(aarch64_enc_strw(heapbase, mem));
7054
7055 ins_pipe(istore_reg_mem);
7056 %}
7057
7058 // Store Float
7059 instruct storeF(vRegF src, memory mem)
7060 %{
7061 match(Set mem (StoreF mem src));
7062 predicate(!needs_releasing_store(n));
7063
7064 ins_cost(INSN_COST);
7065 format %{ "strs $src, $mem\t# float" %}
7066
7067 ins_encode( aarch64_enc_strs(src, mem) );
7068
7069 ins_pipe(pipe_class_memory);
7070 %}
7071
7072 // TODO
7073 // implement storeImmF0 and storeFImmPacked
7074
7075 // Store Double
7076 instruct storeD(vRegD src, memory mem)
7077 %{
7078 match(Set mem (StoreD mem src));
7079 predicate(!needs_releasing_store(n));
7080
7081 ins_cost(INSN_COST);
7082 format %{ "strd $src, $mem\t# double" %}
7083
7084 ins_encode( aarch64_enc_strd(src, mem) );
7085
7086 ins_pipe(pipe_class_memory);
7087 %}
7088
7089 // Store Compressed Klass Pointer
7090 instruct storeNKlass(iRegN src, memory mem)
7091 %{
7092 predicate(!needs_releasing_store(n));
7093 match(Set mem (StoreNKlass mem src));
7094
7095 ins_cost(INSN_COST);
7096 format %{ "strw $src, $mem\t# compressed klass ptr" %}
7097
7098 ins_encode(aarch64_enc_strw(src, mem));
7099
7100 ins_pipe(istore_reg_mem);
7101 %}
7102
7103 // TODO
7104 // implement storeImmD0 and storeDImmPacked
7105
7106 // prefetch instructions
7107 // Must be safe to execute with invalid address (cannot fault).
7108
7109 instruct prefetchr( memory mem ) %{
7110 match(PrefetchRead mem);
7111
7112 ins_cost(INSN_COST);
7113 format %{ "prfm $mem, PLDL1KEEP\t# Prefetch into level 1 cache read keep" %}
7114
7115 ins_encode( aarch64_enc_prefetchr(mem) );
7116
7117 ins_pipe(iload_prefetch);
7118 %}
7119
7120 instruct prefetchw( memory mem ) %{
7121 match(PrefetchAllocation mem);
7122
7123 ins_cost(INSN_COST);
7124 format %{ "prfm $mem, PSTL1KEEP\t# Prefetch into level 1 cache write keep" %}
7125
7126 ins_encode( aarch64_enc_prefetchw(mem) );
7127
7128 ins_pipe(iload_prefetch);
7129 %}
7130
7131 instruct prefetchnta( memory mem ) %{
7132 match(PrefetchWrite mem);
7133
7134 ins_cost(INSN_COST);
7135 format %{ "prfm $mem, PSTL1STRM\t# Prefetch into level 1 cache write streaming" %}
7136
7137 ins_encode( aarch64_enc_prefetchnta(mem) );
7138
7139 ins_pipe(iload_prefetch);
7140 %}
7141
7142 // ---------------- volatile loads and stores ----------------
7143
7144 // Load Byte (8 bit signed)
7145 instruct loadB_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
7146 %{
7147 match(Set dst (LoadB mem));
7148
7149 ins_cost(VOLATILE_REF_COST);
7150 format %{ "ldarsb $dst, $mem\t# byte" %}
7151
7152 ins_encode(aarch64_enc_ldarsb(dst, mem));
7153
7154 ins_pipe(pipe_serial);
7155 %}
7156
7157 // Load Byte (8 bit signed) into long
7158 instruct loadB2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
7159 %{
7160 match(Set dst (ConvI2L (LoadB mem)));
7161
7162 ins_cost(VOLATILE_REF_COST);
7163 format %{ "ldarsb $dst, $mem\t# byte" %}
7164
7165 ins_encode(aarch64_enc_ldarsb(dst, mem));
7166
7167 ins_pipe(pipe_serial);
7168 %}
7169
7170 // Load Byte (8 bit unsigned)
7171 instruct loadUB_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
7172 %{
7173 match(Set dst (LoadUB mem));
7174
7175 ins_cost(VOLATILE_REF_COST);
7176 format %{ "ldarb $dst, $mem\t# byte" %}
7177
7178 ins_encode(aarch64_enc_ldarb(dst, mem));
7179
7180 ins_pipe(pipe_serial);
7181 %}
7182
7183 // Load Byte (8 bit unsigned) into long
7184 instruct loadUB2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
7185 %{
7186 match(Set dst (ConvI2L (LoadUB mem)));
7187
7188 ins_cost(VOLATILE_REF_COST);
7189 format %{ "ldarb $dst, $mem\t# byte" %}
7190
7191 ins_encode(aarch64_enc_ldarb(dst, mem));
7192
7193 ins_pipe(pipe_serial);
7194 %}
7195
7196 // Load Short (16 bit signed)
7197 instruct loadS_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
7198 %{
7199 match(Set dst (LoadS mem));
7200
7201 ins_cost(VOLATILE_REF_COST);
7202 format %{ "ldarshw $dst, $mem\t# short" %}
7203
7204 ins_encode(aarch64_enc_ldarshw(dst, mem));
7205
7206 ins_pipe(pipe_serial);
7207 %}
7208
7209 instruct loadUS_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
7210 %{
7211 match(Set dst (LoadUS mem));
7212
7213 ins_cost(VOLATILE_REF_COST);
7214 format %{ "ldarhw $dst, $mem\t# short" %}
7215
7216 ins_encode(aarch64_enc_ldarhw(dst, mem));
7217
7218 ins_pipe(pipe_serial);
7219 %}
7220
7221 // Load Short/Char (16 bit unsigned) into long
7222 instruct loadUS2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
7223 %{
7224 match(Set dst (ConvI2L (LoadUS mem)));
7225
7226 ins_cost(VOLATILE_REF_COST);
7227 format %{ "ldarh $dst, $mem\t# short" %}
7228
7229 ins_encode(aarch64_enc_ldarh(dst, mem));
7230
7231 ins_pipe(pipe_serial);
7232 %}
7233
7234 // Load Short/Char (16 bit signed) into long
7235 instruct loadS2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
7236 %{
7237 match(Set dst (ConvI2L (LoadS mem)));
7238
7239 ins_cost(VOLATILE_REF_COST);
7240 format %{ "ldarh $dst, $mem\t# short" %}
7241
7242 ins_encode(aarch64_enc_ldarsh(dst, mem));
7243
7244 ins_pipe(pipe_serial);
7245 %}
7246
7247 // Load Integer (32 bit signed)
7248 instruct loadI_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
7249 %{
7250 match(Set dst (LoadI mem));
7251
7252 ins_cost(VOLATILE_REF_COST);
7253 format %{ "ldarw $dst, $mem\t# int" %}
7254
7255 ins_encode(aarch64_enc_ldarw(dst, mem));
7256
7257 ins_pipe(pipe_serial);
7258 %}
7259
7260 // Load Integer (32 bit unsigned) into long
7261 instruct loadUI2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem, immL_32bits mask)
7262 %{
7263 match(Set dst (AndL (ConvI2L (LoadI mem)) mask));
7264
7265 ins_cost(VOLATILE_REF_COST);
7266 format %{ "ldarw $dst, $mem\t# int" %}
7267
7268 ins_encode(aarch64_enc_ldarw(dst, mem));
7269
7270 ins_pipe(pipe_serial);
7271 %}
7272
7273 // Load Long (64 bit signed)
7274 instruct loadL_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
7275 %{
7276 match(Set dst (LoadL mem));
7277
7278 ins_cost(VOLATILE_REF_COST);
7279 format %{ "ldar $dst, $mem\t# int" %}
7280
7281 ins_encode(aarch64_enc_ldar(dst, mem));
7282
7283 ins_pipe(pipe_serial);
7284 %}
7285
7286 // Load Pointer
7287 instruct loadP_volatile(iRegPNoSp dst, /* sync_memory*/indirect mem)
7288 %{
7289 match(Set dst (LoadP mem));
7290
7291 ins_cost(VOLATILE_REF_COST);
7292 format %{ "ldar $dst, $mem\t# ptr" %}
7293
7294 ins_encode(aarch64_enc_ldar(dst, mem));
7295
7296 ins_pipe(pipe_serial);
7297 %}
7298
7299 // Load Compressed Pointer
7300 instruct loadN_volatile(iRegNNoSp dst, /* sync_memory*/indirect mem)
7301 %{
7302 match(Set dst (LoadN mem));
7303
7304 ins_cost(VOLATILE_REF_COST);
7305 format %{ "ldarw $dst, $mem\t# compressed ptr" %}
7306
7307 ins_encode(aarch64_enc_ldarw(dst, mem));
7308
7309 ins_pipe(pipe_serial);
7310 %}
7311
7312 // Load Float
7313 instruct loadF_volatile(vRegF dst, /* sync_memory*/indirect mem)
7314 %{
7315 match(Set dst (LoadF mem));
7316
7317 ins_cost(VOLATILE_REF_COST);
7318 format %{ "ldars $dst, $mem\t# float" %}
7319
7320 ins_encode( aarch64_enc_fldars(dst, mem) );
7321
7322 ins_pipe(pipe_serial);
7323 %}
7324
7325 // Load Double
7326 instruct loadD_volatile(vRegD dst, /* sync_memory*/indirect mem)
7327 %{
7328 match(Set dst (LoadD mem));
7329
7330 ins_cost(VOLATILE_REF_COST);
7331 format %{ "ldard $dst, $mem\t# double" %}
7332
7333 ins_encode( aarch64_enc_fldard(dst, mem) );
7334
7335 ins_pipe(pipe_serial);
7336 %}
7337
7338 // Store Byte
7339 instruct storeB_volatile(iRegIorL2I src, /* sync_memory*/indirect mem)
7340 %{
7341 match(Set mem (StoreB mem src));
7342
7343 ins_cost(VOLATILE_REF_COST);
7344 format %{ "stlrb $src, $mem\t# byte" %}
7345
7346 ins_encode(aarch64_enc_stlrb(src, mem));
7347
7348 ins_pipe(pipe_class_memory);
7349 %}
7350
7351 // Store Char/Short
7352 instruct storeC_volatile(iRegIorL2I src, /* sync_memory*/indirect mem)
7353 %{
7354 match(Set mem (StoreC mem src));
7355
7356 ins_cost(VOLATILE_REF_COST);
7357 format %{ "stlrh $src, $mem\t# short" %}
7358
7359 ins_encode(aarch64_enc_stlrh(src, mem));
7360
7361 ins_pipe(pipe_class_memory);
7362 %}
7363
7364 // Store Integer
7365
7366 instruct storeI_volatile(iRegIorL2I src, /* sync_memory*/indirect mem)
7367 %{
7368 match(Set mem(StoreI mem src));
7369
7370 ins_cost(VOLATILE_REF_COST);
7371 format %{ "stlrw $src, $mem\t# int" %}
7372
7373 ins_encode(aarch64_enc_stlrw(src, mem));
7374
7375 ins_pipe(pipe_class_memory);
7376 %}
7377
7378 // Store Long (64 bit signed)
7379 instruct storeL_volatile(iRegL src, /* sync_memory*/indirect mem)
7380 %{
7381 match(Set mem (StoreL mem src));
7382
7383 ins_cost(VOLATILE_REF_COST);
7384 format %{ "stlr $src, $mem\t# int" %}
7385
7386 ins_encode(aarch64_enc_stlr(src, mem));
7387
7388 ins_pipe(pipe_class_memory);
7389 %}
7390
7391 // Store Pointer
7392 instruct storeP_volatile(iRegP src, /* sync_memory*/indirect mem)
7393 %{
7394 match(Set mem (StoreP mem src));
7395
7396 ins_cost(VOLATILE_REF_COST);
7397 format %{ "stlr $src, $mem\t# ptr" %}
7398
7399 ins_encode(aarch64_enc_stlr(src, mem));
7400
7401 ins_pipe(pipe_class_memory);
7402 %}
7403
7404 // Store Compressed Pointer
7405 instruct storeN_volatile(iRegN src, /* sync_memory*/indirect mem)
7406 %{
7407 match(Set mem (StoreN mem src));
7408
7409 ins_cost(VOLATILE_REF_COST);
7410 format %{ "stlrw $src, $mem\t# compressed ptr" %}
7411
7412 ins_encode(aarch64_enc_stlrw(src, mem));
7413
7414 ins_pipe(pipe_class_memory);
7415 %}
7416
7417 // Store Float
7418 instruct storeF_volatile(vRegF src, /* sync_memory*/indirect mem)
7419 %{
7420 match(Set mem (StoreF mem src));
7421
7422 ins_cost(VOLATILE_REF_COST);
7423 format %{ "stlrs $src, $mem\t# float" %}
7424
7425 ins_encode( aarch64_enc_fstlrs(src, mem) );
7426
7427 ins_pipe(pipe_class_memory);
7428 %}
7429
7430 // TODO
7431 // implement storeImmF0 and storeFImmPacked
7432
7433 // Store Double
7434 instruct storeD_volatile(vRegD src, /* sync_memory*/indirect mem)
7435 %{
7436 match(Set mem (StoreD mem src));
7437
7438 ins_cost(VOLATILE_REF_COST);
7439 format %{ "stlrd $src, $mem\t# double" %}
7440
7441 ins_encode( aarch64_enc_fstlrd(src, mem) );
7442
7443 ins_pipe(pipe_class_memory);
7444 %}
7445
7446 // ---------------- end of volatile loads and stores ----------------
7447
7448 // ============================================================================
7449 // BSWAP Instructions
7450
7451 instruct bytes_reverse_int(iRegINoSp dst, iRegIorL2I src) %{
7452 match(Set dst (ReverseBytesI src));
7453
7454 ins_cost(INSN_COST);
7455 format %{ "revw $dst, $src" %}
7456
7457 ins_encode %{
7458 __ revw(as_Register($dst$$reg), as_Register($src$$reg));
7459 %}
7460
7461 ins_pipe(ialu_reg);
7462 %}
7463
7464 instruct bytes_reverse_long(iRegLNoSp dst, iRegL src) %{
7465 match(Set dst (ReverseBytesL src));
7466
7467 ins_cost(INSN_COST);
7468 format %{ "rev $dst, $src" %}
7469
7470 ins_encode %{
7471 __ rev(as_Register($dst$$reg), as_Register($src$$reg));
7472 %}
7473
7474 ins_pipe(ialu_reg);
7475 %}
7476
7477 instruct bytes_reverse_unsigned_short(iRegINoSp dst, iRegIorL2I src) %{
7478 match(Set dst (ReverseBytesUS src));
7479
7480 ins_cost(INSN_COST);
7481 format %{ "rev16w $dst, $src" %}
7482
7483 ins_encode %{
7484 __ rev16w(as_Register($dst$$reg), as_Register($src$$reg));
7485 %}
7486
7487 ins_pipe(ialu_reg);
7488 %}
7489
7490 instruct bytes_reverse_short(iRegINoSp dst, iRegIorL2I src) %{
7491 match(Set dst (ReverseBytesS src));
7492
7493 ins_cost(INSN_COST);
7494 format %{ "rev16w $dst, $src\n\t"
7495 "sbfmw $dst, $dst, #0, #15" %}
7496
7497 ins_encode %{
7498 __ rev16w(as_Register($dst$$reg), as_Register($src$$reg));
7499 __ sbfmw(as_Register($dst$$reg), as_Register($dst$$reg), 0U, 15U);
7500 %}
7501
7502 ins_pipe(ialu_reg);
7503 %}
7504
7505 // ============================================================================
7506 // Zero Count Instructions
7507
7508 instruct countLeadingZerosI(iRegINoSp dst, iRegIorL2I src) %{
7509 match(Set dst (CountLeadingZerosI src));
7510
7511 ins_cost(INSN_COST);
7512 format %{ "clzw $dst, $src" %}
7513 ins_encode %{
7514 __ clzw(as_Register($dst$$reg), as_Register($src$$reg));
7515 %}
7516
7517 ins_pipe(ialu_reg);
7518 %}
7519
7520 instruct countLeadingZerosL(iRegINoSp dst, iRegL src) %{
7521 match(Set dst (CountLeadingZerosL src));
7522
7523 ins_cost(INSN_COST);
7524 format %{ "clz $dst, $src" %}
7525 ins_encode %{
7526 __ clz(as_Register($dst$$reg), as_Register($src$$reg));
7527 %}
7528
7529 ins_pipe(ialu_reg);
7530 %}
7531
7532 instruct countTrailingZerosI(iRegINoSp dst, iRegIorL2I src) %{
7533 match(Set dst (CountTrailingZerosI src));
7534
7535 ins_cost(INSN_COST * 2);
7536 format %{ "rbitw $dst, $src\n\t"
7537 "clzw $dst, $dst" %}
7538 ins_encode %{
7539 __ rbitw(as_Register($dst$$reg), as_Register($src$$reg));
7540 __ clzw(as_Register($dst$$reg), as_Register($dst$$reg));
7541 %}
7542
7543 ins_pipe(ialu_reg);
7544 %}
7545
7546 instruct countTrailingZerosL(iRegINoSp dst, iRegL src) %{
7547 match(Set dst (CountTrailingZerosL src));
7548
7549 ins_cost(INSN_COST * 2);
7550 format %{ "rbit $dst, $src\n\t"
7551 "clz $dst, $dst" %}
7552 ins_encode %{
7553 __ rbit(as_Register($dst$$reg), as_Register($src$$reg));
7554 __ clz(as_Register($dst$$reg), as_Register($dst$$reg));
7555 %}
7556
7557 ins_pipe(ialu_reg);
7558 %}
7559
7560 //---------- Population Count Instructions -------------------------------------
7561 //
7562
7563 instruct popCountI(iRegINoSp dst, iRegIorL2I src, vRegF tmp) %{
7564 predicate(UsePopCountInstruction);
7565 match(Set dst (PopCountI src));
7566 effect(TEMP tmp);
7567 ins_cost(INSN_COST * 13);
7568
7569 format %{ "movw $src, $src\n\t"
7570 "mov $tmp, $src\t# vector (1D)\n\t"
7571 "cnt $tmp, $tmp\t# vector (8B)\n\t"
7572 "addv $tmp, $tmp\t# vector (8B)\n\t"
7573 "mov $dst, $tmp\t# vector (1D)" %}
7574 ins_encode %{
7575 __ movw($src$$Register, $src$$Register); // ensure top 32 bits 0
7576 __ mov($tmp$$FloatRegister, __ T1D, 0, $src$$Register);
7577 __ cnt($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
7578 __ addv($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
7579 __ mov($dst$$Register, $tmp$$FloatRegister, __ T1D, 0);
7580 %}
7581
7582 ins_pipe(pipe_class_default);
7583 %}
7584
7585 instruct popCountI_mem(iRegINoSp dst, memory mem, vRegF tmp) %{
7586 predicate(UsePopCountInstruction);
7587 match(Set dst (PopCountI (LoadI mem)));
7588 effect(TEMP tmp);
7589 ins_cost(INSN_COST * 13);
7590
7591 format %{ "ldrs $tmp, $mem\n\t"
7592 "cnt $tmp, $tmp\t# vector (8B)\n\t"
7593 "addv $tmp, $tmp\t# vector (8B)\n\t"
7594 "mov $dst, $tmp\t# vector (1D)" %}
7595 ins_encode %{
7596 FloatRegister tmp_reg = as_FloatRegister($tmp$$reg);
7597 loadStore(MacroAssembler(&cbuf), &MacroAssembler::ldrs, tmp_reg, $mem->opcode(),
7598 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
7599 __ cnt($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
7600 __ addv($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
7601 __ mov($dst$$Register, $tmp$$FloatRegister, __ T1D, 0);
7602 %}
7603
7604 ins_pipe(pipe_class_default);
7605 %}
7606
7607 // Note: Long.bitCount(long) returns an int.
7608 instruct popCountL(iRegINoSp dst, iRegL src, vRegD tmp) %{
7609 predicate(UsePopCountInstruction);
7610 match(Set dst (PopCountL src));
7611 effect(TEMP tmp);
7612 ins_cost(INSN_COST * 13);
7613
7614 format %{ "mov $tmp, $src\t# vector (1D)\n\t"
7615 "cnt $tmp, $tmp\t# vector (8B)\n\t"
7616 "addv $tmp, $tmp\t# vector (8B)\n\t"
7617 "mov $dst, $tmp\t# vector (1D)" %}
7618 ins_encode %{
7619 __ mov($tmp$$FloatRegister, __ T1D, 0, $src$$Register);
7620 __ cnt($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
7621 __ addv($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
7622 __ mov($dst$$Register, $tmp$$FloatRegister, __ T1D, 0);
7623 %}
7624
7625 ins_pipe(pipe_class_default);
7626 %}
7627
7628 instruct popCountL_mem(iRegINoSp dst, memory mem, vRegD tmp) %{
7629 predicate(UsePopCountInstruction);
7630 match(Set dst (PopCountL (LoadL mem)));
7631 effect(TEMP tmp);
7632 ins_cost(INSN_COST * 13);
7633
7634 format %{ "ldrd $tmp, $mem\n\t"
7635 "cnt $tmp, $tmp\t# vector (8B)\n\t"
7636 "addv $tmp, $tmp\t# vector (8B)\n\t"
7637 "mov $dst, $tmp\t# vector (1D)" %}
7638 ins_encode %{
7639 FloatRegister tmp_reg = as_FloatRegister($tmp$$reg);
7640 loadStore(MacroAssembler(&cbuf), &MacroAssembler::ldrd, tmp_reg, $mem->opcode(),
7641 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
7642 __ cnt($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
7643 __ addv($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
7644 __ mov($dst$$Register, $tmp$$FloatRegister, __ T1D, 0);
7645 %}
7646
7647 ins_pipe(pipe_class_default);
7648 %}
7649
7650 // ============================================================================
7651 // MemBar Instruction
7652
7653 instruct load_fence() %{
7654 match(LoadFence);
7655 ins_cost(VOLATILE_REF_COST);
7656
7657 format %{ "load_fence" %}
7658
7659 ins_encode %{
7660 __ membar(Assembler::LoadLoad|Assembler::LoadStore);
7661 %}
7662 ins_pipe(pipe_serial);
7663 %}
7664
7665 instruct unnecessary_membar_acquire() %{
7666 predicate(unnecessary_acquire(n));
7667 match(MemBarAcquire);
7668 ins_cost(0);
7669
7670 format %{ "membar_acquire (elided)" %}
7671
7672 ins_encode %{
7673 __ block_comment("membar_acquire (elided)");
7674 %}
7675
7676 ins_pipe(pipe_class_empty);
7677 %}
7678
7679 instruct membar_acquire() %{
7680 match(MemBarAcquire);
7681 ins_cost(VOLATILE_REF_COST);
7682
7683 format %{ "membar_acquire" %}
7684
7685 ins_encode %{
7686 __ block_comment("membar_acquire");
7687 __ membar(Assembler::LoadLoad|Assembler::LoadStore);
7688 %}
7689
7690 ins_pipe(pipe_serial);
7691 %}
7692
7693
7694 instruct membar_acquire_lock() %{
7695 match(MemBarAcquireLock);
7696 ins_cost(VOLATILE_REF_COST);
7697
7698 format %{ "membar_acquire_lock (elided)" %}
7699
7700 ins_encode %{
7701 __ block_comment("membar_acquire_lock (elided)");
7702 %}
7703
7704 ins_pipe(pipe_serial);
7705 %}
7706
7707 instruct store_fence() %{
7708 match(StoreFence);
7709 ins_cost(VOLATILE_REF_COST);
7710
7711 format %{ "store_fence" %}
7712
7713 ins_encode %{
7714 __ membar(Assembler::LoadStore|Assembler::StoreStore);
7715 %}
7716 ins_pipe(pipe_serial);
7717 %}
7718
7719 instruct unnecessary_membar_release() %{
7720 predicate(unnecessary_release(n));
7721 match(MemBarRelease);
7722 ins_cost(0);
7723
7724 format %{ "membar_release (elided)" %}
7725
7726 ins_encode %{
7727 __ block_comment("membar_release (elided)");
7728 %}
7729 ins_pipe(pipe_serial);
7730 %}
7731
7732 instruct membar_release() %{
7733 match(MemBarRelease);
7734 ins_cost(VOLATILE_REF_COST);
7735
7736 format %{ "membar_release" %}
7737
7738 ins_encode %{
7739 __ block_comment("membar_release");
7740 __ membar(Assembler::LoadStore|Assembler::StoreStore);
7741 %}
7742 ins_pipe(pipe_serial);
7743 %}
7744
7745 instruct membar_storestore() %{
7746 match(MemBarStoreStore);
7747 ins_cost(VOLATILE_REF_COST);
7748
7749 format %{ "MEMBAR-store-store" %}
7750
7751 ins_encode %{
7752 __ membar(Assembler::StoreStore);
7753 %}
7754 ins_pipe(pipe_serial);
7755 %}
7756
7757 instruct membar_release_lock() %{
7758 match(MemBarReleaseLock);
7759 ins_cost(VOLATILE_REF_COST);
7760
7761 format %{ "membar_release_lock (elided)" %}
7762
7763 ins_encode %{
7764 __ block_comment("membar_release_lock (elided)");
7765 %}
7766
7767 ins_pipe(pipe_serial);
7768 %}
7769
7770 instruct unnecessary_membar_volatile() %{
7771 predicate(unnecessary_volatile(n));
7772 match(MemBarVolatile);
7773 ins_cost(0);
7774
7775 format %{ "membar_volatile (elided)" %}
7776
7777 ins_encode %{
7778 __ block_comment("membar_volatile (elided)");
7779 %}
7780
7781 ins_pipe(pipe_serial);
7782 %}
7783
7784 instruct membar_volatile() %{
7785 match(MemBarVolatile);
7786 ins_cost(VOLATILE_REF_COST*100);
7787
7788 format %{ "membar_volatile" %}
7789
7790 ins_encode %{
7791 __ block_comment("membar_volatile");
7792 __ membar(Assembler::StoreLoad);
7793 %}
7794
7795 ins_pipe(pipe_serial);
7796 %}
7797
7798 // ============================================================================
7799 // Cast/Convert Instructions
7800
7801 instruct castX2P(iRegPNoSp dst, iRegL src) %{
7802 match(Set dst (CastX2P src));
7803
7804 ins_cost(INSN_COST);
7805 format %{ "mov $dst, $src\t# long -> ptr" %}
7806
7807 ins_encode %{
7808 if ($dst$$reg != $src$$reg) {
7809 __ mov(as_Register($dst$$reg), as_Register($src$$reg));
7810 }
7811 %}
7812
7813 ins_pipe(ialu_reg);
7814 %}
7815
7816 instruct castP2X(iRegLNoSp dst, iRegP src) %{
7817 match(Set dst (CastP2X src));
7818
7819 ins_cost(INSN_COST);
7820 format %{ "mov $dst, $src\t# ptr -> long" %}
7821
7822 ins_encode %{
7823 if ($dst$$reg != $src$$reg) {
7824 __ mov(as_Register($dst$$reg), as_Register($src$$reg));
7825 }
7826 %}
7827
7828 ins_pipe(ialu_reg);
7829 %}
7830
7831 // Convert oop into int for vectors alignment masking
7832 instruct convP2I(iRegINoSp dst, iRegP src) %{
7833 match(Set dst (ConvL2I (CastP2X src)));
7834
7835 ins_cost(INSN_COST);
7836 format %{ "movw $dst, $src\t# ptr -> int" %}
7837 ins_encode %{
7838 __ movw($dst$$Register, $src$$Register);
7839 %}
7840
7841 ins_pipe(ialu_reg);
7842 %}
7843
7844 // Convert compressed oop into int for vectors alignment masking
7845 // in case of 32bit oops (heap < 4Gb).
7846 instruct convN2I(iRegINoSp dst, iRegN src)
7847 %{
7848 predicate(Universe::narrow_oop_shift() == 0);
7849 match(Set dst (ConvL2I (CastP2X (DecodeN src))));
7850
7851 ins_cost(INSN_COST);
7852 format %{ "mov dst, $src\t# compressed ptr -> int" %}
7853 ins_encode %{
7854 __ movw($dst$$Register, $src$$Register);
7855 %}
7856
7857 ins_pipe(ialu_reg);
7858 %}
7859
7860
7861 // Convert oop pointer into compressed form
7862 instruct encodeHeapOop(iRegNNoSp dst, iRegP src, rFlagsReg cr) %{
7863 predicate(n->bottom_type()->make_ptr()->ptr() != TypePtr::NotNull);
7864 match(Set dst (EncodeP src));
7865 effect(KILL cr);
7866 ins_cost(INSN_COST * 3);
7867 format %{ "encode_heap_oop $dst, $src" %}
7868 ins_encode %{
7869 Register s = $src$$Register;
7870 Register d = $dst$$Register;
7871 __ encode_heap_oop(d, s);
7872 %}
7873 ins_pipe(ialu_reg);
7874 %}
7875
7876 instruct encodeHeapOop_not_null(iRegNNoSp dst, iRegP src, rFlagsReg cr) %{
7877 predicate(n->bottom_type()->make_ptr()->ptr() == TypePtr::NotNull);
7878 match(Set dst (EncodeP src));
7879 ins_cost(INSN_COST * 3);
7880 format %{ "encode_heap_oop_not_null $dst, $src" %}
7881 ins_encode %{
7882 __ encode_heap_oop_not_null($dst$$Register, $src$$Register);
7883 %}
7884 ins_pipe(ialu_reg);
7885 %}
7886
7887 instruct decodeHeapOop(iRegPNoSp dst, iRegN src, rFlagsReg cr) %{
7888 predicate(n->bottom_type()->is_ptr()->ptr() != TypePtr::NotNull &&
7889 n->bottom_type()->is_ptr()->ptr() != TypePtr::Constant);
7890 match(Set dst (DecodeN src));
7891 ins_cost(INSN_COST * 3);
7892 format %{ "decode_heap_oop $dst, $src" %}
7893 ins_encode %{
7894 Register s = $src$$Register;
7895 Register d = $dst$$Register;
7896 __ decode_heap_oop(d, s);
7897 %}
7898 ins_pipe(ialu_reg);
7899 %}
7900
7901 instruct decodeHeapOop_not_null(iRegPNoSp dst, iRegN src, rFlagsReg cr) %{
7902 predicate(n->bottom_type()->is_ptr()->ptr() == TypePtr::NotNull ||
7903 n->bottom_type()->is_ptr()->ptr() == TypePtr::Constant);
7904 match(Set dst (DecodeN src));
7905 ins_cost(INSN_COST * 3);
7906 format %{ "decode_heap_oop_not_null $dst, $src" %}
7907 ins_encode %{
7908 Register s = $src$$Register;
7909 Register d = $dst$$Register;
7910 __ decode_heap_oop_not_null(d, s);
7911 %}
7912 ins_pipe(ialu_reg);
7913 %}
7914
7915 // n.b. AArch64 implementations of encode_klass_not_null and
7916 // decode_klass_not_null do not modify the flags register so, unlike
7917 // Intel, we don't kill CR as a side effect here
7918
7919 instruct encodeKlass_not_null(iRegNNoSp dst, iRegP src) %{
7920 match(Set dst (EncodePKlass src));
7921
7922 ins_cost(INSN_COST * 3);
7923 format %{ "encode_klass_not_null $dst,$src" %}
7924
7925 ins_encode %{
7926 Register src_reg = as_Register($src$$reg);
7927 Register dst_reg = as_Register($dst$$reg);
7928 __ encode_klass_not_null(dst_reg, src_reg);
7929 %}
7930
7931 ins_pipe(ialu_reg);
7932 %}
7933
7934 instruct decodeKlass_not_null(iRegPNoSp dst, iRegN src) %{
7935 match(Set dst (DecodeNKlass src));
7936
7937 ins_cost(INSN_COST * 3);
7938 format %{ "decode_klass_not_null $dst,$src" %}
7939
7940 ins_encode %{
7941 Register src_reg = as_Register($src$$reg);
7942 Register dst_reg = as_Register($dst$$reg);
7943 if (dst_reg != src_reg) {
7944 __ decode_klass_not_null(dst_reg, src_reg);
7945 } else {
7946 __ decode_klass_not_null(dst_reg);
7947 }
7948 %}
7949
7950 ins_pipe(ialu_reg);
7951 %}
7952
7953 instruct checkCastPP(iRegPNoSp dst)
7954 %{
7955 match(Set dst (CheckCastPP dst));
7956
7957 size(0);
7958 format %{ "# checkcastPP of $dst" %}
7959 ins_encode(/* empty encoding */);
7960 ins_pipe(pipe_class_empty);
7961 %}
7962
7963 instruct castPP(iRegPNoSp dst)
7964 %{
7965 match(Set dst (CastPP dst));
7966
7967 size(0);
7968 format %{ "# castPP of $dst" %}
7969 ins_encode(/* empty encoding */);
7970 ins_pipe(pipe_class_empty);
7971 %}
7972
7973 instruct castII(iRegI dst)
7974 %{
7975 match(Set dst (CastII dst));
7976
7977 size(0);
7978 format %{ "# castII of $dst" %}
7979 ins_encode(/* empty encoding */);
7980 ins_cost(0);
7981 ins_pipe(pipe_class_empty);
7982 %}
7983
7984 // ============================================================================
7985 // Atomic operation instructions
7986 //
7987 // Intel and SPARC both implement Ideal Node LoadPLocked and
7988 // Store{PIL}Conditional instructions using a normal load for the
7989 // LoadPLocked and a CAS for the Store{PIL}Conditional.
7990 //
7991 // The ideal code appears only to use LoadPLocked/StorePLocked as a
7992 // pair to lock object allocations from Eden space when not using
7993 // TLABs.
7994 //
7995 // There does not appear to be a Load{IL}Locked Ideal Node and the
7996 // Ideal code appears to use Store{IL}Conditional as an alias for CAS
7997 // and to use StoreIConditional only for 32-bit and StoreLConditional
7998 // only for 64-bit.
7999 //
8000 // We implement LoadPLocked and StorePLocked instructions using,
8001 // respectively the AArch64 hw load-exclusive and store-conditional
8002 // instructions. Whereas we must implement each of
8003 // Store{IL}Conditional using a CAS which employs a pair of
8004 // instructions comprising a load-exclusive followed by a
8005 // store-conditional.
8006
8007
8008 // Locked-load (linked load) of the current heap-top
8009 // used when updating the eden heap top
8010 // implemented using ldaxr on AArch64
8011
8012 instruct loadPLocked(iRegPNoSp dst, indirect mem)
8013 %{
8014 match(Set dst (LoadPLocked mem));
8015
8016 ins_cost(VOLATILE_REF_COST);
8017
8018 format %{ "ldaxr $dst, $mem\t# ptr linked acquire" %}
8019
8020 ins_encode(aarch64_enc_ldaxr(dst, mem));
8021
8022 ins_pipe(pipe_serial);
8023 %}
8024
8025 // Conditional-store of the updated heap-top.
8026 // Used during allocation of the shared heap.
8027 // Sets flag (EQ) on success.
8028 // implemented using stlxr on AArch64.
8029
8030 instruct storePConditional(memory heap_top_ptr, iRegP oldval, iRegP newval, rFlagsReg cr)
8031 %{
8032 match(Set cr (StorePConditional heap_top_ptr (Binary oldval newval)));
8033
8034 ins_cost(VOLATILE_REF_COST);
8035
8036 // TODO
8037 // do we need to do a store-conditional release or can we just use a
8038 // plain store-conditional?
8039
8040 format %{
8041 "stlxr rscratch1, $newval, $heap_top_ptr\t# ptr cond release"
8042 "cmpw rscratch1, zr\t# EQ on successful write"
8043 %}
8044
8045 ins_encode(aarch64_enc_stlxr(newval, heap_top_ptr));
8046
8047 ins_pipe(pipe_serial);
8048 %}
8049
8050
8051 // storeLConditional is used by PhaseMacroExpand::expand_lock_node
8052 // when attempting to rebias a lock towards the current thread. We
8053 // must use the acquire form of cmpxchg in order to guarantee acquire
8054 // semantics in this case.
8055 instruct storeLConditional(indirect mem, iRegLNoSp oldval, iRegLNoSp newval, rFlagsReg cr)
8056 %{
8057 match(Set cr (StoreLConditional mem (Binary oldval newval)));
8058
8059 ins_cost(VOLATILE_REF_COST);
8060
8061 format %{
8062 "cmpxchg rscratch1, $mem, $oldval, $newval, $mem\t# if $mem == $oldval then $mem <-- $newval"
8063 "cmpw rscratch1, zr\t# EQ on successful write"
8064 %}
8065
8066 ins_encode(aarch64_enc_cmpxchg_acq(mem, oldval, newval));
8067
8068 ins_pipe(pipe_slow);
8069 %}
8070
8071 // storeIConditional also has acquire semantics, for no better reason
8072 // than matching storeLConditional. At the time of writing this
8073 // comment storeIConditional was not used anywhere by AArch64.
8074 instruct storeIConditional(indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr)
8075 %{
8076 match(Set cr (StoreIConditional mem (Binary oldval newval)));
8077
8078 ins_cost(VOLATILE_REF_COST);
8079
8080 format %{
8081 "cmpxchgw rscratch1, $mem, $oldval, $newval, $mem\t# if $mem == $oldval then $mem <-- $newval"
8082 "cmpw rscratch1, zr\t# EQ on successful write"
8083 %}
8084
8085 ins_encode(aarch64_enc_cmpxchgw_acq(mem, oldval, newval));
8086
8087 ins_pipe(pipe_slow);
8088 %}
8089
8090 // XXX No flag versions for CompareAndSwap{I,L,P,N} because matcher
8091 // can't match them
8092
8093 // standard CompareAndSwapX when we are using barriers
8094 // these have higher priority than the rules selected by a predicate
8095
8096 instruct compareAndSwapI(iRegINoSp res, indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr) %{
8097
8098 match(Set res (CompareAndSwapI mem (Binary oldval newval)));
8099 ins_cost(2 * VOLATILE_REF_COST);
8100
8101 effect(KILL cr);
8102
8103 format %{
8104 "cmpxchgw $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval"
8105 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8106 %}
8107
8108 ins_encode(aarch64_enc_cmpxchgw(mem, oldval, newval),
8109 aarch64_enc_cset_eq(res));
8110
8111 ins_pipe(pipe_slow);
8112 %}
8113
8114 instruct compareAndSwapL(iRegINoSp res, indirect mem, iRegLNoSp oldval, iRegLNoSp newval, rFlagsReg cr) %{
8115
8116 match(Set res (CompareAndSwapL mem (Binary oldval newval)));
8117 ins_cost(2 * VOLATILE_REF_COST);
8118
8119 effect(KILL cr);
8120
8121 format %{
8122 "cmpxchg $mem, $oldval, $newval\t# (long) if $mem == $oldval then $mem <-- $newval"
8123 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8124 %}
8125
8126 ins_encode(aarch64_enc_cmpxchg(mem, oldval, newval),
8127 aarch64_enc_cset_eq(res));
8128
8129 ins_pipe(pipe_slow);
8130 %}
8131
8132 instruct compareAndSwapP(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
8133
8134 predicate(!UseShenandoahGC || !ShenandoahCASBarrier || n->in(3)->in(1)->bottom_type() == TypePtr::NULL_PTR);
8135 match(Set res (CompareAndSwapP mem (Binary oldval newval)));
8136 ins_cost(2 * VOLATILE_REF_COST);
8137
8138 effect(KILL cr);
8139
8140 format %{
8141 "cmpxchg $mem, $oldval, $newval\t# (ptr) if $mem == $oldval then $mem <-- $newval"
8142 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8143 %}
8144
8145 ins_encode(aarch64_enc_cmpxchg(mem, oldval, newval),
8146 aarch64_enc_cset_eq(res));
8147
8148 ins_pipe(pipe_slow);
8149 %}
8150
8151 instruct compareAndSwapP_shenandoah(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval, iRegPNoSp tmp, rFlagsReg cr) %{
8152
8153 predicate(UseShenandoahGC && ShenandoahCASBarrier && n->in(3)->in(1)->bottom_type() != TypePtr::NULL_PTR);
8154 match(Set res (CompareAndSwapP mem (Binary oldval newval)));
8155 ins_cost(2 * VOLATILE_REF_COST);
8156
8157 effect(TEMP tmp, KILL cr);
8158
8159 format %{
8160 "cmpxchg_shenandoah_oop $mem, $oldval, $newval\t# (ptr) if $mem == $oldval then $mem <-- $newval with temp $tmp"
8161 %}
8162
8163 ins_encode(aarch64_enc_cmpxchg_oop_shenandoah(mem, oldval, newval, tmp, res));
8164
8165 ins_pipe(pipe_slow);
8166 %}
8167
8168 instruct compareAndSwapN(iRegINoSp res, indirect mem, iRegNNoSp oldval, iRegNNoSp newval, rFlagsReg cr) %{
8169
8170 predicate(!UseShenandoahGC || !ShenandoahCASBarrier || n->in(3)->in(1)->bottom_type() == TypeNarrowOop::NULL_PTR);
8171 match(Set res (CompareAndSwapN mem (Binary oldval newval)));
8172 ins_cost(2 * VOLATILE_REF_COST);
8173
8174 effect(KILL cr);
8175
8176 format %{
8177 "cmpxchgw $mem, $oldval, $newval\t# (narrow oop) if $mem == $oldval then $mem <-- $newval"
8178 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8179 %}
8180
8181 ins_encode(aarch64_enc_cmpxchgw(mem, oldval, newval),
8182 aarch64_enc_cset_eq(res));
8183
8184 ins_pipe(pipe_slow);
8185 %}
8186
8187 instruct compareAndSwapN_shenandoah(iRegINoSp res, indirect mem, iRegN oldval, iRegN newval, iRegNNoSp tmp, rFlagsReg cr) %{
8188
8189 predicate(UseShenandoahGC && ShenandoahCASBarrier && n->in(3)->in(1)->bottom_type() != TypeNarrowOop::NULL_PTR);
8190 match(Set res (CompareAndSwapN mem (Binary oldval newval)));
8191 ins_cost(2 * VOLATILE_REF_COST);
8192
8193 effect(TEMP tmp, KILL cr);
8194
8195 format %{
8196 "cmpxchgw_shenandoah_narrow_oop $mem, $oldval, $newval\t# (ptr) if $mem == $oldval then $mem <-- $newval with temp $tmp"
8197 %}
8198
8199 ins_encode %{
8200 Register tmp = $tmp$$Register;
8201 __ mov(tmp, $oldval$$Register); // Must not clobber oldval.
8202 ShenandoahBarrierSetAssembler::bsasm()->cmpxchg_oop(&_masm, $mem$$Register, tmp, $newval$$Register, /*acquire*/ false, /*release*/ true, /*weak*/ false, /*is_cae*/ false, $res$$Register);
8203 %}
8204
8205 ins_pipe(pipe_slow);
8206 %}
8207
8208 // alternative CompareAndSwapX when we are eliding barriers
8209
8210 instruct compareAndSwapIAcq(iRegINoSp res, indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr) %{
8211
8212 predicate(needs_acquiring_load_exclusive(n));
8213 match(Set res (CompareAndSwapI mem (Binary oldval newval)));
8214 ins_cost(VOLATILE_REF_COST);
8215
8216 effect(KILL cr);
8217
8218 format %{
8219 "cmpxchgw_acq $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval"
8220 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8221 %}
8222
8223 ins_encode(aarch64_enc_cmpxchgw_acq(mem, oldval, newval),
8224 aarch64_enc_cset_eq(res));
8225
8226 ins_pipe(pipe_slow);
8227 %}
8228
8229 instruct compareAndSwapLAcq(iRegINoSp res, indirect mem, iRegLNoSp oldval, iRegLNoSp newval, rFlagsReg cr) %{
8230
8231 predicate(needs_acquiring_load_exclusive(n));
8232 match(Set res (CompareAndSwapL mem (Binary oldval newval)));
8233 ins_cost(VOLATILE_REF_COST);
8234
8235 effect(KILL cr);
8236
8237 format %{
8238 "cmpxchg_acq $mem, $oldval, $newval\t# (long) if $mem == $oldval then $mem <-- $newval"
8239 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8240 %}
8241
8242 ins_encode(aarch64_enc_cmpxchg_acq(mem, oldval, newval),
8243 aarch64_enc_cset_eq(res));
8244
8245 ins_pipe(pipe_slow);
8246 %}
8247
8248 instruct compareAndSwapPAcq(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
8249
8250 predicate(needs_acquiring_load_exclusive(n) && (!UseShenandoahGC || !ShenandoahCASBarrier || n->in(3)->in(1)->bottom_type() == TypePtr::NULL_PTR));
8251 match(Set res (CompareAndSwapP mem (Binary oldval newval)));
8252 ins_cost(VOLATILE_REF_COST);
8253
8254 effect(KILL cr);
8255
8256 format %{
8257 "cmpxchg_acq $mem, $oldval, $newval\t# (ptr) if $mem == $oldval then $mem <-- $newval"
8258 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8259 %}
8260
8261 ins_encode(aarch64_enc_cmpxchg_acq(mem, oldval, newval),
8262 aarch64_enc_cset_eq(res));
8263
8264 ins_pipe(pipe_slow);
8265 %}
8266
8267 instruct compareAndSwapPAcq_shenandoah(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval, iRegPNoSp tmp, rFlagsReg cr) %{
8268
8269 predicate(needs_acquiring_load_exclusive(n) && UseShenandoahGC && ShenandoahCASBarrier && n->in(3)->in(1)->bottom_type() != TypePtr::NULL_PTR);
8270 match(Set res (CompareAndSwapP mem (Binary oldval newval)));
8271 ins_cost(VOLATILE_REF_COST);
8272
8273 effect(TEMP tmp, KILL cr);
8274
8275 format %{
8276 "cmpxchg_acq_shenandoah_oop $mem, $oldval, $newval\t# (ptr) if $mem == $oldval then $mem <-- $newval with temp $tmp"
8277 %}
8278
8279 ins_encode(aarch64_enc_cmpxchg_acq_oop_shenandoah(mem, oldval, newval, tmp, res));
8280
8281 ins_pipe(pipe_slow);
8282 %}
8283
8284 instruct compareAndSwapNAcq(iRegINoSp res, indirect mem, iRegNNoSp oldval, iRegNNoSp newval, rFlagsReg cr) %{
8285
8286 predicate(needs_acquiring_load_exclusive(n) && (!UseShenandoahGC || !ShenandoahCASBarrier|| n->in(3)->in(1)->bottom_type() == TypeNarrowOop::NULL_PTR));
8287 match(Set res (CompareAndSwapN mem (Binary oldval newval)));
8288 ins_cost(VOLATILE_REF_COST);
8289
8290 effect(KILL cr);
8291
8292 format %{
8293 "cmpxchgw_acq $mem, $oldval, $newval\t# (narrow oop) if $mem == $oldval then $mem <-- $newval"
8294 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8295 %}
8296
8297 ins_encode(aarch64_enc_cmpxchgw_acq(mem, oldval, newval),
8298 aarch64_enc_cset_eq(res));
8299
8300 ins_pipe(pipe_slow);
8301 %}
8302
8303 instruct compareAndSwapNAcq_shenandoah(iRegINoSp res, indirect mem, iRegN oldval, iRegN newval, iRegNNoSp tmp, rFlagsReg cr) %{
8304
8305 predicate(needs_acquiring_load_exclusive(n) && UseShenandoahGC && ShenandoahCASBarrier && n->in(3)->in(1)->bottom_type() != TypeNarrowOop::NULL_PTR);
8306 match(Set res (CompareAndSwapN mem (Binary oldval newval)));
8307 ins_cost(VOLATILE_REF_COST);
8308
8309 effect(TEMP tmp, KILL cr);
8310
8311 format %{
8312 "cmpxchgw_acq_shenandoah_narrow_oop $mem, $oldval, $newval\t# (ptr) if $mem == $oldval then $mem <-- $newval with temp $tmp"
8313 %}
8314
8315 ins_encode %{
8316 Register tmp = $tmp$$Register;
8317 __ mov(tmp, $oldval$$Register); // Must not clobber oldval.
8318 ShenandoahBarrierSetAssembler::bsasm()->cmpxchg_oop(&_masm, $mem$$Register, tmp, $newval$$Register, /*acquire*/ true, /*release*/ true, /*weak*/ false, /*is_cae*/ false, $res$$Register);
8319 %}
8320
8321 ins_pipe(pipe_slow);
8322 %}
8323
8324 instruct get_and_setI(indirect mem, iRegI newv, iRegINoSp prev) %{
8325 match(Set prev (GetAndSetI mem newv));
8326 ins_cost(2 * VOLATILE_REF_COST);
8327 format %{ "atomic_xchgw $prev, $newv, [$mem]" %}
8328 ins_encode %{
8329 __ atomic_xchgw($prev$$Register, $newv$$Register, as_Register($mem$$base));
8330 %}
8331 ins_pipe(pipe_serial);
8332 %}
8333
8334 instruct get_and_setL(indirect mem, iRegL newv, iRegLNoSp prev) %{
8335 match(Set prev (GetAndSetL mem newv));
8336 ins_cost(2 * VOLATILE_REF_COST);
8337 format %{ "atomic_xchg $prev, $newv, [$mem]" %}
8338 ins_encode %{
8339 __ atomic_xchg($prev$$Register, $newv$$Register, as_Register($mem$$base));
8340 %}
8341 ins_pipe(pipe_serial);
8342 %}
8343
8344 instruct get_and_setN(indirect mem, iRegN newv, iRegINoSp prev) %{
8345 match(Set prev (GetAndSetN mem newv));
8346 ins_cost(2 * VOLATILE_REF_COST);
8347 format %{ "atomic_xchgw $prev, $newv, [$mem]" %}
8348 ins_encode %{
8349 __ atomic_xchgw($prev$$Register, $newv$$Register, as_Register($mem$$base));
8350 %}
8351 ins_pipe(pipe_serial);
8352 %}
8353
8354 instruct get_and_setP(indirect mem, iRegP newv, iRegPNoSp prev) %{
8355 match(Set prev (GetAndSetP mem newv));
8356 ins_cost(2 * VOLATILE_REF_COST);
8357 format %{ "atomic_xchg $prev, $newv, [$mem]" %}
8358 ins_encode %{
8359 __ atomic_xchg($prev$$Register, $newv$$Register, as_Register($mem$$base));
8360 %}
8361 ins_pipe(pipe_serial);
8362 %}
8363
8364 instruct get_and_setIAcq(indirect mem, iRegI newv, iRegINoSp prev) %{
8365 predicate(needs_acquiring_load_exclusive(n));
8366 match(Set prev (GetAndSetI mem newv));
8367 ins_cost(VOLATILE_REF_COST);
8368 format %{ "atomic_xchgw_acq $prev, $newv, [$mem]" %}
8369 ins_encode %{
8370 __ atomic_xchgalw($prev$$Register, $newv$$Register, as_Register($mem$$base));
8371 %}
8372 ins_pipe(pipe_serial);
8373 %}
8374
8375 instruct get_and_setLAcq(indirect mem, iRegL newv, iRegLNoSp prev) %{
8376 predicate(needs_acquiring_load_exclusive(n));
8377 match(Set prev (GetAndSetL mem newv));
8378 ins_cost(VOLATILE_REF_COST);
8379 format %{ "atomic_xchg_acq $prev, $newv, [$mem]" %}
8380 ins_encode %{
8381 __ atomic_xchgal($prev$$Register, $newv$$Register, as_Register($mem$$base));
8382 %}
8383 ins_pipe(pipe_serial);
8384 %}
8385
8386 instruct get_and_setNAcq(indirect mem, iRegN newv, iRegINoSp prev) %{
8387 predicate(needs_acquiring_load_exclusive(n));
8388 match(Set prev (GetAndSetN mem newv));
8389 ins_cost(VOLATILE_REF_COST);
8390 format %{ "atomic_xchgw_acq $prev, $newv, [$mem]" %}
8391 ins_encode %{
8392 __ atomic_xchgalw($prev$$Register, $newv$$Register, as_Register($mem$$base));
8393 %}
8394 ins_pipe(pipe_serial);
8395 %}
8396
8397 instruct get_and_setPAcq(indirect mem, iRegP newv, iRegPNoSp prev) %{
8398 predicate(needs_acquiring_load_exclusive(n));
8399 match(Set prev (GetAndSetP mem newv));
8400 ins_cost(VOLATILE_REF_COST);
8401 format %{ "atomic_xchg_acq $prev, $newv, [$mem]" %}
8402 ins_encode %{
8403 __ atomic_xchgal($prev$$Register, $newv$$Register, as_Register($mem$$base));
8404 %}
8405 ins_pipe(pipe_serial);
8406 %}
8407
8408
8409 instruct get_and_addL(indirect mem, iRegLNoSp newval, iRegL incr) %{
8410 match(Set newval (GetAndAddL mem incr));
8411 ins_cost(2 * VOLATILE_REF_COST + 1);
8412 format %{ "get_and_addL $newval, [$mem], $incr" %}
8413 ins_encode %{
8414 __ atomic_add($newval$$Register, $incr$$Register, as_Register($mem$$base));
8415 %}
8416 ins_pipe(pipe_serial);
8417 %}
8418
8419 instruct get_and_addL_no_res(indirect mem, Universe dummy, iRegL incr) %{
8420 predicate(n->as_LoadStore()->result_not_used());
8421 match(Set dummy (GetAndAddL mem incr));
8422 ins_cost(2 * VOLATILE_REF_COST);
8423 format %{ "get_and_addL [$mem], $incr" %}
8424 ins_encode %{
8425 __ atomic_add(noreg, $incr$$Register, as_Register($mem$$base));
8426 %}
8427 ins_pipe(pipe_serial);
8428 %}
8429
8430 instruct get_and_addLi(indirect mem, iRegLNoSp newval, immLAddSub incr) %{
8431 match(Set newval (GetAndAddL mem incr));
8432 ins_cost(2 * VOLATILE_REF_COST + 1);
8433 format %{ "get_and_addL $newval, [$mem], $incr" %}
8434 ins_encode %{
8435 __ atomic_add($newval$$Register, $incr$$constant, as_Register($mem$$base));
8436 %}
8437 ins_pipe(pipe_serial);
8438 %}
8439
8440 instruct get_and_addLi_no_res(indirect mem, Universe dummy, immLAddSub incr) %{
8441 predicate(n->as_LoadStore()->result_not_used());
8442 match(Set dummy (GetAndAddL mem incr));
8443 ins_cost(2 * VOLATILE_REF_COST);
8444 format %{ "get_and_addL [$mem], $incr" %}
8445 ins_encode %{
8446 __ atomic_add(noreg, $incr$$constant, as_Register($mem$$base));
8447 %}
8448 ins_pipe(pipe_serial);
8449 %}
8450
8451 instruct get_and_addI(indirect mem, iRegINoSp newval, iRegIorL2I incr) %{
8452 match(Set newval (GetAndAddI mem incr));
8453 ins_cost(2 * VOLATILE_REF_COST + 1);
8454 format %{ "get_and_addI $newval, [$mem], $incr" %}
8455 ins_encode %{
8456 __ atomic_addw($newval$$Register, $incr$$Register, as_Register($mem$$base));
8457 %}
8458 ins_pipe(pipe_serial);
8459 %}
8460
8461 instruct get_and_addI_no_res(indirect mem, Universe dummy, iRegIorL2I incr) %{
8462 predicate(n->as_LoadStore()->result_not_used());
8463 match(Set dummy (GetAndAddI mem incr));
8464 ins_cost(2 * VOLATILE_REF_COST);
8465 format %{ "get_and_addI [$mem], $incr" %}
8466 ins_encode %{
8467 __ atomic_addw(noreg, $incr$$Register, as_Register($mem$$base));
8468 %}
8469 ins_pipe(pipe_serial);
8470 %}
8471
8472 instruct get_and_addIi(indirect mem, iRegINoSp newval, immIAddSub incr) %{
8473 match(Set newval (GetAndAddI mem incr));
8474 ins_cost(2 * VOLATILE_REF_COST + 1);
8475 format %{ "get_and_addI $newval, [$mem], $incr" %}
8476 ins_encode %{
8477 __ atomic_addw($newval$$Register, $incr$$constant, as_Register($mem$$base));
8478 %}
8479 ins_pipe(pipe_serial);
8480 %}
8481
8482 instruct get_and_addIi_no_res(indirect mem, Universe dummy, immIAddSub incr) %{
8483 predicate(n->as_LoadStore()->result_not_used());
8484 match(Set dummy (GetAndAddI mem incr));
8485 ins_cost(2 * VOLATILE_REF_COST);
8486 format %{ "get_and_addI [$mem], $incr" %}
8487 ins_encode %{
8488 __ atomic_addw(noreg, $incr$$constant, as_Register($mem$$base));
8489 %}
8490 ins_pipe(pipe_serial);
8491 %}
8492
8493 instruct get_and_addLAcq(indirect mem, iRegLNoSp newval, iRegL incr) %{
8494 predicate(needs_acquiring_load_exclusive(n));
8495 match(Set newval (GetAndAddL mem incr));
8496 ins_cost(VOLATILE_REF_COST + 1);
8497 format %{ "get_and_addL_acq $newval, [$mem], $incr" %}
8498 ins_encode %{
8499 __ atomic_addal($newval$$Register, $incr$$Register, as_Register($mem$$base));
8500 %}
8501 ins_pipe(pipe_serial);
8502 %}
8503
8504 instruct get_and_addL_no_resAcq(indirect mem, Universe dummy, iRegL incr) %{
8505 predicate(n->as_LoadStore()->result_not_used() && needs_acquiring_load_exclusive(n));
8506 match(Set dummy (GetAndAddL mem incr));
8507 ins_cost(VOLATILE_REF_COST);
8508 format %{ "get_and_addL_acq [$mem], $incr" %}
8509 ins_encode %{
8510 __ atomic_addal(noreg, $incr$$Register, as_Register($mem$$base));
8511 %}
8512 ins_pipe(pipe_serial);
8513 %}
8514
8515 instruct get_and_addLiAcq(indirect mem, iRegLNoSp newval, immLAddSub incr) %{
8516 predicate(needs_acquiring_load_exclusive(n));
8517 match(Set newval (GetAndAddL mem incr));
8518 ins_cost(VOLATILE_REF_COST + 1);
8519 format %{ "get_and_addL_acq $newval, [$mem], $incr" %}
8520 ins_encode %{
8521 __ atomic_addal($newval$$Register, $incr$$constant, as_Register($mem$$base));
8522 %}
8523 ins_pipe(pipe_serial);
8524 %}
8525
8526 instruct get_and_addLi_no_resAcq(indirect mem, Universe dummy, immLAddSub incr) %{
8527 predicate(n->as_LoadStore()->result_not_used() && needs_acquiring_load_exclusive(n));
8528 match(Set dummy (GetAndAddL mem incr));
8529 ins_cost(VOLATILE_REF_COST);
8530 format %{ "get_and_addL_acq [$mem], $incr" %}
8531 ins_encode %{
8532 __ atomic_addal(noreg, $incr$$constant, as_Register($mem$$base));
8533 %}
8534 ins_pipe(pipe_serial);
8535 %}
8536
8537 instruct get_and_addIAcq(indirect mem, iRegINoSp newval, iRegIorL2I incr) %{
8538 predicate(needs_acquiring_load_exclusive(n));
8539 match(Set newval (GetAndAddI mem incr));
8540 ins_cost(VOLATILE_REF_COST + 1);
8541 format %{ "get_and_addI_acq $newval, [$mem], $incr" %}
8542 ins_encode %{
8543 __ atomic_addalw($newval$$Register, $incr$$Register, as_Register($mem$$base));
8544 %}
8545 ins_pipe(pipe_serial);
8546 %}
8547
8548 instruct get_and_addI_no_resAcq(indirect mem, Universe dummy, iRegIorL2I incr) %{
8549 predicate(n->as_LoadStore()->result_not_used() && needs_acquiring_load_exclusive(n));
8550 match(Set dummy (GetAndAddI mem incr));
8551 ins_cost(VOLATILE_REF_COST);
8552 format %{ "get_and_addI_acq [$mem], $incr" %}
8553 ins_encode %{
8554 __ atomic_addalw(noreg, $incr$$Register, as_Register($mem$$base));
8555 %}
8556 ins_pipe(pipe_serial);
8557 %}
8558
8559 instruct get_and_addIiAcq(indirect mem, iRegINoSp newval, immIAddSub incr) %{
8560 predicate(needs_acquiring_load_exclusive(n));
8561 match(Set newval (GetAndAddI mem incr));
8562 ins_cost(VOLATILE_REF_COST + 1);
8563 format %{ "get_and_addI_acq $newval, [$mem], $incr" %}
8564 ins_encode %{
8565 __ atomic_addalw($newval$$Register, $incr$$constant, as_Register($mem$$base));
8566 %}
8567 ins_pipe(pipe_serial);
8568 %}
8569
8570 instruct get_and_addIi_no_resAcq(indirect mem, Universe dummy, immIAddSub incr) %{
8571 predicate(n->as_LoadStore()->result_not_used() && needs_acquiring_load_exclusive(n));
8572 match(Set dummy (GetAndAddI mem incr));
8573 ins_cost(VOLATILE_REF_COST);
8574 format %{ "get_and_addI_acq [$mem], $incr" %}
8575 ins_encode %{
8576 __ atomic_addalw(noreg, $incr$$constant, as_Register($mem$$base));
8577 %}
8578 ins_pipe(pipe_serial);
8579 %}
8580
8581 // ============================================================================
8582 // Conditional Move Instructions
8583
8584 // n.b. we have identical rules for both a signed compare op (cmpOp)
8585 // and an unsigned compare op (cmpOpU). it would be nice if we could
8586 // define an op class which merged both inputs and use it to type the
8587 // argument to a single rule. unfortunatelyt his fails because the
8588 // opclass does not live up to the COND_INTER interface of its
8589 // component operands. When the generic code tries to negate the
8590 // operand it ends up running the generci Machoper::negate method
8591 // which throws a ShouldNotHappen. So, we have to provide two flavours
8592 // of each rule, one for a cmpOp and a second for a cmpOpU (sigh).
8593
8594 instruct cmovI_reg_reg(cmpOp cmp, rFlagsReg cr, iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
8595 match(Set dst (CMoveI (Binary cmp cr) (Binary src1 src2)));
8596
8597 ins_cost(INSN_COST * 2);
8598 format %{ "cselw $dst, $src2, $src1 $cmp\t# signed, int" %}
8599
8600 ins_encode %{
8601 __ cselw(as_Register($dst$$reg),
8602 as_Register($src2$$reg),
8603 as_Register($src1$$reg),
8604 (Assembler::Condition)$cmp$$cmpcode);
8605 %}
8606
8607 ins_pipe(icond_reg_reg);
8608 %}
8609
8610 instruct cmovUI_reg_reg(cmpOpU cmp, rFlagsRegU cr, iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
8611 match(Set dst (CMoveI (Binary cmp cr) (Binary src1 src2)));
8612
8613 ins_cost(INSN_COST * 2);
8614 format %{ "cselw $dst, $src2, $src1 $cmp\t# unsigned, int" %}
8615
8616 ins_encode %{
8617 __ cselw(as_Register($dst$$reg),
8618 as_Register($src2$$reg),
8619 as_Register($src1$$reg),
8620 (Assembler::Condition)$cmp$$cmpcode);
8621 %}
8622
8623 ins_pipe(icond_reg_reg);
8624 %}
8625
8626 // special cases where one arg is zero
8627
8628 // n.b. this is selected in preference to the rule above because it
8629 // avoids loading constant 0 into a source register
8630
8631 // TODO
8632 // we ought only to be able to cull one of these variants as the ideal
8633 // transforms ought always to order the zero consistently (to left/right?)
8634
8635 instruct cmovI_zero_reg(cmpOp cmp, rFlagsReg cr, iRegINoSp dst, immI0 zero, iRegIorL2I src) %{
8636 match(Set dst (CMoveI (Binary cmp cr) (Binary zero src)));
8637
8638 ins_cost(INSN_COST * 2);
8639 format %{ "cselw $dst, $src, zr $cmp\t# signed, int" %}
8640
8641 ins_encode %{
8642 __ cselw(as_Register($dst$$reg),
8643 as_Register($src$$reg),
8644 zr,
8645 (Assembler::Condition)$cmp$$cmpcode);
8646 %}
8647
8648 ins_pipe(icond_reg);
8649 %}
8650
8651 instruct cmovUI_zero_reg(cmpOpU cmp, rFlagsRegU cr, iRegINoSp dst, immI0 zero, iRegIorL2I src) %{
8652 match(Set dst (CMoveI (Binary cmp cr) (Binary zero src)));
8653
8654 ins_cost(INSN_COST * 2);
8655 format %{ "cselw $dst, $src, zr $cmp\t# unsigned, int" %}
8656
8657 ins_encode %{
8658 __ cselw(as_Register($dst$$reg),
8659 as_Register($src$$reg),
8660 zr,
8661 (Assembler::Condition)$cmp$$cmpcode);
8662 %}
8663
8664 ins_pipe(icond_reg);
8665 %}
8666
8667 instruct cmovI_reg_zero(cmpOp cmp, rFlagsReg cr, iRegINoSp dst, iRegIorL2I src, immI0 zero) %{
8668 match(Set dst (CMoveI (Binary cmp cr) (Binary src zero)));
8669
8670 ins_cost(INSN_COST * 2);
8671 format %{ "cselw $dst, zr, $src $cmp\t# signed, int" %}
8672
8673 ins_encode %{
8674 __ cselw(as_Register($dst$$reg),
8675 zr,
8676 as_Register($src$$reg),
8677 (Assembler::Condition)$cmp$$cmpcode);
8678 %}
8679
8680 ins_pipe(icond_reg);
8681 %}
8682
8683 instruct cmovUI_reg_zero(cmpOpU cmp, rFlagsRegU cr, iRegINoSp dst, iRegIorL2I src, immI0 zero) %{
8684 match(Set dst (CMoveI (Binary cmp cr) (Binary src zero)));
8685
8686 ins_cost(INSN_COST * 2);
8687 format %{ "cselw $dst, zr, $src $cmp\t# unsigned, int" %}
8688
8689 ins_encode %{
8690 __ cselw(as_Register($dst$$reg),
8691 zr,
8692 as_Register($src$$reg),
8693 (Assembler::Condition)$cmp$$cmpcode);
8694 %}
8695
8696 ins_pipe(icond_reg);
8697 %}
8698
8699 // special case for creating a boolean 0 or 1
8700
8701 // n.b. this is selected in preference to the rule above because it
8702 // avoids loading constants 0 and 1 into a source register
8703
8704 instruct cmovI_reg_zero_one(cmpOp cmp, rFlagsReg cr, iRegINoSp dst, immI0 zero, immI_1 one) %{
8705 match(Set dst (CMoveI (Binary cmp cr) (Binary one zero)));
8706
8707 ins_cost(INSN_COST * 2);
8708 format %{ "csincw $dst, zr, zr $cmp\t# signed, int" %}
8709
8710 ins_encode %{
8711 // equivalently
8712 // cset(as_Register($dst$$reg),
8713 // negate_condition((Assembler::Condition)$cmp$$cmpcode));
8714 __ csincw(as_Register($dst$$reg),
8715 zr,
8716 zr,
8717 (Assembler::Condition)$cmp$$cmpcode);
8718 %}
8719
8720 ins_pipe(icond_none);
8721 %}
8722
8723 instruct cmovUI_reg_zero_one(cmpOpU cmp, rFlagsRegU cr, iRegINoSp dst, immI0 zero, immI_1 one) %{
8724 match(Set dst (CMoveI (Binary cmp cr) (Binary one zero)));
8725
8726 ins_cost(INSN_COST * 2);
8727 format %{ "csincw $dst, zr, zr $cmp\t# unsigned, int" %}
8728
8729 ins_encode %{
8730 // equivalently
8731 // cset(as_Register($dst$$reg),
8732 // negate_condition((Assembler::Condition)$cmp$$cmpcode));
8733 __ csincw(as_Register($dst$$reg),
8734 zr,
8735 zr,
8736 (Assembler::Condition)$cmp$$cmpcode);
8737 %}
8738
8739 ins_pipe(icond_none);
8740 %}
8741
8742 instruct cmovL_reg_reg(cmpOp cmp, rFlagsReg cr, iRegLNoSp dst, iRegL src1, iRegL src2) %{
8743 match(Set dst (CMoveL (Binary cmp cr) (Binary src1 src2)));
8744
8745 ins_cost(INSN_COST * 2);
8746 format %{ "csel $dst, $src2, $src1 $cmp\t# signed, long" %}
8747
8748 ins_encode %{
8749 __ csel(as_Register($dst$$reg),
8750 as_Register($src2$$reg),
8751 as_Register($src1$$reg),
8752 (Assembler::Condition)$cmp$$cmpcode);
8753 %}
8754
8755 ins_pipe(icond_reg_reg);
8756 %}
8757
8758 instruct cmovUL_reg_reg(cmpOpU cmp, rFlagsRegU cr, iRegLNoSp dst, iRegL src1, iRegL src2) %{
8759 match(Set dst (CMoveL (Binary cmp cr) (Binary src1 src2)));
8760
8761 ins_cost(INSN_COST * 2);
8762 format %{ "csel $dst, $src2, $src1 $cmp\t# unsigned, long" %}
8763
8764 ins_encode %{
8765 __ csel(as_Register($dst$$reg),
8766 as_Register($src2$$reg),
8767 as_Register($src1$$reg),
8768 (Assembler::Condition)$cmp$$cmpcode);
8769 %}
8770
8771 ins_pipe(icond_reg_reg);
8772 %}
8773
8774 // special cases where one arg is zero
8775
8776 instruct cmovL_reg_zero(cmpOp cmp, rFlagsReg cr, iRegLNoSp dst, iRegL src, immL0 zero) %{
8777 match(Set dst (CMoveL (Binary cmp cr) (Binary src zero)));
8778
8779 ins_cost(INSN_COST * 2);
8780 format %{ "csel $dst, zr, $src $cmp\t# signed, long" %}
8781
8782 ins_encode %{
8783 __ csel(as_Register($dst$$reg),
8784 zr,
8785 as_Register($src$$reg),
8786 (Assembler::Condition)$cmp$$cmpcode);
8787 %}
8788
8789 ins_pipe(icond_reg);
8790 %}
8791
8792 instruct cmovUL_reg_zero(cmpOpU cmp, rFlagsRegU cr, iRegLNoSp dst, iRegL src, immL0 zero) %{
8793 match(Set dst (CMoveL (Binary cmp cr) (Binary src zero)));
8794
8795 ins_cost(INSN_COST * 2);
8796 format %{ "csel $dst, zr, $src $cmp\t# unsigned, long" %}
8797
8798 ins_encode %{
8799 __ csel(as_Register($dst$$reg),
8800 zr,
8801 as_Register($src$$reg),
8802 (Assembler::Condition)$cmp$$cmpcode);
8803 %}
8804
8805 ins_pipe(icond_reg);
8806 %}
8807
8808 instruct cmovL_zero_reg(cmpOp cmp, rFlagsReg cr, iRegLNoSp dst, immL0 zero, iRegL src) %{
8809 match(Set dst (CMoveL (Binary cmp cr) (Binary zero src)));
8810
8811 ins_cost(INSN_COST * 2);
8812 format %{ "csel $dst, $src, zr $cmp\t# signed, long" %}
8813
8814 ins_encode %{
8815 __ csel(as_Register($dst$$reg),
8816 as_Register($src$$reg),
8817 zr,
8818 (Assembler::Condition)$cmp$$cmpcode);
8819 %}
8820
8821 ins_pipe(icond_reg);
8822 %}
8823
8824 instruct cmovUL_zero_reg(cmpOpU cmp, rFlagsRegU cr, iRegLNoSp dst, immL0 zero, iRegL src) %{
8825 match(Set dst (CMoveL (Binary cmp cr) (Binary zero src)));
8826
8827 ins_cost(INSN_COST * 2);
8828 format %{ "csel $dst, $src, zr $cmp\t# unsigned, long" %}
8829
8830 ins_encode %{
8831 __ csel(as_Register($dst$$reg),
8832 as_Register($src$$reg),
8833 zr,
8834 (Assembler::Condition)$cmp$$cmpcode);
8835 %}
8836
8837 ins_pipe(icond_reg);
8838 %}
8839
8840 instruct cmovP_reg_reg(cmpOp cmp, rFlagsReg cr, iRegPNoSp dst, iRegP src1, iRegP src2) %{
8841 match(Set dst (CMoveP (Binary cmp cr) (Binary src1 src2)));
8842
8843 ins_cost(INSN_COST * 2);
8844 format %{ "csel $dst, $src2, $src1 $cmp\t# signed, ptr" %}
8845
8846 ins_encode %{
8847 __ csel(as_Register($dst$$reg),
8848 as_Register($src2$$reg),
8849 as_Register($src1$$reg),
8850 (Assembler::Condition)$cmp$$cmpcode);
8851 %}
8852
8853 ins_pipe(icond_reg_reg);
8854 %}
8855
8856 instruct cmovUP_reg_reg(cmpOpU cmp, rFlagsRegU cr, iRegPNoSp dst, iRegP src1, iRegP src2) %{
8857 match(Set dst (CMoveP (Binary cmp cr) (Binary src1 src2)));
8858
8859 ins_cost(INSN_COST * 2);
8860 format %{ "csel $dst, $src2, $src1 $cmp\t# unsigned, ptr" %}
8861
8862 ins_encode %{
8863 __ csel(as_Register($dst$$reg),
8864 as_Register($src2$$reg),
8865 as_Register($src1$$reg),
8866 (Assembler::Condition)$cmp$$cmpcode);
8867 %}
8868
8869 ins_pipe(icond_reg_reg);
8870 %}
8871
8872 // special cases where one arg is zero
8873
8874 instruct cmovP_reg_zero(cmpOp cmp, rFlagsReg cr, iRegPNoSp dst, iRegP src, immP0 zero) %{
8875 match(Set dst (CMoveP (Binary cmp cr) (Binary src zero)));
8876
8877 ins_cost(INSN_COST * 2);
8878 format %{ "csel $dst, zr, $src $cmp\t# signed, ptr" %}
8879
8880 ins_encode %{
8881 __ csel(as_Register($dst$$reg),
8882 zr,
8883 as_Register($src$$reg),
8884 (Assembler::Condition)$cmp$$cmpcode);
8885 %}
8886
8887 ins_pipe(icond_reg);
8888 %}
8889
8890 instruct cmovUP_reg_zero(cmpOpU cmp, rFlagsRegU cr, iRegPNoSp dst, iRegP src, immP0 zero) %{
8891 match(Set dst (CMoveP (Binary cmp cr) (Binary src zero)));
8892
8893 ins_cost(INSN_COST * 2);
8894 format %{ "csel $dst, zr, $src $cmp\t# unsigned, ptr" %}
8895
8896 ins_encode %{
8897 __ csel(as_Register($dst$$reg),
8898 zr,
8899 as_Register($src$$reg),
8900 (Assembler::Condition)$cmp$$cmpcode);
8901 %}
8902
8903 ins_pipe(icond_reg);
8904 %}
8905
8906 instruct cmovP_zero_reg(cmpOp cmp, rFlagsReg cr, iRegPNoSp dst, immP0 zero, iRegP src) %{
8907 match(Set dst (CMoveP (Binary cmp cr) (Binary zero src)));
8908
8909 ins_cost(INSN_COST * 2);
8910 format %{ "csel $dst, $src, zr $cmp\t# signed, ptr" %}
8911
8912 ins_encode %{
8913 __ csel(as_Register($dst$$reg),
8914 as_Register($src$$reg),
8915 zr,
8916 (Assembler::Condition)$cmp$$cmpcode);
8917 %}
8918
8919 ins_pipe(icond_reg);
8920 %}
8921
8922 instruct cmovUP_zero_reg(cmpOpU cmp, rFlagsRegU cr, iRegPNoSp dst, immP0 zero, iRegP src) %{
8923 match(Set dst (CMoveP (Binary cmp cr) (Binary zero src)));
8924
8925 ins_cost(INSN_COST * 2);
8926 format %{ "csel $dst, $src, zr $cmp\t# unsigned, ptr" %}
8927
8928 ins_encode %{
8929 __ csel(as_Register($dst$$reg),
8930 as_Register($src$$reg),
8931 zr,
8932 (Assembler::Condition)$cmp$$cmpcode);
8933 %}
8934
8935 ins_pipe(icond_reg);
8936 %}
8937
8938 instruct cmovN_reg_reg(cmpOp cmp, rFlagsReg cr, iRegNNoSp dst, iRegN src1, iRegN src2) %{
8939 match(Set dst (CMoveN (Binary cmp cr) (Binary src1 src2)));
8940
8941 ins_cost(INSN_COST * 2);
8942 format %{ "cselw $dst, $src2, $src1 $cmp\t# signed, compressed ptr" %}
8943
8944 ins_encode %{
8945 __ cselw(as_Register($dst$$reg),
8946 as_Register($src2$$reg),
8947 as_Register($src1$$reg),
8948 (Assembler::Condition)$cmp$$cmpcode);
8949 %}
8950
8951 ins_pipe(icond_reg_reg);
8952 %}
8953
8954 instruct cmovUN_reg_reg(cmpOpU cmp, rFlagsRegU cr, iRegNNoSp dst, iRegN src1, iRegN src2) %{
8955 match(Set dst (CMoveN (Binary cmp cr) (Binary src1 src2)));
8956
8957 ins_cost(INSN_COST * 2);
8958 format %{ "cselw $dst, $src2, $src1 $cmp\t# signed, compressed ptr" %}
8959
8960 ins_encode %{
8961 __ cselw(as_Register($dst$$reg),
8962 as_Register($src2$$reg),
8963 as_Register($src1$$reg),
8964 (Assembler::Condition)$cmp$$cmpcode);
8965 %}
8966
8967 ins_pipe(icond_reg_reg);
8968 %}
8969
8970 // special cases where one arg is zero
8971
8972 instruct cmovN_reg_zero(cmpOp cmp, rFlagsReg cr, iRegNNoSp dst, iRegN src, immN0 zero) %{
8973 match(Set dst (CMoveN (Binary cmp cr) (Binary src zero)));
8974
8975 ins_cost(INSN_COST * 2);
8976 format %{ "cselw $dst, zr, $src $cmp\t# signed, compressed ptr" %}
8977
8978 ins_encode %{
8979 __ cselw(as_Register($dst$$reg),
8980 zr,
8981 as_Register($src$$reg),
8982 (Assembler::Condition)$cmp$$cmpcode);
8983 %}
8984
8985 ins_pipe(icond_reg);
8986 %}
8987
8988 instruct cmovUN_reg_zero(cmpOpU cmp, rFlagsRegU cr, iRegNNoSp dst, iRegN src, immN0 zero) %{
8989 match(Set dst (CMoveN (Binary cmp cr) (Binary src zero)));
8990
8991 ins_cost(INSN_COST * 2);
8992 format %{ "cselw $dst, zr, $src $cmp\t# unsigned, compressed ptr" %}
8993
8994 ins_encode %{
8995 __ cselw(as_Register($dst$$reg),
8996 zr,
8997 as_Register($src$$reg),
8998 (Assembler::Condition)$cmp$$cmpcode);
8999 %}
9000
9001 ins_pipe(icond_reg);
9002 %}
9003
9004 instruct cmovN_zero_reg(cmpOp cmp, rFlagsReg cr, iRegNNoSp dst, immN0 zero, iRegN src) %{
9005 match(Set dst (CMoveN (Binary cmp cr) (Binary zero src)));
9006
9007 ins_cost(INSN_COST * 2);
9008 format %{ "cselw $dst, $src, zr $cmp\t# signed, compressed ptr" %}
9009
9010 ins_encode %{
9011 __ cselw(as_Register($dst$$reg),
9012 as_Register($src$$reg),
9013 zr,
9014 (Assembler::Condition)$cmp$$cmpcode);
9015 %}
9016
9017 ins_pipe(icond_reg);
9018 %}
9019
9020 instruct cmovUN_zero_reg(cmpOpU cmp, rFlagsRegU cr, iRegNNoSp dst, immN0 zero, iRegN src) %{
9021 match(Set dst (CMoveN (Binary cmp cr) (Binary zero src)));
9022
9023 ins_cost(INSN_COST * 2);
9024 format %{ "cselw $dst, $src, zr $cmp\t# unsigned, compressed ptr" %}
9025
9026 ins_encode %{
9027 __ cselw(as_Register($dst$$reg),
9028 as_Register($src$$reg),
9029 zr,
9030 (Assembler::Condition)$cmp$$cmpcode);
9031 %}
9032
9033 ins_pipe(icond_reg);
9034 %}
9035
9036 instruct cmovF_reg(cmpOp cmp, rFlagsReg cr, vRegF dst, vRegF src1, vRegF src2)
9037 %{
9038 match(Set dst (CMoveF (Binary cmp cr) (Binary src1 src2)));
9039
9040 ins_cost(INSN_COST * 3);
9041
9042 format %{ "fcsels $dst, $src1, $src2, $cmp\t# signed cmove float\n\t" %}
9043 ins_encode %{
9044 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
9045 __ fcsels(as_FloatRegister($dst$$reg),
9046 as_FloatRegister($src2$$reg),
9047 as_FloatRegister($src1$$reg),
9048 cond);
9049 %}
9050
9051 ins_pipe(fp_cond_reg_reg_s);
9052 %}
9053
9054 instruct cmovUF_reg(cmpOpU cmp, rFlagsRegU cr, vRegF dst, vRegF src1, vRegF src2)
9055 %{
9056 match(Set dst (CMoveF (Binary cmp cr) (Binary src1 src2)));
9057
9058 ins_cost(INSN_COST * 3);
9059
9060 format %{ "fcsels $dst, $src1, $src2, $cmp\t# unsigned cmove float\n\t" %}
9061 ins_encode %{
9062 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
9063 __ fcsels(as_FloatRegister($dst$$reg),
9064 as_FloatRegister($src2$$reg),
9065 as_FloatRegister($src1$$reg),
9066 cond);
9067 %}
9068
9069 ins_pipe(fp_cond_reg_reg_s);
9070 %}
9071
9072 instruct cmovD_reg(cmpOp cmp, rFlagsReg cr, vRegD dst, vRegD src1, vRegD src2)
9073 %{
9074 match(Set dst (CMoveD (Binary cmp cr) (Binary src1 src2)));
9075
9076 ins_cost(INSN_COST * 3);
9077
9078 format %{ "fcseld $dst, $src1, $src2, $cmp\t# signed cmove float\n\t" %}
9079 ins_encode %{
9080 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
9081 __ fcseld(as_FloatRegister($dst$$reg),
9082 as_FloatRegister($src2$$reg),
9083 as_FloatRegister($src1$$reg),
9084 cond);
9085 %}
9086
9087 ins_pipe(fp_cond_reg_reg_d);
9088 %}
9089
9090 instruct cmovUD_reg(cmpOpU cmp, rFlagsRegU cr, vRegD dst, vRegD src1, vRegD src2)
9091 %{
9092 match(Set dst (CMoveD (Binary cmp cr) (Binary src1 src2)));
9093
9094 ins_cost(INSN_COST * 3);
9095
9096 format %{ "fcseld $dst, $src1, $src2, $cmp\t# unsigned cmove float\n\t" %}
9097 ins_encode %{
9098 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
9099 __ fcseld(as_FloatRegister($dst$$reg),
9100 as_FloatRegister($src2$$reg),
9101 as_FloatRegister($src1$$reg),
9102 cond);
9103 %}
9104
9105 ins_pipe(fp_cond_reg_reg_d);
9106 %}
9107
9108 // ============================================================================
9109 // Arithmetic Instructions
9110 //
9111
9112 // Integer Addition
9113
9114 // TODO
9115 // these currently employ operations which do not set CR and hence are
9116 // not flagged as killing CR but we would like to isolate the cases
9117 // where we want to set flags from those where we don't. need to work
9118 // out how to do that.
9119
9120 instruct addI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
9121 match(Set dst (AddI src1 src2));
9122
9123 ins_cost(INSN_COST);
9124 format %{ "addw $dst, $src1, $src2" %}
9125
9126 ins_encode %{
9127 __ addw(as_Register($dst$$reg),
9128 as_Register($src1$$reg),
9129 as_Register($src2$$reg));
9130 %}
9131
9132 ins_pipe(ialu_reg_reg);
9133 %}
9134
9135 instruct addI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immIAddSub src2) %{
9136 match(Set dst (AddI src1 src2));
9137
9138 ins_cost(INSN_COST);
9139 format %{ "addw $dst, $src1, $src2" %}
9140
9141 // use opcode to indicate that this is an add not a sub
9142 opcode(0x0);
9143
9144 ins_encode(aarch64_enc_addsubw_imm(dst, src1, src2));
9145
9146 ins_pipe(ialu_reg_imm);
9147 %}
9148
9149 instruct addI_reg_imm_i2l(iRegINoSp dst, iRegL src1, immIAddSub src2) %{
9150 match(Set dst (AddI (ConvL2I src1) src2));
9151
9152 ins_cost(INSN_COST);
9153 format %{ "addw $dst, $src1, $src2" %}
9154
9155 // use opcode to indicate that this is an add not a sub
9156 opcode(0x0);
9157
9158 ins_encode(aarch64_enc_addsubw_imm(dst, src1, src2));
9159
9160 ins_pipe(ialu_reg_imm);
9161 %}
9162
9163 // Pointer Addition
9164 instruct addP_reg_reg(iRegPNoSp dst, iRegP src1, iRegL src2) %{
9165 match(Set dst (AddP src1 src2));
9166
9167 ins_cost(INSN_COST);
9168 format %{ "add $dst, $src1, $src2\t# ptr" %}
9169
9170 ins_encode %{
9171 __ add(as_Register($dst$$reg),
9172 as_Register($src1$$reg),
9173 as_Register($src2$$reg));
9174 %}
9175
9176 ins_pipe(ialu_reg_reg);
9177 %}
9178
9179 instruct addP_reg_reg_ext(iRegPNoSp dst, iRegP src1, iRegIorL2I src2) %{
9180 match(Set dst (AddP src1 (ConvI2L src2)));
9181
9182 ins_cost(1.9 * INSN_COST);
9183 format %{ "add $dst, $src1, $src2, sxtw\t# ptr" %}
9184
9185 ins_encode %{
9186 __ add(as_Register($dst$$reg),
9187 as_Register($src1$$reg),
9188 as_Register($src2$$reg), ext::sxtw);
9189 %}
9190
9191 ins_pipe(ialu_reg_reg);
9192 %}
9193
9194 instruct addP_reg_reg_lsl(iRegPNoSp dst, iRegP src1, iRegL src2, immIScale scale) %{
9195 match(Set dst (AddP src1 (LShiftL src2 scale)));
9196
9197 ins_cost(1.9 * INSN_COST);
9198 format %{ "add $dst, $src1, $src2, LShiftL $scale\t# ptr" %}
9199
9200 ins_encode %{
9201 __ lea(as_Register($dst$$reg),
9202 Address(as_Register($src1$$reg), as_Register($src2$$reg),
9203 Address::lsl($scale$$constant)));
9204 %}
9205
9206 ins_pipe(ialu_reg_reg_shift);
9207 %}
9208
9209 instruct addP_reg_reg_ext_shift(iRegPNoSp dst, iRegP src1, iRegIorL2I src2, immIScale scale) %{
9210 match(Set dst (AddP src1 (LShiftL (ConvI2L src2) scale)));
9211
9212 ins_cost(1.9 * INSN_COST);
9213 format %{ "add $dst, $src1, $src2, I2L $scale\t# ptr" %}
9214
9215 ins_encode %{
9216 __ lea(as_Register($dst$$reg),
9217 Address(as_Register($src1$$reg), as_Register($src2$$reg),
9218 Address::sxtw($scale$$constant)));
9219 %}
9220
9221 ins_pipe(ialu_reg_reg_shift);
9222 %}
9223
9224 instruct lshift_ext(iRegLNoSp dst, iRegIorL2I src, immI scale, rFlagsReg cr) %{
9225 match(Set dst (LShiftL (ConvI2L src) scale));
9226
9227 ins_cost(INSN_COST);
9228 format %{ "sbfiz $dst, $src, $scale & 63, -$scale & 63\t" %}
9229
9230 ins_encode %{
9231 __ sbfiz(as_Register($dst$$reg),
9232 as_Register($src$$reg),
9233 $scale$$constant & 63, MIN(32, (-$scale$$constant) & 63));
9234 %}
9235
9236 ins_pipe(ialu_reg_shift);
9237 %}
9238
9239 // Pointer Immediate Addition
9240 // n.b. this needs to be more expensive than using an indirect memory
9241 // operand
9242 instruct addP_reg_imm(iRegPNoSp dst, iRegP src1, immLAddSub src2) %{
9243 match(Set dst (AddP src1 src2));
9244
9245 ins_cost(INSN_COST);
9246 format %{ "add $dst, $src1, $src2\t# ptr" %}
9247
9248 // use opcode to indicate that this is an add not a sub
9249 opcode(0x0);
9250
9251 ins_encode( aarch64_enc_addsub_imm(dst, src1, src2) );
9252
9253 ins_pipe(ialu_reg_imm);
9254 %}
9255
9256 // Long Addition
9257 instruct addL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
9258
9259 match(Set dst (AddL src1 src2));
9260
9261 ins_cost(INSN_COST);
9262 format %{ "add $dst, $src1, $src2" %}
9263
9264 ins_encode %{
9265 __ add(as_Register($dst$$reg),
9266 as_Register($src1$$reg),
9267 as_Register($src2$$reg));
9268 %}
9269
9270 ins_pipe(ialu_reg_reg);
9271 %}
9272
9273 // No constant pool entries requiredLong Immediate Addition.
9274 instruct addL_reg_imm(iRegLNoSp dst, iRegL src1, immLAddSub src2) %{
9275 match(Set dst (AddL src1 src2));
9276
9277 ins_cost(INSN_COST);
9278 format %{ "add $dst, $src1, $src2" %}
9279
9280 // use opcode to indicate that this is an add not a sub
9281 opcode(0x0);
9282
9283 ins_encode( aarch64_enc_addsub_imm(dst, src1, src2) );
9284
9285 ins_pipe(ialu_reg_imm);
9286 %}
9287
9288 // Integer Subtraction
9289 instruct subI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
9290 match(Set dst (SubI src1 src2));
9291
9292 ins_cost(INSN_COST);
9293 format %{ "subw $dst, $src1, $src2" %}
9294
9295 ins_encode %{
9296 __ subw(as_Register($dst$$reg),
9297 as_Register($src1$$reg),
9298 as_Register($src2$$reg));
9299 %}
9300
9301 ins_pipe(ialu_reg_reg);
9302 %}
9303
9304 // Immediate Subtraction
9305 instruct subI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immIAddSub src2) %{
9306 match(Set dst (SubI src1 src2));
9307
9308 ins_cost(INSN_COST);
9309 format %{ "subw $dst, $src1, $src2" %}
9310
9311 // use opcode to indicate that this is a sub not an add
9312 opcode(0x1);
9313
9314 ins_encode(aarch64_enc_addsubw_imm(dst, src1, src2));
9315
9316 ins_pipe(ialu_reg_imm);
9317 %}
9318
9319 // Long Subtraction
9320 instruct subL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
9321
9322 match(Set dst (SubL src1 src2));
9323
9324 ins_cost(INSN_COST);
9325 format %{ "sub $dst, $src1, $src2" %}
9326
9327 ins_encode %{
9328 __ sub(as_Register($dst$$reg),
9329 as_Register($src1$$reg),
9330 as_Register($src2$$reg));
9331 %}
9332
9333 ins_pipe(ialu_reg_reg);
9334 %}
9335
9336 // No constant pool entries requiredLong Immediate Subtraction.
9337 instruct subL_reg_imm(iRegLNoSp dst, iRegL src1, immLAddSub src2) %{
9338 match(Set dst (SubL src1 src2));
9339
9340 ins_cost(INSN_COST);
9341 format %{ "sub$dst, $src1, $src2" %}
9342
9343 // use opcode to indicate that this is a sub not an add
9344 opcode(0x1);
9345
9346 ins_encode( aarch64_enc_addsub_imm(dst, src1, src2) );
9347
9348 ins_pipe(ialu_reg_imm);
9349 %}
9350
9351 // Integer Negation (special case for sub)
9352
9353 instruct negI_reg(iRegINoSp dst, iRegIorL2I src, immI0 zero, rFlagsReg cr) %{
9354 match(Set dst (SubI zero src));
9355
9356 ins_cost(INSN_COST);
9357 format %{ "negw $dst, $src\t# int" %}
9358
9359 ins_encode %{
9360 __ negw(as_Register($dst$$reg),
9361 as_Register($src$$reg));
9362 %}
9363
9364 ins_pipe(ialu_reg);
9365 %}
9366
9367 // Long Negation
9368
9369 instruct negL_reg(iRegLNoSp dst, iRegIorL2I src, immL0 zero, rFlagsReg cr) %{
9370 match(Set dst (SubL zero src));
9371
9372 ins_cost(INSN_COST);
9373 format %{ "neg $dst, $src\t# long" %}
9374
9375 ins_encode %{
9376 __ neg(as_Register($dst$$reg),
9377 as_Register($src$$reg));
9378 %}
9379
9380 ins_pipe(ialu_reg);
9381 %}
9382
9383 // Integer Multiply
9384
9385 instruct mulI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
9386 match(Set dst (MulI src1 src2));
9387
9388 ins_cost(INSN_COST * 3);
9389 format %{ "mulw $dst, $src1, $src2" %}
9390
9391 ins_encode %{
9392 __ mulw(as_Register($dst$$reg),
9393 as_Register($src1$$reg),
9394 as_Register($src2$$reg));
9395 %}
9396
9397 ins_pipe(imul_reg_reg);
9398 %}
9399
9400 instruct smulI(iRegLNoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
9401 match(Set dst (MulL (ConvI2L src1) (ConvI2L src2)));
9402
9403 ins_cost(INSN_COST * 3);
9404 format %{ "smull $dst, $src1, $src2" %}
9405
9406 ins_encode %{
9407 __ smull(as_Register($dst$$reg),
9408 as_Register($src1$$reg),
9409 as_Register($src2$$reg));
9410 %}
9411
9412 ins_pipe(imul_reg_reg);
9413 %}
9414
9415 // Long Multiply
9416
9417 instruct mulL(iRegLNoSp dst, iRegL src1, iRegL src2) %{
9418 match(Set dst (MulL src1 src2));
9419
9420 ins_cost(INSN_COST * 5);
9421 format %{ "mul $dst, $src1, $src2" %}
9422
9423 ins_encode %{
9424 __ mul(as_Register($dst$$reg),
9425 as_Register($src1$$reg),
9426 as_Register($src2$$reg));
9427 %}
9428
9429 ins_pipe(lmul_reg_reg);
9430 %}
9431
9432 instruct mulHiL_rReg(iRegLNoSp dst, iRegL src1, iRegL src2, rFlagsReg cr)
9433 %{
9434 match(Set dst (MulHiL src1 src2));
9435
9436 ins_cost(INSN_COST * 7);
9437 format %{ "smulh $dst, $src1, $src2, \t# mulhi" %}
9438
9439 ins_encode %{
9440 __ smulh(as_Register($dst$$reg),
9441 as_Register($src1$$reg),
9442 as_Register($src2$$reg));
9443 %}
9444
9445 ins_pipe(lmul_reg_reg);
9446 %}
9447
9448 // Combined Integer Multiply & Add/Sub
9449
9450 instruct maddI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegIorL2I src3) %{
9451 match(Set dst (AddI src3 (MulI src1 src2)));
9452
9453 ins_cost(INSN_COST * 3);
9454 format %{ "madd $dst, $src1, $src2, $src3" %}
9455
9456 ins_encode %{
9457 __ maddw(as_Register($dst$$reg),
9458 as_Register($src1$$reg),
9459 as_Register($src2$$reg),
9460 as_Register($src3$$reg));
9461 %}
9462
9463 ins_pipe(imac_reg_reg);
9464 %}
9465
9466 instruct msubI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegIorL2I src3) %{
9467 match(Set dst (SubI src3 (MulI src1 src2)));
9468
9469 ins_cost(INSN_COST * 3);
9470 format %{ "msub $dst, $src1, $src2, $src3" %}
9471
9472 ins_encode %{
9473 __ msubw(as_Register($dst$$reg),
9474 as_Register($src1$$reg),
9475 as_Register($src2$$reg),
9476 as_Register($src3$$reg));
9477 %}
9478
9479 ins_pipe(imac_reg_reg);
9480 %}
9481
9482 // Combined Long Multiply & Add/Sub
9483
9484 instruct maddL(iRegLNoSp dst, iRegL src1, iRegL src2, iRegL src3) %{
9485 match(Set dst (AddL src3 (MulL src1 src2)));
9486
9487 ins_cost(INSN_COST * 5);
9488 format %{ "madd $dst, $src1, $src2, $src3" %}
9489
9490 ins_encode %{
9491 __ madd(as_Register($dst$$reg),
9492 as_Register($src1$$reg),
9493 as_Register($src2$$reg),
9494 as_Register($src3$$reg));
9495 %}
9496
9497 ins_pipe(lmac_reg_reg);
9498 %}
9499
9500 instruct msubL(iRegLNoSp dst, iRegL src1, iRegL src2, iRegL src3) %{
9501 match(Set dst (SubL src3 (MulL src1 src2)));
9502
9503 ins_cost(INSN_COST * 5);
9504 format %{ "msub $dst, $src1, $src2, $src3" %}
9505
9506 ins_encode %{
9507 __ msub(as_Register($dst$$reg),
9508 as_Register($src1$$reg),
9509 as_Register($src2$$reg),
9510 as_Register($src3$$reg));
9511 %}
9512
9513 ins_pipe(lmac_reg_reg);
9514 %}
9515
9516 // Integer Divide
9517
9518 instruct divI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
9519 match(Set dst (DivI src1 src2));
9520
9521 ins_cost(INSN_COST * 19);
9522 format %{ "sdivw $dst, $src1, $src2" %}
9523
9524 ins_encode(aarch64_enc_divw(dst, src1, src2));
9525 ins_pipe(idiv_reg_reg);
9526 %}
9527
9528 instruct signExtract(iRegINoSp dst, iRegIorL2I src1, immI_31 div1, immI_31 div2) %{
9529 match(Set dst (URShiftI (RShiftI src1 div1) div2));
9530 ins_cost(INSN_COST);
9531 format %{ "lsrw $dst, $src1, $div1" %}
9532 ins_encode %{
9533 __ lsrw(as_Register($dst$$reg), as_Register($src1$$reg), 31);
9534 %}
9535 ins_pipe(ialu_reg_shift);
9536 %}
9537
9538 instruct div2Round(iRegINoSp dst, iRegIorL2I src, immI_31 div1, immI_31 div2) %{
9539 match(Set dst (AddI src (URShiftI (RShiftI src div1) div2)));
9540 ins_cost(INSN_COST);
9541 format %{ "addw $dst, $src, LSR $div1" %}
9542
9543 ins_encode %{
9544 __ addw(as_Register($dst$$reg),
9545 as_Register($src$$reg),
9546 as_Register($src$$reg),
9547 Assembler::LSR, 31);
9548 %}
9549 ins_pipe(ialu_reg);
9550 %}
9551
9552 // Long Divide
9553
9554 instruct divL(iRegLNoSp dst, iRegL src1, iRegL src2) %{
9555 match(Set dst (DivL src1 src2));
9556
9557 ins_cost(INSN_COST * 35);
9558 format %{ "sdiv $dst, $src1, $src2" %}
9559
9560 ins_encode(aarch64_enc_div(dst, src1, src2));
9561 ins_pipe(ldiv_reg_reg);
9562 %}
9563
9564 instruct signExtractL(iRegLNoSp dst, iRegL src1, immL_63 div1, immL_63 div2) %{
9565 match(Set dst (URShiftL (RShiftL src1 div1) div2));
9566 ins_cost(INSN_COST);
9567 format %{ "lsr $dst, $src1, $div1" %}
9568 ins_encode %{
9569 __ lsr(as_Register($dst$$reg), as_Register($src1$$reg), 63);
9570 %}
9571 ins_pipe(ialu_reg_shift);
9572 %}
9573
9574 instruct div2RoundL(iRegLNoSp dst, iRegL src, immL_63 div1, immL_63 div2) %{
9575 match(Set dst (AddL src (URShiftL (RShiftL src div1) div2)));
9576 ins_cost(INSN_COST);
9577 format %{ "add $dst, $src, $div1" %}
9578
9579 ins_encode %{
9580 __ add(as_Register($dst$$reg),
9581 as_Register($src$$reg),
9582 as_Register($src$$reg),
9583 Assembler::LSR, 63);
9584 %}
9585 ins_pipe(ialu_reg);
9586 %}
9587
9588 // Integer Remainder
9589
9590 instruct modI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
9591 match(Set dst (ModI src1 src2));
9592
9593 ins_cost(INSN_COST * 22);
9594 format %{ "sdivw rscratch1, $src1, $src2\n\t"
9595 "msubw($dst, rscratch1, $src2, $src1" %}
9596
9597 ins_encode(aarch64_enc_modw(dst, src1, src2));
9598 ins_pipe(idiv_reg_reg);
9599 %}
9600
9601 // Long Remainder
9602
9603 instruct modL(iRegLNoSp dst, iRegL src1, iRegL src2) %{
9604 match(Set dst (ModL src1 src2));
9605
9606 ins_cost(INSN_COST * 38);
9607 format %{ "sdiv rscratch1, $src1, $src2\n"
9608 "msub($dst, rscratch1, $src2, $src1" %}
9609
9610 ins_encode(aarch64_enc_mod(dst, src1, src2));
9611 ins_pipe(ldiv_reg_reg);
9612 %}
9613
9614 // Integer Shifts
9615
9616 // Shift Left Register
9617 instruct lShiftI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
9618 match(Set dst (LShiftI src1 src2));
9619
9620 ins_cost(INSN_COST * 2);
9621 format %{ "lslvw $dst, $src1, $src2" %}
9622
9623 ins_encode %{
9624 __ lslvw(as_Register($dst$$reg),
9625 as_Register($src1$$reg),
9626 as_Register($src2$$reg));
9627 %}
9628
9629 ins_pipe(ialu_reg_reg_vshift);
9630 %}
9631
9632 // Shift Left Immediate
9633 instruct lShiftI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immI src2) %{
9634 match(Set dst (LShiftI src1 src2));
9635
9636 ins_cost(INSN_COST);
9637 format %{ "lslw $dst, $src1, ($src2 & 0x1f)" %}
9638
9639 ins_encode %{
9640 __ lslw(as_Register($dst$$reg),
9641 as_Register($src1$$reg),
9642 $src2$$constant & 0x1f);
9643 %}
9644
9645 ins_pipe(ialu_reg_shift);
9646 %}
9647
9648 // Shift Right Logical Register
9649 instruct urShiftI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
9650 match(Set dst (URShiftI src1 src2));
9651
9652 ins_cost(INSN_COST * 2);
9653 format %{ "lsrvw $dst, $src1, $src2" %}
9654
9655 ins_encode %{
9656 __ lsrvw(as_Register($dst$$reg),
9657 as_Register($src1$$reg),
9658 as_Register($src2$$reg));
9659 %}
9660
9661 ins_pipe(ialu_reg_reg_vshift);
9662 %}
9663
9664 // Shift Right Logical Immediate
9665 instruct urShiftI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immI src2) %{
9666 match(Set dst (URShiftI src1 src2));
9667
9668 ins_cost(INSN_COST);
9669 format %{ "lsrw $dst, $src1, ($src2 & 0x1f)" %}
9670
9671 ins_encode %{
9672 __ lsrw(as_Register($dst$$reg),
9673 as_Register($src1$$reg),
9674 $src2$$constant & 0x1f);
9675 %}
9676
9677 ins_pipe(ialu_reg_shift);
9678 %}
9679
9680 // Shift Right Arithmetic Register
9681 instruct rShiftI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
9682 match(Set dst (RShiftI src1 src2));
9683
9684 ins_cost(INSN_COST * 2);
9685 format %{ "asrvw $dst, $src1, $src2" %}
9686
9687 ins_encode %{
9688 __ asrvw(as_Register($dst$$reg),
9689 as_Register($src1$$reg),
9690 as_Register($src2$$reg));
9691 %}
9692
9693 ins_pipe(ialu_reg_reg_vshift);
9694 %}
9695
9696 // Shift Right Arithmetic Immediate
9697 instruct rShiftI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immI src2) %{
9698 match(Set dst (RShiftI src1 src2));
9699
9700 ins_cost(INSN_COST);
9701 format %{ "asrw $dst, $src1, ($src2 & 0x1f)" %}
9702
9703 ins_encode %{
9704 __ asrw(as_Register($dst$$reg),
9705 as_Register($src1$$reg),
9706 $src2$$constant & 0x1f);
9707 %}
9708
9709 ins_pipe(ialu_reg_shift);
9710 %}
9711
9712 // Combined Int Mask and Right Shift (using UBFM)
9713 // TODO
9714
9715 // Long Shifts
9716
9717 // Shift Left Register
9718 instruct lShiftL_reg_reg(iRegLNoSp dst, iRegL src1, iRegIorL2I src2) %{
9719 match(Set dst (LShiftL src1 src2));
9720
9721 ins_cost(INSN_COST * 2);
9722 format %{ "lslv $dst, $src1, $src2" %}
9723
9724 ins_encode %{
9725 __ lslv(as_Register($dst$$reg),
9726 as_Register($src1$$reg),
9727 as_Register($src2$$reg));
9728 %}
9729
9730 ins_pipe(ialu_reg_reg_vshift);
9731 %}
9732
9733 // Shift Left Immediate
9734 instruct lShiftL_reg_imm(iRegLNoSp dst, iRegL src1, immI src2) %{
9735 match(Set dst (LShiftL src1 src2));
9736
9737 ins_cost(INSN_COST);
9738 format %{ "lsl $dst, $src1, ($src2 & 0x3f)" %}
9739
9740 ins_encode %{
9741 __ lsl(as_Register($dst$$reg),
9742 as_Register($src1$$reg),
9743 $src2$$constant & 0x3f);
9744 %}
9745
9746 ins_pipe(ialu_reg_shift);
9747 %}
9748
9749 // Shift Right Logical Register
9750 instruct urShiftL_reg_reg(iRegLNoSp dst, iRegL src1, iRegIorL2I src2) %{
9751 match(Set dst (URShiftL src1 src2));
9752
9753 ins_cost(INSN_COST * 2);
9754 format %{ "lsrv $dst, $src1, $src2" %}
9755
9756 ins_encode %{
9757 __ lsrv(as_Register($dst$$reg),
9758 as_Register($src1$$reg),
9759 as_Register($src2$$reg));
9760 %}
9761
9762 ins_pipe(ialu_reg_reg_vshift);
9763 %}
9764
9765 // Shift Right Logical Immediate
9766 instruct urShiftL_reg_imm(iRegLNoSp dst, iRegL src1, immI src2) %{
9767 match(Set dst (URShiftL src1 src2));
9768
9769 ins_cost(INSN_COST);
9770 format %{ "lsr $dst, $src1, ($src2 & 0x3f)" %}
9771
9772 ins_encode %{
9773 __ lsr(as_Register($dst$$reg),
9774 as_Register($src1$$reg),
9775 $src2$$constant & 0x3f);
9776 %}
9777
9778 ins_pipe(ialu_reg_shift);
9779 %}
9780
9781 // A special-case pattern for card table stores.
9782 instruct urShiftP_reg_imm(iRegLNoSp dst, iRegP src1, immI src2) %{
9783 match(Set dst (URShiftL (CastP2X src1) src2));
9784
9785 ins_cost(INSN_COST);
9786 format %{ "lsr $dst, p2x($src1), ($src2 & 0x3f)" %}
9787
9788 ins_encode %{
9789 __ lsr(as_Register($dst$$reg),
9790 as_Register($src1$$reg),
9791 $src2$$constant & 0x3f);
9792 %}
9793
9794 ins_pipe(ialu_reg_shift);
9795 %}
9796
9797 // Shift Right Arithmetic Register
9798 instruct rShiftL_reg_reg(iRegLNoSp dst, iRegL src1, iRegIorL2I src2) %{
9799 match(Set dst (RShiftL src1 src2));
9800
9801 ins_cost(INSN_COST * 2);
9802 format %{ "asrv $dst, $src1, $src2" %}
9803
9804 ins_encode %{
9805 __ asrv(as_Register($dst$$reg),
9806 as_Register($src1$$reg),
9807 as_Register($src2$$reg));
9808 %}
9809
9810 ins_pipe(ialu_reg_reg_vshift);
9811 %}
9812
9813 // Shift Right Arithmetic Immediate
9814 instruct rShiftL_reg_imm(iRegLNoSp dst, iRegL src1, immI src2) %{
9815 match(Set dst (RShiftL src1 src2));
9816
9817 ins_cost(INSN_COST);
9818 format %{ "asr $dst, $src1, ($src2 & 0x3f)" %}
9819
9820 ins_encode %{
9821 __ asr(as_Register($dst$$reg),
9822 as_Register($src1$$reg),
9823 $src2$$constant & 0x3f);
9824 %}
9825
9826 ins_pipe(ialu_reg_shift);
9827 %}
9828
9829 // BEGIN This section of the file is automatically generated. Do not edit --------------
9830
9831 instruct regL_not_reg(iRegLNoSp dst,
9832 iRegL src1, immL_M1 m1,
9833 rFlagsReg cr) %{
9834 match(Set dst (XorL src1 m1));
9835 ins_cost(INSN_COST);
9836 format %{ "eon $dst, $src1, zr" %}
9837
9838 ins_encode %{
9839 __ eon(as_Register($dst$$reg),
9840 as_Register($src1$$reg),
9841 zr,
9842 Assembler::LSL, 0);
9843 %}
9844
9845 ins_pipe(ialu_reg);
9846 %}
9847 instruct regI_not_reg(iRegINoSp dst,
9848 iRegIorL2I src1, immI_M1 m1,
9849 rFlagsReg cr) %{
9850 match(Set dst (XorI src1 m1));
9851 ins_cost(INSN_COST);
9852 format %{ "eonw $dst, $src1, zr" %}
9853
9854 ins_encode %{
9855 __ eonw(as_Register($dst$$reg),
9856 as_Register($src1$$reg),
9857 zr,
9858 Assembler::LSL, 0);
9859 %}
9860
9861 ins_pipe(ialu_reg);
9862 %}
9863
9864 instruct AndI_reg_not_reg(iRegINoSp dst,
9865 iRegIorL2I src1, iRegIorL2I src2, immI_M1 m1,
9866 rFlagsReg cr) %{
9867 match(Set dst (AndI src1 (XorI src2 m1)));
9868 ins_cost(INSN_COST);
9869 format %{ "bicw $dst, $src1, $src2" %}
9870
9871 ins_encode %{
9872 __ bicw(as_Register($dst$$reg),
9873 as_Register($src1$$reg),
9874 as_Register($src2$$reg),
9875 Assembler::LSL, 0);
9876 %}
9877
9878 ins_pipe(ialu_reg_reg);
9879 %}
9880
9881 instruct AndL_reg_not_reg(iRegLNoSp dst,
9882 iRegL src1, iRegL src2, immL_M1 m1,
9883 rFlagsReg cr) %{
9884 match(Set dst (AndL src1 (XorL src2 m1)));
9885 ins_cost(INSN_COST);
9886 format %{ "bic $dst, $src1, $src2" %}
9887
9888 ins_encode %{
9889 __ bic(as_Register($dst$$reg),
9890 as_Register($src1$$reg),
9891 as_Register($src2$$reg),
9892 Assembler::LSL, 0);
9893 %}
9894
9895 ins_pipe(ialu_reg_reg);
9896 %}
9897
9898 instruct OrI_reg_not_reg(iRegINoSp dst,
9899 iRegIorL2I src1, iRegIorL2I src2, immI_M1 m1,
9900 rFlagsReg cr) %{
9901 match(Set dst (OrI src1 (XorI src2 m1)));
9902 ins_cost(INSN_COST);
9903 format %{ "ornw $dst, $src1, $src2" %}
9904
9905 ins_encode %{
9906 __ ornw(as_Register($dst$$reg),
9907 as_Register($src1$$reg),
9908 as_Register($src2$$reg),
9909 Assembler::LSL, 0);
9910 %}
9911
9912 ins_pipe(ialu_reg_reg);
9913 %}
9914
9915 instruct OrL_reg_not_reg(iRegLNoSp dst,
9916 iRegL src1, iRegL src2, immL_M1 m1,
9917 rFlagsReg cr) %{
9918 match(Set dst (OrL src1 (XorL src2 m1)));
9919 ins_cost(INSN_COST);
9920 format %{ "orn $dst, $src1, $src2" %}
9921
9922 ins_encode %{
9923 __ orn(as_Register($dst$$reg),
9924 as_Register($src1$$reg),
9925 as_Register($src2$$reg),
9926 Assembler::LSL, 0);
9927 %}
9928
9929 ins_pipe(ialu_reg_reg);
9930 %}
9931
9932 instruct XorI_reg_not_reg(iRegINoSp dst,
9933 iRegIorL2I src1, iRegIorL2I src2, immI_M1 m1,
9934 rFlagsReg cr) %{
9935 match(Set dst (XorI m1 (XorI src2 src1)));
9936 ins_cost(INSN_COST);
9937 format %{ "eonw $dst, $src1, $src2" %}
9938
9939 ins_encode %{
9940 __ eonw(as_Register($dst$$reg),
9941 as_Register($src1$$reg),
9942 as_Register($src2$$reg),
9943 Assembler::LSL, 0);
9944 %}
9945
9946 ins_pipe(ialu_reg_reg);
9947 %}
9948
9949 instruct XorL_reg_not_reg(iRegLNoSp dst,
9950 iRegL src1, iRegL src2, immL_M1 m1,
9951 rFlagsReg cr) %{
9952 match(Set dst (XorL m1 (XorL src2 src1)));
9953 ins_cost(INSN_COST);
9954 format %{ "eon $dst, $src1, $src2" %}
9955
9956 ins_encode %{
9957 __ eon(as_Register($dst$$reg),
9958 as_Register($src1$$reg),
9959 as_Register($src2$$reg),
9960 Assembler::LSL, 0);
9961 %}
9962
9963 ins_pipe(ialu_reg_reg);
9964 %}
9965
9966 instruct AndI_reg_URShift_not_reg(iRegINoSp dst,
9967 iRegIorL2I src1, iRegIorL2I src2,
9968 immI src3, immI_M1 src4, rFlagsReg cr) %{
9969 match(Set dst (AndI src1 (XorI(URShiftI src2 src3) src4)));
9970 ins_cost(1.9 * INSN_COST);
9971 format %{ "bicw $dst, $src1, $src2, LSR $src3" %}
9972
9973 ins_encode %{
9974 __ bicw(as_Register($dst$$reg),
9975 as_Register($src1$$reg),
9976 as_Register($src2$$reg),
9977 Assembler::LSR,
9978 $src3$$constant & 0x1f);
9979 %}
9980
9981 ins_pipe(ialu_reg_reg_shift);
9982 %}
9983
9984 instruct AndL_reg_URShift_not_reg(iRegLNoSp dst,
9985 iRegL src1, iRegL src2,
9986 immI src3, immL_M1 src4, rFlagsReg cr) %{
9987 match(Set dst (AndL src1 (XorL(URShiftL src2 src3) src4)));
9988 ins_cost(1.9 * INSN_COST);
9989 format %{ "bic $dst, $src1, $src2, LSR $src3" %}
9990
9991 ins_encode %{
9992 __ bic(as_Register($dst$$reg),
9993 as_Register($src1$$reg),
9994 as_Register($src2$$reg),
9995 Assembler::LSR,
9996 $src3$$constant & 0x3f);
9997 %}
9998
9999 ins_pipe(ialu_reg_reg_shift);
10000 %}
10001
10002 instruct AndI_reg_RShift_not_reg(iRegINoSp dst,
10003 iRegIorL2I src1, iRegIorL2I src2,
10004 immI src3, immI_M1 src4, rFlagsReg cr) %{
10005 match(Set dst (AndI src1 (XorI(RShiftI src2 src3) src4)));
10006 ins_cost(1.9 * INSN_COST);
10007 format %{ "bicw $dst, $src1, $src2, ASR $src3" %}
10008
10009 ins_encode %{
10010 __ bicw(as_Register($dst$$reg),
10011 as_Register($src1$$reg),
10012 as_Register($src2$$reg),
10013 Assembler::ASR,
10014 $src3$$constant & 0x1f);
10015 %}
10016
10017 ins_pipe(ialu_reg_reg_shift);
10018 %}
10019
10020 instruct AndL_reg_RShift_not_reg(iRegLNoSp dst,
10021 iRegL src1, iRegL src2,
10022 immI src3, immL_M1 src4, rFlagsReg cr) %{
10023 match(Set dst (AndL src1 (XorL(RShiftL src2 src3) src4)));
10024 ins_cost(1.9 * INSN_COST);
10025 format %{ "bic $dst, $src1, $src2, ASR $src3" %}
10026
10027 ins_encode %{
10028 __ bic(as_Register($dst$$reg),
10029 as_Register($src1$$reg),
10030 as_Register($src2$$reg),
10031 Assembler::ASR,
10032 $src3$$constant & 0x3f);
10033 %}
10034
10035 ins_pipe(ialu_reg_reg_shift);
10036 %}
10037
10038 instruct AndI_reg_LShift_not_reg(iRegINoSp dst,
10039 iRegIorL2I src1, iRegIorL2I src2,
10040 immI src3, immI_M1 src4, rFlagsReg cr) %{
10041 match(Set dst (AndI src1 (XorI(LShiftI src2 src3) src4)));
10042 ins_cost(1.9 * INSN_COST);
10043 format %{ "bicw $dst, $src1, $src2, LSL $src3" %}
10044
10045 ins_encode %{
10046 __ bicw(as_Register($dst$$reg),
10047 as_Register($src1$$reg),
10048 as_Register($src2$$reg),
10049 Assembler::LSL,
10050 $src3$$constant & 0x1f);
10051 %}
10052
10053 ins_pipe(ialu_reg_reg_shift);
10054 %}
10055
10056 instruct AndL_reg_LShift_not_reg(iRegLNoSp dst,
10057 iRegL src1, iRegL src2,
10058 immI src3, immL_M1 src4, rFlagsReg cr) %{
10059 match(Set dst (AndL src1 (XorL(LShiftL src2 src3) src4)));
10060 ins_cost(1.9 * INSN_COST);
10061 format %{ "bic $dst, $src1, $src2, LSL $src3" %}
10062
10063 ins_encode %{
10064 __ bic(as_Register($dst$$reg),
10065 as_Register($src1$$reg),
10066 as_Register($src2$$reg),
10067 Assembler::LSL,
10068 $src3$$constant & 0x3f);
10069 %}
10070
10071 ins_pipe(ialu_reg_reg_shift);
10072 %}
10073
10074 instruct XorI_reg_URShift_not_reg(iRegINoSp dst,
10075 iRegIorL2I src1, iRegIorL2I src2,
10076 immI src3, immI_M1 src4, rFlagsReg cr) %{
10077 match(Set dst (XorI src4 (XorI(URShiftI src2 src3) src1)));
10078 ins_cost(1.9 * INSN_COST);
10079 format %{ "eonw $dst, $src1, $src2, LSR $src3" %}
10080
10081 ins_encode %{
10082 __ eonw(as_Register($dst$$reg),
10083 as_Register($src1$$reg),
10084 as_Register($src2$$reg),
10085 Assembler::LSR,
10086 $src3$$constant & 0x1f);
10087 %}
10088
10089 ins_pipe(ialu_reg_reg_shift);
10090 %}
10091
10092 instruct XorL_reg_URShift_not_reg(iRegLNoSp dst,
10093 iRegL src1, iRegL src2,
10094 immI src3, immL_M1 src4, rFlagsReg cr) %{
10095 match(Set dst (XorL src4 (XorL(URShiftL src2 src3) src1)));
10096 ins_cost(1.9 * INSN_COST);
10097 format %{ "eon $dst, $src1, $src2, LSR $src3" %}
10098
10099 ins_encode %{
10100 __ eon(as_Register($dst$$reg),
10101 as_Register($src1$$reg),
10102 as_Register($src2$$reg),
10103 Assembler::LSR,
10104 $src3$$constant & 0x3f);
10105 %}
10106
10107 ins_pipe(ialu_reg_reg_shift);
10108 %}
10109
10110 instruct XorI_reg_RShift_not_reg(iRegINoSp dst,
10111 iRegIorL2I src1, iRegIorL2I src2,
10112 immI src3, immI_M1 src4, rFlagsReg cr) %{
10113 match(Set dst (XorI src4 (XorI(RShiftI src2 src3) src1)));
10114 ins_cost(1.9 * INSN_COST);
10115 format %{ "eonw $dst, $src1, $src2, ASR $src3" %}
10116
10117 ins_encode %{
10118 __ eonw(as_Register($dst$$reg),
10119 as_Register($src1$$reg),
10120 as_Register($src2$$reg),
10121 Assembler::ASR,
10122 $src3$$constant & 0x1f);
10123 %}
10124
10125 ins_pipe(ialu_reg_reg_shift);
10126 %}
10127
10128 instruct XorL_reg_RShift_not_reg(iRegLNoSp dst,
10129 iRegL src1, iRegL src2,
10130 immI src3, immL_M1 src4, rFlagsReg cr) %{
10131 match(Set dst (XorL src4 (XorL(RShiftL src2 src3) src1)));
10132 ins_cost(1.9 * INSN_COST);
10133 format %{ "eon $dst, $src1, $src2, ASR $src3" %}
10134
10135 ins_encode %{
10136 __ eon(as_Register($dst$$reg),
10137 as_Register($src1$$reg),
10138 as_Register($src2$$reg),
10139 Assembler::ASR,
10140 $src3$$constant & 0x3f);
10141 %}
10142
10143 ins_pipe(ialu_reg_reg_shift);
10144 %}
10145
10146 instruct XorI_reg_LShift_not_reg(iRegINoSp dst,
10147 iRegIorL2I src1, iRegIorL2I src2,
10148 immI src3, immI_M1 src4, rFlagsReg cr) %{
10149 match(Set dst (XorI src4 (XorI(LShiftI src2 src3) src1)));
10150 ins_cost(1.9 * INSN_COST);
10151 format %{ "eonw $dst, $src1, $src2, LSL $src3" %}
10152
10153 ins_encode %{
10154 __ eonw(as_Register($dst$$reg),
10155 as_Register($src1$$reg),
10156 as_Register($src2$$reg),
10157 Assembler::LSL,
10158 $src3$$constant & 0x1f);
10159 %}
10160
10161 ins_pipe(ialu_reg_reg_shift);
10162 %}
10163
10164 instruct XorL_reg_LShift_not_reg(iRegLNoSp dst,
10165 iRegL src1, iRegL src2,
10166 immI src3, immL_M1 src4, rFlagsReg cr) %{
10167 match(Set dst (XorL src4 (XorL(LShiftL src2 src3) src1)));
10168 ins_cost(1.9 * INSN_COST);
10169 format %{ "eon $dst, $src1, $src2, LSL $src3" %}
10170
10171 ins_encode %{
10172 __ eon(as_Register($dst$$reg),
10173 as_Register($src1$$reg),
10174 as_Register($src2$$reg),
10175 Assembler::LSL,
10176 $src3$$constant & 0x3f);
10177 %}
10178
10179 ins_pipe(ialu_reg_reg_shift);
10180 %}
10181
10182 instruct OrI_reg_URShift_not_reg(iRegINoSp dst,
10183 iRegIorL2I src1, iRegIorL2I src2,
10184 immI src3, immI_M1 src4, rFlagsReg cr) %{
10185 match(Set dst (OrI src1 (XorI(URShiftI src2 src3) src4)));
10186 ins_cost(1.9 * INSN_COST);
10187 format %{ "ornw $dst, $src1, $src2, LSR $src3" %}
10188
10189 ins_encode %{
10190 __ ornw(as_Register($dst$$reg),
10191 as_Register($src1$$reg),
10192 as_Register($src2$$reg),
10193 Assembler::LSR,
10194 $src3$$constant & 0x1f);
10195 %}
10196
10197 ins_pipe(ialu_reg_reg_shift);
10198 %}
10199
10200 instruct OrL_reg_URShift_not_reg(iRegLNoSp dst,
10201 iRegL src1, iRegL src2,
10202 immI src3, immL_M1 src4, rFlagsReg cr) %{
10203 match(Set dst (OrL src1 (XorL(URShiftL src2 src3) src4)));
10204 ins_cost(1.9 * INSN_COST);
10205 format %{ "orn $dst, $src1, $src2, LSR $src3" %}
10206
10207 ins_encode %{
10208 __ orn(as_Register($dst$$reg),
10209 as_Register($src1$$reg),
10210 as_Register($src2$$reg),
10211 Assembler::LSR,
10212 $src3$$constant & 0x3f);
10213 %}
10214
10215 ins_pipe(ialu_reg_reg_shift);
10216 %}
10217
10218 instruct OrI_reg_RShift_not_reg(iRegINoSp dst,
10219 iRegIorL2I src1, iRegIorL2I src2,
10220 immI src3, immI_M1 src4, rFlagsReg cr) %{
10221 match(Set dst (OrI src1 (XorI(RShiftI src2 src3) src4)));
10222 ins_cost(1.9 * INSN_COST);
10223 format %{ "ornw $dst, $src1, $src2, ASR $src3" %}
10224
10225 ins_encode %{
10226 __ ornw(as_Register($dst$$reg),
10227 as_Register($src1$$reg),
10228 as_Register($src2$$reg),
10229 Assembler::ASR,
10230 $src3$$constant & 0x1f);
10231 %}
10232
10233 ins_pipe(ialu_reg_reg_shift);
10234 %}
10235
10236 instruct OrL_reg_RShift_not_reg(iRegLNoSp dst,
10237 iRegL src1, iRegL src2,
10238 immI src3, immL_M1 src4, rFlagsReg cr) %{
10239 match(Set dst (OrL src1 (XorL(RShiftL src2 src3) src4)));
10240 ins_cost(1.9 * INSN_COST);
10241 format %{ "orn $dst, $src1, $src2, ASR $src3" %}
10242
10243 ins_encode %{
10244 __ orn(as_Register($dst$$reg),
10245 as_Register($src1$$reg),
10246 as_Register($src2$$reg),
10247 Assembler::ASR,
10248 $src3$$constant & 0x3f);
10249 %}
10250
10251 ins_pipe(ialu_reg_reg_shift);
10252 %}
10253
10254 instruct OrI_reg_LShift_not_reg(iRegINoSp dst,
10255 iRegIorL2I src1, iRegIorL2I src2,
10256 immI src3, immI_M1 src4, rFlagsReg cr) %{
10257 match(Set dst (OrI src1 (XorI(LShiftI src2 src3) src4)));
10258 ins_cost(1.9 * INSN_COST);
10259 format %{ "ornw $dst, $src1, $src2, LSL $src3" %}
10260
10261 ins_encode %{
10262 __ ornw(as_Register($dst$$reg),
10263 as_Register($src1$$reg),
10264 as_Register($src2$$reg),
10265 Assembler::LSL,
10266 $src3$$constant & 0x1f);
10267 %}
10268
10269 ins_pipe(ialu_reg_reg_shift);
10270 %}
10271
10272 instruct OrL_reg_LShift_not_reg(iRegLNoSp dst,
10273 iRegL src1, iRegL src2,
10274 immI src3, immL_M1 src4, rFlagsReg cr) %{
10275 match(Set dst (OrL src1 (XorL(LShiftL src2 src3) src4)));
10276 ins_cost(1.9 * INSN_COST);
10277 format %{ "orn $dst, $src1, $src2, LSL $src3" %}
10278
10279 ins_encode %{
10280 __ orn(as_Register($dst$$reg),
10281 as_Register($src1$$reg),
10282 as_Register($src2$$reg),
10283 Assembler::LSL,
10284 $src3$$constant & 0x3f);
10285 %}
10286
10287 ins_pipe(ialu_reg_reg_shift);
10288 %}
10289
10290 instruct AndI_reg_URShift_reg(iRegINoSp dst,
10291 iRegIorL2I src1, iRegIorL2I src2,
10292 immI src3, rFlagsReg cr) %{
10293 match(Set dst (AndI src1 (URShiftI src2 src3)));
10294
10295 ins_cost(1.9 * INSN_COST);
10296 format %{ "andw $dst, $src1, $src2, LSR $src3" %}
10297
10298 ins_encode %{
10299 __ andw(as_Register($dst$$reg),
10300 as_Register($src1$$reg),
10301 as_Register($src2$$reg),
10302 Assembler::LSR,
10303 $src3$$constant & 0x1f);
10304 %}
10305
10306 ins_pipe(ialu_reg_reg_shift);
10307 %}
10308
10309 instruct AndL_reg_URShift_reg(iRegLNoSp dst,
10310 iRegL src1, iRegL src2,
10311 immI src3, rFlagsReg cr) %{
10312 match(Set dst (AndL src1 (URShiftL src2 src3)));
10313
10314 ins_cost(1.9 * INSN_COST);
10315 format %{ "andr $dst, $src1, $src2, LSR $src3" %}
10316
10317 ins_encode %{
10318 __ andr(as_Register($dst$$reg),
10319 as_Register($src1$$reg),
10320 as_Register($src2$$reg),
10321 Assembler::LSR,
10322 $src3$$constant & 0x3f);
10323 %}
10324
10325 ins_pipe(ialu_reg_reg_shift);
10326 %}
10327
10328 instruct AndI_reg_RShift_reg(iRegINoSp dst,
10329 iRegIorL2I src1, iRegIorL2I src2,
10330 immI src3, rFlagsReg cr) %{
10331 match(Set dst (AndI src1 (RShiftI src2 src3)));
10332
10333 ins_cost(1.9 * INSN_COST);
10334 format %{ "andw $dst, $src1, $src2, ASR $src3" %}
10335
10336 ins_encode %{
10337 __ andw(as_Register($dst$$reg),
10338 as_Register($src1$$reg),
10339 as_Register($src2$$reg),
10340 Assembler::ASR,
10341 $src3$$constant & 0x1f);
10342 %}
10343
10344 ins_pipe(ialu_reg_reg_shift);
10345 %}
10346
10347 instruct AndL_reg_RShift_reg(iRegLNoSp dst,
10348 iRegL src1, iRegL src2,
10349 immI src3, rFlagsReg cr) %{
10350 match(Set dst (AndL src1 (RShiftL src2 src3)));
10351
10352 ins_cost(1.9 * INSN_COST);
10353 format %{ "andr $dst, $src1, $src2, ASR $src3" %}
10354
10355 ins_encode %{
10356 __ andr(as_Register($dst$$reg),
10357 as_Register($src1$$reg),
10358 as_Register($src2$$reg),
10359 Assembler::ASR,
10360 $src3$$constant & 0x3f);
10361 %}
10362
10363 ins_pipe(ialu_reg_reg_shift);
10364 %}
10365
10366 instruct AndI_reg_LShift_reg(iRegINoSp dst,
10367 iRegIorL2I src1, iRegIorL2I src2,
10368 immI src3, rFlagsReg cr) %{
10369 match(Set dst (AndI src1 (LShiftI src2 src3)));
10370
10371 ins_cost(1.9 * INSN_COST);
10372 format %{ "andw $dst, $src1, $src2, LSL $src3" %}
10373
10374 ins_encode %{
10375 __ andw(as_Register($dst$$reg),
10376 as_Register($src1$$reg),
10377 as_Register($src2$$reg),
10378 Assembler::LSL,
10379 $src3$$constant & 0x1f);
10380 %}
10381
10382 ins_pipe(ialu_reg_reg_shift);
10383 %}
10384
10385 instruct AndL_reg_LShift_reg(iRegLNoSp dst,
10386 iRegL src1, iRegL src2,
10387 immI src3, rFlagsReg cr) %{
10388 match(Set dst (AndL src1 (LShiftL src2 src3)));
10389
10390 ins_cost(1.9 * INSN_COST);
10391 format %{ "andr $dst, $src1, $src2, LSL $src3" %}
10392
10393 ins_encode %{
10394 __ andr(as_Register($dst$$reg),
10395 as_Register($src1$$reg),
10396 as_Register($src2$$reg),
10397 Assembler::LSL,
10398 $src3$$constant & 0x3f);
10399 %}
10400
10401 ins_pipe(ialu_reg_reg_shift);
10402 %}
10403
10404 instruct XorI_reg_URShift_reg(iRegINoSp dst,
10405 iRegIorL2I src1, iRegIorL2I src2,
10406 immI src3, rFlagsReg cr) %{
10407 match(Set dst (XorI src1 (URShiftI src2 src3)));
10408
10409 ins_cost(1.9 * INSN_COST);
10410 format %{ "eorw $dst, $src1, $src2, LSR $src3" %}
10411
10412 ins_encode %{
10413 __ eorw(as_Register($dst$$reg),
10414 as_Register($src1$$reg),
10415 as_Register($src2$$reg),
10416 Assembler::LSR,
10417 $src3$$constant & 0x1f);
10418 %}
10419
10420 ins_pipe(ialu_reg_reg_shift);
10421 %}
10422
10423 instruct XorL_reg_URShift_reg(iRegLNoSp dst,
10424 iRegL src1, iRegL src2,
10425 immI src3, rFlagsReg cr) %{
10426 match(Set dst (XorL src1 (URShiftL src2 src3)));
10427
10428 ins_cost(1.9 * INSN_COST);
10429 format %{ "eor $dst, $src1, $src2, LSR $src3" %}
10430
10431 ins_encode %{
10432 __ eor(as_Register($dst$$reg),
10433 as_Register($src1$$reg),
10434 as_Register($src2$$reg),
10435 Assembler::LSR,
10436 $src3$$constant & 0x3f);
10437 %}
10438
10439 ins_pipe(ialu_reg_reg_shift);
10440 %}
10441
10442 instruct XorI_reg_RShift_reg(iRegINoSp dst,
10443 iRegIorL2I src1, iRegIorL2I src2,
10444 immI src3, rFlagsReg cr) %{
10445 match(Set dst (XorI src1 (RShiftI src2 src3)));
10446
10447 ins_cost(1.9 * INSN_COST);
10448 format %{ "eorw $dst, $src1, $src2, ASR $src3" %}
10449
10450 ins_encode %{
10451 __ eorw(as_Register($dst$$reg),
10452 as_Register($src1$$reg),
10453 as_Register($src2$$reg),
10454 Assembler::ASR,
10455 $src3$$constant & 0x1f);
10456 %}
10457
10458 ins_pipe(ialu_reg_reg_shift);
10459 %}
10460
10461 instruct XorL_reg_RShift_reg(iRegLNoSp dst,
10462 iRegL src1, iRegL src2,
10463 immI src3, rFlagsReg cr) %{
10464 match(Set dst (XorL src1 (RShiftL src2 src3)));
10465
10466 ins_cost(1.9 * INSN_COST);
10467 format %{ "eor $dst, $src1, $src2, ASR $src3" %}
10468
10469 ins_encode %{
10470 __ eor(as_Register($dst$$reg),
10471 as_Register($src1$$reg),
10472 as_Register($src2$$reg),
10473 Assembler::ASR,
10474 $src3$$constant & 0x3f);
10475 %}
10476
10477 ins_pipe(ialu_reg_reg_shift);
10478 %}
10479
10480 instruct XorI_reg_LShift_reg(iRegINoSp dst,
10481 iRegIorL2I src1, iRegIorL2I src2,
10482 immI src3, rFlagsReg cr) %{
10483 match(Set dst (XorI src1 (LShiftI src2 src3)));
10484
10485 ins_cost(1.9 * INSN_COST);
10486 format %{ "eorw $dst, $src1, $src2, LSL $src3" %}
10487
10488 ins_encode %{
10489 __ eorw(as_Register($dst$$reg),
10490 as_Register($src1$$reg),
10491 as_Register($src2$$reg),
10492 Assembler::LSL,
10493 $src3$$constant & 0x1f);
10494 %}
10495
10496 ins_pipe(ialu_reg_reg_shift);
10497 %}
10498
10499 instruct XorL_reg_LShift_reg(iRegLNoSp dst,
10500 iRegL src1, iRegL src2,
10501 immI src3, rFlagsReg cr) %{
10502 match(Set dst (XorL src1 (LShiftL src2 src3)));
10503
10504 ins_cost(1.9 * INSN_COST);
10505 format %{ "eor $dst, $src1, $src2, LSL $src3" %}
10506
10507 ins_encode %{
10508 __ eor(as_Register($dst$$reg),
10509 as_Register($src1$$reg),
10510 as_Register($src2$$reg),
10511 Assembler::LSL,
10512 $src3$$constant & 0x3f);
10513 %}
10514
10515 ins_pipe(ialu_reg_reg_shift);
10516 %}
10517
10518 instruct OrI_reg_URShift_reg(iRegINoSp dst,
10519 iRegIorL2I src1, iRegIorL2I src2,
10520 immI src3, rFlagsReg cr) %{
10521 match(Set dst (OrI src1 (URShiftI src2 src3)));
10522
10523 ins_cost(1.9 * INSN_COST);
10524 format %{ "orrw $dst, $src1, $src2, LSR $src3" %}
10525
10526 ins_encode %{
10527 __ orrw(as_Register($dst$$reg),
10528 as_Register($src1$$reg),
10529 as_Register($src2$$reg),
10530 Assembler::LSR,
10531 $src3$$constant & 0x1f);
10532 %}
10533
10534 ins_pipe(ialu_reg_reg_shift);
10535 %}
10536
10537 instruct OrL_reg_URShift_reg(iRegLNoSp dst,
10538 iRegL src1, iRegL src2,
10539 immI src3, rFlagsReg cr) %{
10540 match(Set dst (OrL src1 (URShiftL src2 src3)));
10541
10542 ins_cost(1.9 * INSN_COST);
10543 format %{ "orr $dst, $src1, $src2, LSR $src3" %}
10544
10545 ins_encode %{
10546 __ orr(as_Register($dst$$reg),
10547 as_Register($src1$$reg),
10548 as_Register($src2$$reg),
10549 Assembler::LSR,
10550 $src3$$constant & 0x3f);
10551 %}
10552
10553 ins_pipe(ialu_reg_reg_shift);
10554 %}
10555
10556 instruct OrI_reg_RShift_reg(iRegINoSp dst,
10557 iRegIorL2I src1, iRegIorL2I src2,
10558 immI src3, rFlagsReg cr) %{
10559 match(Set dst (OrI src1 (RShiftI src2 src3)));
10560
10561 ins_cost(1.9 * INSN_COST);
10562 format %{ "orrw $dst, $src1, $src2, ASR $src3" %}
10563
10564 ins_encode %{
10565 __ orrw(as_Register($dst$$reg),
10566 as_Register($src1$$reg),
10567 as_Register($src2$$reg),
10568 Assembler::ASR,
10569 $src3$$constant & 0x1f);
10570 %}
10571
10572 ins_pipe(ialu_reg_reg_shift);
10573 %}
10574
10575 instruct OrL_reg_RShift_reg(iRegLNoSp dst,
10576 iRegL src1, iRegL src2,
10577 immI src3, rFlagsReg cr) %{
10578 match(Set dst (OrL src1 (RShiftL src2 src3)));
10579
10580 ins_cost(1.9 * INSN_COST);
10581 format %{ "orr $dst, $src1, $src2, ASR $src3" %}
10582
10583 ins_encode %{
10584 __ orr(as_Register($dst$$reg),
10585 as_Register($src1$$reg),
10586 as_Register($src2$$reg),
10587 Assembler::ASR,
10588 $src3$$constant & 0x3f);
10589 %}
10590
10591 ins_pipe(ialu_reg_reg_shift);
10592 %}
10593
10594 instruct OrI_reg_LShift_reg(iRegINoSp dst,
10595 iRegIorL2I src1, iRegIorL2I src2,
10596 immI src3, rFlagsReg cr) %{
10597 match(Set dst (OrI src1 (LShiftI src2 src3)));
10598
10599 ins_cost(1.9 * INSN_COST);
10600 format %{ "orrw $dst, $src1, $src2, LSL $src3" %}
10601
10602 ins_encode %{
10603 __ orrw(as_Register($dst$$reg),
10604 as_Register($src1$$reg),
10605 as_Register($src2$$reg),
10606 Assembler::LSL,
10607 $src3$$constant & 0x1f);
10608 %}
10609
10610 ins_pipe(ialu_reg_reg_shift);
10611 %}
10612
10613 instruct OrL_reg_LShift_reg(iRegLNoSp dst,
10614 iRegL src1, iRegL src2,
10615 immI src3, rFlagsReg cr) %{
10616 match(Set dst (OrL src1 (LShiftL src2 src3)));
10617
10618 ins_cost(1.9 * INSN_COST);
10619 format %{ "orr $dst, $src1, $src2, LSL $src3" %}
10620
10621 ins_encode %{
10622 __ orr(as_Register($dst$$reg),
10623 as_Register($src1$$reg),
10624 as_Register($src2$$reg),
10625 Assembler::LSL,
10626 $src3$$constant & 0x3f);
10627 %}
10628
10629 ins_pipe(ialu_reg_reg_shift);
10630 %}
10631
10632 instruct AddI_reg_URShift_reg(iRegINoSp dst,
10633 iRegIorL2I src1, iRegIorL2I src2,
10634 immI src3, rFlagsReg cr) %{
10635 match(Set dst (AddI src1 (URShiftI src2 src3)));
10636
10637 ins_cost(1.9 * INSN_COST);
10638 format %{ "addw $dst, $src1, $src2, LSR $src3" %}
10639
10640 ins_encode %{
10641 __ addw(as_Register($dst$$reg),
10642 as_Register($src1$$reg),
10643 as_Register($src2$$reg),
10644 Assembler::LSR,
10645 $src3$$constant & 0x1f);
10646 %}
10647
10648 ins_pipe(ialu_reg_reg_shift);
10649 %}
10650
10651 instruct AddL_reg_URShift_reg(iRegLNoSp dst,
10652 iRegL src1, iRegL src2,
10653 immI src3, rFlagsReg cr) %{
10654 match(Set dst (AddL src1 (URShiftL src2 src3)));
10655
10656 ins_cost(1.9 * INSN_COST);
10657 format %{ "add $dst, $src1, $src2, LSR $src3" %}
10658
10659 ins_encode %{
10660 __ add(as_Register($dst$$reg),
10661 as_Register($src1$$reg),
10662 as_Register($src2$$reg),
10663 Assembler::LSR,
10664 $src3$$constant & 0x3f);
10665 %}
10666
10667 ins_pipe(ialu_reg_reg_shift);
10668 %}
10669
10670 instruct AddI_reg_RShift_reg(iRegINoSp dst,
10671 iRegIorL2I src1, iRegIorL2I src2,
10672 immI src3, rFlagsReg cr) %{
10673 match(Set dst (AddI src1 (RShiftI src2 src3)));
10674
10675 ins_cost(1.9 * INSN_COST);
10676 format %{ "addw $dst, $src1, $src2, ASR $src3" %}
10677
10678 ins_encode %{
10679 __ addw(as_Register($dst$$reg),
10680 as_Register($src1$$reg),
10681 as_Register($src2$$reg),
10682 Assembler::ASR,
10683 $src3$$constant & 0x1f);
10684 %}
10685
10686 ins_pipe(ialu_reg_reg_shift);
10687 %}
10688
10689 instruct AddL_reg_RShift_reg(iRegLNoSp dst,
10690 iRegL src1, iRegL src2,
10691 immI src3, rFlagsReg cr) %{
10692 match(Set dst (AddL src1 (RShiftL src2 src3)));
10693
10694 ins_cost(1.9 * INSN_COST);
10695 format %{ "add $dst, $src1, $src2, ASR $src3" %}
10696
10697 ins_encode %{
10698 __ add(as_Register($dst$$reg),
10699 as_Register($src1$$reg),
10700 as_Register($src2$$reg),
10701 Assembler::ASR,
10702 $src3$$constant & 0x3f);
10703 %}
10704
10705 ins_pipe(ialu_reg_reg_shift);
10706 %}
10707
10708 instruct AddI_reg_LShift_reg(iRegINoSp dst,
10709 iRegIorL2I src1, iRegIorL2I src2,
10710 immI src3, rFlagsReg cr) %{
10711 match(Set dst (AddI src1 (LShiftI src2 src3)));
10712
10713 ins_cost(1.9 * INSN_COST);
10714 format %{ "addw $dst, $src1, $src2, LSL $src3" %}
10715
10716 ins_encode %{
10717 __ addw(as_Register($dst$$reg),
10718 as_Register($src1$$reg),
10719 as_Register($src2$$reg),
10720 Assembler::LSL,
10721 $src3$$constant & 0x1f);
10722 %}
10723
10724 ins_pipe(ialu_reg_reg_shift);
10725 %}
10726
10727 instruct AddL_reg_LShift_reg(iRegLNoSp dst,
10728 iRegL src1, iRegL src2,
10729 immI src3, rFlagsReg cr) %{
10730 match(Set dst (AddL src1 (LShiftL src2 src3)));
10731
10732 ins_cost(1.9 * INSN_COST);
10733 format %{ "add $dst, $src1, $src2, LSL $src3" %}
10734
10735 ins_encode %{
10736 __ add(as_Register($dst$$reg),
10737 as_Register($src1$$reg),
10738 as_Register($src2$$reg),
10739 Assembler::LSL,
10740 $src3$$constant & 0x3f);
10741 %}
10742
10743 ins_pipe(ialu_reg_reg_shift);
10744 %}
10745
10746 instruct SubI_reg_URShift_reg(iRegINoSp dst,
10747 iRegIorL2I src1, iRegIorL2I src2,
10748 immI src3, rFlagsReg cr) %{
10749 match(Set dst (SubI src1 (URShiftI src2 src3)));
10750
10751 ins_cost(1.9 * INSN_COST);
10752 format %{ "subw $dst, $src1, $src2, LSR $src3" %}
10753
10754 ins_encode %{
10755 __ subw(as_Register($dst$$reg),
10756 as_Register($src1$$reg),
10757 as_Register($src2$$reg),
10758 Assembler::LSR,
10759 $src3$$constant & 0x1f);
10760 %}
10761
10762 ins_pipe(ialu_reg_reg_shift);
10763 %}
10764
10765 instruct SubL_reg_URShift_reg(iRegLNoSp dst,
10766 iRegL src1, iRegL src2,
10767 immI src3, rFlagsReg cr) %{
10768 match(Set dst (SubL src1 (URShiftL src2 src3)));
10769
10770 ins_cost(1.9 * INSN_COST);
10771 format %{ "sub $dst, $src1, $src2, LSR $src3" %}
10772
10773 ins_encode %{
10774 __ sub(as_Register($dst$$reg),
10775 as_Register($src1$$reg),
10776 as_Register($src2$$reg),
10777 Assembler::LSR,
10778 $src3$$constant & 0x3f);
10779 %}
10780
10781 ins_pipe(ialu_reg_reg_shift);
10782 %}
10783
10784 instruct SubI_reg_RShift_reg(iRegINoSp dst,
10785 iRegIorL2I src1, iRegIorL2I src2,
10786 immI src3, rFlagsReg cr) %{
10787 match(Set dst (SubI src1 (RShiftI src2 src3)));
10788
10789 ins_cost(1.9 * INSN_COST);
10790 format %{ "subw $dst, $src1, $src2, ASR $src3" %}
10791
10792 ins_encode %{
10793 __ subw(as_Register($dst$$reg),
10794 as_Register($src1$$reg),
10795 as_Register($src2$$reg),
10796 Assembler::ASR,
10797 $src3$$constant & 0x1f);
10798 %}
10799
10800 ins_pipe(ialu_reg_reg_shift);
10801 %}
10802
10803 instruct SubL_reg_RShift_reg(iRegLNoSp dst,
10804 iRegL src1, iRegL src2,
10805 immI src3, rFlagsReg cr) %{
10806 match(Set dst (SubL src1 (RShiftL src2 src3)));
10807
10808 ins_cost(1.9 * INSN_COST);
10809 format %{ "sub $dst, $src1, $src2, ASR $src3" %}
10810
10811 ins_encode %{
10812 __ sub(as_Register($dst$$reg),
10813 as_Register($src1$$reg),
10814 as_Register($src2$$reg),
10815 Assembler::ASR,
10816 $src3$$constant & 0x3f);
10817 %}
10818
10819 ins_pipe(ialu_reg_reg_shift);
10820 %}
10821
10822 instruct SubI_reg_LShift_reg(iRegINoSp dst,
10823 iRegIorL2I src1, iRegIorL2I src2,
10824 immI src3, rFlagsReg cr) %{
10825 match(Set dst (SubI src1 (LShiftI src2 src3)));
10826
10827 ins_cost(1.9 * INSN_COST);
10828 format %{ "subw $dst, $src1, $src2, LSL $src3" %}
10829
10830 ins_encode %{
10831 __ subw(as_Register($dst$$reg),
10832 as_Register($src1$$reg),
10833 as_Register($src2$$reg),
10834 Assembler::LSL,
10835 $src3$$constant & 0x1f);
10836 %}
10837
10838 ins_pipe(ialu_reg_reg_shift);
10839 %}
10840
10841 instruct SubL_reg_LShift_reg(iRegLNoSp dst,
10842 iRegL src1, iRegL src2,
10843 immI src3, rFlagsReg cr) %{
10844 match(Set dst (SubL src1 (LShiftL src2 src3)));
10845
10846 ins_cost(1.9 * INSN_COST);
10847 format %{ "sub $dst, $src1, $src2, LSL $src3" %}
10848
10849 ins_encode %{
10850 __ sub(as_Register($dst$$reg),
10851 as_Register($src1$$reg),
10852 as_Register($src2$$reg),
10853 Assembler::LSL,
10854 $src3$$constant & 0x3f);
10855 %}
10856
10857 ins_pipe(ialu_reg_reg_shift);
10858 %}
10859
10860
10861
10862 // Shift Left followed by Shift Right.
10863 // This idiom is used by the compiler for the i2b bytecode etc.
10864 instruct sbfmL(iRegLNoSp dst, iRegL src, immI lshift_count, immI rshift_count)
10865 %{
10866 match(Set dst (RShiftL (LShiftL src lshift_count) rshift_count));
10867 // Make sure we are not going to exceed what sbfm can do.
10868 predicate((unsigned int)n->in(2)->get_int() <= 63
10869 && (unsigned int)n->in(1)->in(2)->get_int() <= 63);
10870
10871 ins_cost(INSN_COST * 2);
10872 format %{ "sbfm $dst, $src, $rshift_count - $lshift_count, #63 - $lshift_count" %}
10873 ins_encode %{
10874 int lshift = $lshift_count$$constant, rshift = $rshift_count$$constant;
10875 int s = 63 - lshift;
10876 int r = (rshift - lshift) & 63;
10877 __ sbfm(as_Register($dst$$reg),
10878 as_Register($src$$reg),
10879 r, s);
10880 %}
10881
10882 ins_pipe(ialu_reg_shift);
10883 %}
10884
10885 // Shift Left followed by Shift Right.
10886 // This idiom is used by the compiler for the i2b bytecode etc.
10887 instruct sbfmwI(iRegINoSp dst, iRegIorL2I src, immI lshift_count, immI rshift_count)
10888 %{
10889 match(Set dst (RShiftI (LShiftI src lshift_count) rshift_count));
10890 // Make sure we are not going to exceed what sbfmw can do.
10891 predicate((unsigned int)n->in(2)->get_int() <= 31
10892 && (unsigned int)n->in(1)->in(2)->get_int() <= 31);
10893
10894 ins_cost(INSN_COST * 2);
10895 format %{ "sbfmw $dst, $src, $rshift_count - $lshift_count, #31 - $lshift_count" %}
10896 ins_encode %{
10897 int lshift = $lshift_count$$constant, rshift = $rshift_count$$constant;
10898 int s = 31 - lshift;
10899 int r = (rshift - lshift) & 31;
10900 __ sbfmw(as_Register($dst$$reg),
10901 as_Register($src$$reg),
10902 r, s);
10903 %}
10904
10905 ins_pipe(ialu_reg_shift);
10906 %}
10907
10908 // Shift Left followed by Shift Right.
10909 // This idiom is used by the compiler for the i2b bytecode etc.
10910 instruct ubfmL(iRegLNoSp dst, iRegL src, immI lshift_count, immI rshift_count)
10911 %{
10912 match(Set dst (URShiftL (LShiftL src lshift_count) rshift_count));
10913 // Make sure we are not going to exceed what ubfm can do.
10914 predicate((unsigned int)n->in(2)->get_int() <= 63
10915 && (unsigned int)n->in(1)->in(2)->get_int() <= 63);
10916
10917 ins_cost(INSN_COST * 2);
10918 format %{ "ubfm $dst, $src, $rshift_count - $lshift_count, #63 - $lshift_count" %}
10919 ins_encode %{
10920 int lshift = $lshift_count$$constant, rshift = $rshift_count$$constant;
10921 int s = 63 - lshift;
10922 int r = (rshift - lshift) & 63;
10923 __ ubfm(as_Register($dst$$reg),
10924 as_Register($src$$reg),
10925 r, s);
10926 %}
10927
10928 ins_pipe(ialu_reg_shift);
10929 %}
10930
10931 // Shift Left followed by Shift Right.
10932 // This idiom is used by the compiler for the i2b bytecode etc.
10933 instruct ubfmwI(iRegINoSp dst, iRegIorL2I src, immI lshift_count, immI rshift_count)
10934 %{
10935 match(Set dst (URShiftI (LShiftI src lshift_count) rshift_count));
10936 // Make sure we are not going to exceed what ubfmw can do.
10937 predicate((unsigned int)n->in(2)->get_int() <= 31
10938 && (unsigned int)n->in(1)->in(2)->get_int() <= 31);
10939
10940 ins_cost(INSN_COST * 2);
10941 format %{ "ubfmw $dst, $src, $rshift_count - $lshift_count, #31 - $lshift_count" %}
10942 ins_encode %{
10943 int lshift = $lshift_count$$constant, rshift = $rshift_count$$constant;
10944 int s = 31 - lshift;
10945 int r = (rshift - lshift) & 31;
10946 __ ubfmw(as_Register($dst$$reg),
10947 as_Register($src$$reg),
10948 r, s);
10949 %}
10950
10951 ins_pipe(ialu_reg_shift);
10952 %}
10953 // Bitfield extract with shift & mask
10954
10955 instruct ubfxwI(iRegINoSp dst, iRegIorL2I src, immI rshift, immI_bitmask mask)
10956 %{
10957 match(Set dst (AndI (URShiftI src rshift) mask));
10958 // Make sure we are not going to exceed what ubfxw can do.
10959 predicate((exact_log2(n->in(2)->get_int() + 1) + (n->in(1)->in(2)->get_int() & 31)) <= (31 + 1));
10960
10961 ins_cost(INSN_COST);
10962 format %{ "ubfxw $dst, $src, $mask" %}
10963 ins_encode %{
10964 int rshift = $rshift$$constant & 31;
10965 long mask = $mask$$constant;
10966 int width = exact_log2(mask+1);
10967 __ ubfxw(as_Register($dst$$reg),
10968 as_Register($src$$reg), rshift, width);
10969 %}
10970 ins_pipe(ialu_reg_shift);
10971 %}
10972 instruct ubfxL(iRegLNoSp dst, iRegL src, immI rshift, immL_bitmask mask)
10973 %{
10974 match(Set dst (AndL (URShiftL src rshift) mask));
10975 // Make sure we are not going to exceed what ubfx can do.
10976 predicate((exact_log2_long(n->in(2)->get_long() + 1) + (n->in(1)->in(2)->get_int() & 63)) <= (63 + 1));
10977
10978 ins_cost(INSN_COST);
10979 format %{ "ubfx $dst, $src, $mask" %}
10980 ins_encode %{
10981 int rshift = $rshift$$constant & 63;
10982 long mask = $mask$$constant;
10983 int width = exact_log2_long(mask+1);
10984 __ ubfx(as_Register($dst$$reg),
10985 as_Register($src$$reg), rshift, width);
10986 %}
10987 ins_pipe(ialu_reg_shift);
10988 %}
10989
10990 // We can use ubfx when extending an And with a mask when we know mask
10991 // is positive. We know that because immI_bitmask guarantees it.
10992 instruct ubfxIConvI2L(iRegLNoSp dst, iRegIorL2I src, immI rshift, immI_bitmask mask)
10993 %{
10994 match(Set dst (ConvI2L (AndI (URShiftI src rshift) mask)));
10995 // Make sure we are not going to exceed what ubfxw can do.
10996 predicate((exact_log2(n->in(1)->in(2)->get_int() + 1) + (n->in(1)->in(1)->in(2)->get_int() & 31)) <= (31 + 1));
10997
10998 ins_cost(INSN_COST * 2);
10999 format %{ "ubfx $dst, $src, $mask" %}
11000 ins_encode %{
11001 int rshift = $rshift$$constant & 31;
11002 long mask = $mask$$constant;
11003 int width = exact_log2(mask+1);
11004 __ ubfx(as_Register($dst$$reg),
11005 as_Register($src$$reg), rshift, width);
11006 %}
11007 ins_pipe(ialu_reg_shift);
11008 %}
11009
11010 // Rotations
11011
11012 instruct extrOrL(iRegLNoSp dst, iRegL src1, iRegL src2, immI lshift, immI rshift, rFlagsReg cr)
11013 %{
11014 match(Set dst (OrL (LShiftL src1 lshift) (URShiftL src2 rshift)));
11015 predicate(0 == ((n->in(1)->in(2)->get_int() + n->in(2)->in(2)->get_int()) & 63));
11016
11017 ins_cost(INSN_COST);
11018 format %{ "extr $dst, $src1, $src2, #$rshift" %}
11019
11020 ins_encode %{
11021 __ extr(as_Register($dst$$reg), as_Register($src1$$reg), as_Register($src2$$reg),
11022 $rshift$$constant & 63);
11023 %}
11024 ins_pipe(ialu_reg_reg_extr);
11025 %}
11026
11027 instruct extrOrI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI lshift, immI rshift, rFlagsReg cr)
11028 %{
11029 match(Set dst (OrI (LShiftI src1 lshift) (URShiftI src2 rshift)));
11030 predicate(0 == ((n->in(1)->in(2)->get_int() + n->in(2)->in(2)->get_int()) & 31));
11031
11032 ins_cost(INSN_COST);
11033 format %{ "extr $dst, $src1, $src2, #$rshift" %}
11034
11035 ins_encode %{
11036 __ extrw(as_Register($dst$$reg), as_Register($src1$$reg), as_Register($src2$$reg),
11037 $rshift$$constant & 31);
11038 %}
11039 ins_pipe(ialu_reg_reg_extr);
11040 %}
11041
11042 instruct extrAddL(iRegLNoSp dst, iRegL src1, iRegL src2, immI lshift, immI rshift, rFlagsReg cr)
11043 %{
11044 match(Set dst (AddL (LShiftL src1 lshift) (URShiftL src2 rshift)));
11045 predicate(0 == ((n->in(1)->in(2)->get_int() + n->in(2)->in(2)->get_int()) & 63));
11046
11047 ins_cost(INSN_COST);
11048 format %{ "extr $dst, $src1, $src2, #$rshift" %}
11049
11050 ins_encode %{
11051 __ extr(as_Register($dst$$reg), as_Register($src1$$reg), as_Register($src2$$reg),
11052 $rshift$$constant & 63);
11053 %}
11054 ins_pipe(ialu_reg_reg_extr);
11055 %}
11056
11057 instruct extrAddI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI lshift, immI rshift, rFlagsReg cr)
11058 %{
11059 match(Set dst (AddI (LShiftI src1 lshift) (URShiftI src2 rshift)));
11060 predicate(0 == ((n->in(1)->in(2)->get_int() + n->in(2)->in(2)->get_int()) & 31));
11061
11062 ins_cost(INSN_COST);
11063 format %{ "extr $dst, $src1, $src2, #$rshift" %}
11064
11065 ins_encode %{
11066 __ extrw(as_Register($dst$$reg), as_Register($src1$$reg), as_Register($src2$$reg),
11067 $rshift$$constant & 31);
11068 %}
11069 ins_pipe(ialu_reg_reg_extr);
11070 %}
11071
11072
11073 // rol expander
11074
11075 instruct rolL_rReg(iRegLNoSp dst, iRegL src, iRegI shift, rFlagsReg cr)
11076 %{
11077 effect(DEF dst, USE src, USE shift);
11078
11079 format %{ "rol $dst, $src, $shift" %}
11080 ins_cost(INSN_COST * 3);
11081 ins_encode %{
11082 __ subw(rscratch1, zr, as_Register($shift$$reg));
11083 __ rorv(as_Register($dst$$reg), as_Register($src$$reg),
11084 rscratch1);
11085 %}
11086 ins_pipe(ialu_reg_reg_vshift);
11087 %}
11088
11089 // rol expander
11090
11091 instruct rolI_rReg(iRegINoSp dst, iRegI src, iRegI shift, rFlagsReg cr)
11092 %{
11093 effect(DEF dst, USE src, USE shift);
11094
11095 format %{ "rol $dst, $src, $shift" %}
11096 ins_cost(INSN_COST * 3);
11097 ins_encode %{
11098 __ subw(rscratch1, zr, as_Register($shift$$reg));
11099 __ rorvw(as_Register($dst$$reg), as_Register($src$$reg),
11100 rscratch1);
11101 %}
11102 ins_pipe(ialu_reg_reg_vshift);
11103 %}
11104
11105 instruct rolL_rReg_Var_C_64(iRegLNoSp dst, iRegL src, iRegI shift, immI_64 c_64, rFlagsReg cr)
11106 %{
11107 match(Set dst (OrL (LShiftL src shift) (URShiftL src (SubI c_64 shift))));
11108
11109 expand %{
11110 rolL_rReg(dst, src, shift, cr);
11111 %}
11112 %}
11113
11114 instruct rolL_rReg_Var_C0(iRegLNoSp dst, iRegL src, iRegI shift, immI0 c0, rFlagsReg cr)
11115 %{
11116 match(Set dst (OrL (LShiftL src shift) (URShiftL src (SubI c0 shift))));
11117
11118 expand %{
11119 rolL_rReg(dst, src, shift, cr);
11120 %}
11121 %}
11122
11123 instruct rolI_rReg_Var_C_32(iRegINoSp dst, iRegI src, iRegI shift, immI_32 c_32, rFlagsReg cr)
11124 %{
11125 match(Set dst (OrI (LShiftI src shift) (URShiftI src (SubI c_32 shift))));
11126
11127 expand %{
11128 rolI_rReg(dst, src, shift, cr);
11129 %}
11130 %}
11131
11132 instruct rolI_rReg_Var_C0(iRegINoSp dst, iRegI src, iRegI shift, immI0 c0, rFlagsReg cr)
11133 %{
11134 match(Set dst (OrI (LShiftI src shift) (URShiftI src (SubI c0 shift))));
11135
11136 expand %{
11137 rolI_rReg(dst, src, shift, cr);
11138 %}
11139 %}
11140
11141 // ror expander
11142
11143 instruct rorL_rReg(iRegLNoSp dst, iRegL src, iRegI shift, rFlagsReg cr)
11144 %{
11145 effect(DEF dst, USE src, USE shift);
11146
11147 format %{ "ror $dst, $src, $shift" %}
11148 ins_cost(INSN_COST);
11149 ins_encode %{
11150 __ rorv(as_Register($dst$$reg), as_Register($src$$reg),
11151 as_Register($shift$$reg));
11152 %}
11153 ins_pipe(ialu_reg_reg_vshift);
11154 %}
11155
11156 // ror expander
11157
11158 instruct rorI_rReg(iRegINoSp dst, iRegI src, iRegI shift, rFlagsReg cr)
11159 %{
11160 effect(DEF dst, USE src, USE shift);
11161
11162 format %{ "ror $dst, $src, $shift" %}
11163 ins_cost(INSN_COST);
11164 ins_encode %{
11165 __ rorvw(as_Register($dst$$reg), as_Register($src$$reg),
11166 as_Register($shift$$reg));
11167 %}
11168 ins_pipe(ialu_reg_reg_vshift);
11169 %}
11170
11171 instruct rorL_rReg_Var_C_64(iRegLNoSp dst, iRegL src, iRegI shift, immI_64 c_64, rFlagsReg cr)
11172 %{
11173 match(Set dst (OrL (URShiftL src shift) (LShiftL src (SubI c_64 shift))));
11174
11175 expand %{
11176 rorL_rReg(dst, src, shift, cr);
11177 %}
11178 %}
11179
11180 instruct rorL_rReg_Var_C0(iRegLNoSp dst, iRegL src, iRegI shift, immI0 c0, rFlagsReg cr)
11181 %{
11182 match(Set dst (OrL (URShiftL src shift) (LShiftL src (SubI c0 shift))));
11183
11184 expand %{
11185 rorL_rReg(dst, src, shift, cr);
11186 %}
11187 %}
11188
11189 instruct rorI_rReg_Var_C_32(iRegINoSp dst, iRegI src, iRegI shift, immI_32 c_32, rFlagsReg cr)
11190 %{
11191 match(Set dst (OrI (URShiftI src shift) (LShiftI src (SubI c_32 shift))));
11192
11193 expand %{
11194 rorI_rReg(dst, src, shift, cr);
11195 %}
11196 %}
11197
11198 instruct rorI_rReg_Var_C0(iRegINoSp dst, iRegI src, iRegI shift, immI0 c0, rFlagsReg cr)
11199 %{
11200 match(Set dst (OrI (URShiftI src shift) (LShiftI src (SubI c0 shift))));
11201
11202 expand %{
11203 rorI_rReg(dst, src, shift, cr);
11204 %}
11205 %}
11206
11207 // Add/subtract (extended)
11208
11209 instruct AddExtI(iRegLNoSp dst, iRegL src1, iRegIorL2I src2, rFlagsReg cr)
11210 %{
11211 match(Set dst (AddL src1 (ConvI2L src2)));
11212 ins_cost(INSN_COST);
11213 format %{ "add $dst, $src1, sxtw $src2" %}
11214
11215 ins_encode %{
11216 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
11217 as_Register($src2$$reg), ext::sxtw);
11218 %}
11219 ins_pipe(ialu_reg_reg);
11220 %};
11221
11222 instruct SubExtI(iRegLNoSp dst, iRegL src1, iRegIorL2I src2, rFlagsReg cr)
11223 %{
11224 match(Set dst (SubL src1 (ConvI2L src2)));
11225 ins_cost(INSN_COST);
11226 format %{ "sub $dst, $src1, sxtw $src2" %}
11227
11228 ins_encode %{
11229 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
11230 as_Register($src2$$reg), ext::sxtw);
11231 %}
11232 ins_pipe(ialu_reg_reg);
11233 %};
11234
11235
11236 instruct AddExtI_sxth(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_16 lshift, immI_16 rshift, rFlagsReg cr)
11237 %{
11238 match(Set dst (AddI src1 (RShiftI (LShiftI src2 lshift) rshift)));
11239 ins_cost(INSN_COST);
11240 format %{ "add $dst, $src1, sxth $src2" %}
11241
11242 ins_encode %{
11243 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
11244 as_Register($src2$$reg), ext::sxth);
11245 %}
11246 ins_pipe(ialu_reg_reg);
11247 %}
11248
11249 instruct AddExtI_sxtb(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_24 lshift, immI_24 rshift, rFlagsReg cr)
11250 %{
11251 match(Set dst (AddI src1 (RShiftI (LShiftI src2 lshift) rshift)));
11252 ins_cost(INSN_COST);
11253 format %{ "add $dst, $src1, sxtb $src2" %}
11254
11255 ins_encode %{
11256 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
11257 as_Register($src2$$reg), ext::sxtb);
11258 %}
11259 ins_pipe(ialu_reg_reg);
11260 %}
11261
11262 instruct AddExtI_uxtb(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_24 lshift, immI_24 rshift, rFlagsReg cr)
11263 %{
11264 match(Set dst (AddI src1 (URShiftI (LShiftI src2 lshift) rshift)));
11265 ins_cost(INSN_COST);
11266 format %{ "add $dst, $src1, uxtb $src2" %}
11267
11268 ins_encode %{
11269 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
11270 as_Register($src2$$reg), ext::uxtb);
11271 %}
11272 ins_pipe(ialu_reg_reg);
11273 %}
11274
11275 instruct AddExtL_sxth(iRegLNoSp dst, iRegL src1, iRegL src2, immI_48 lshift, immI_48 rshift, rFlagsReg cr)
11276 %{
11277 match(Set dst (AddL src1 (RShiftL (LShiftL src2 lshift) rshift)));
11278 ins_cost(INSN_COST);
11279 format %{ "add $dst, $src1, sxth $src2" %}
11280
11281 ins_encode %{
11282 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
11283 as_Register($src2$$reg), ext::sxth);
11284 %}
11285 ins_pipe(ialu_reg_reg);
11286 %}
11287
11288 instruct AddExtL_sxtw(iRegLNoSp dst, iRegL src1, iRegL src2, immI_32 lshift, immI_32 rshift, rFlagsReg cr)
11289 %{
11290 match(Set dst (AddL src1 (RShiftL (LShiftL src2 lshift) rshift)));
11291 ins_cost(INSN_COST);
11292 format %{ "add $dst, $src1, sxtw $src2" %}
11293
11294 ins_encode %{
11295 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
11296 as_Register($src2$$reg), ext::sxtw);
11297 %}
11298 ins_pipe(ialu_reg_reg);
11299 %}
11300
11301 instruct AddExtL_sxtb(iRegLNoSp dst, iRegL src1, iRegL src2, immI_56 lshift, immI_56 rshift, rFlagsReg cr)
11302 %{
11303 match(Set dst (AddL src1 (RShiftL (LShiftL src2 lshift) rshift)));
11304 ins_cost(INSN_COST);
11305 format %{ "add $dst, $src1, sxtb $src2" %}
11306
11307 ins_encode %{
11308 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
11309 as_Register($src2$$reg), ext::sxtb);
11310 %}
11311 ins_pipe(ialu_reg_reg);
11312 %}
11313
11314 instruct AddExtL_uxtb(iRegLNoSp dst, iRegL src1, iRegL src2, immI_56 lshift, immI_56 rshift, rFlagsReg cr)
11315 %{
11316 match(Set dst (AddL src1 (URShiftL (LShiftL src2 lshift) rshift)));
11317 ins_cost(INSN_COST);
11318 format %{ "add $dst, $src1, uxtb $src2" %}
11319
11320 ins_encode %{
11321 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
11322 as_Register($src2$$reg), ext::uxtb);
11323 %}
11324 ins_pipe(ialu_reg_reg);
11325 %}
11326
11327
11328 instruct AddExtI_uxtb_and(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_255 mask, rFlagsReg cr)
11329 %{
11330 match(Set dst (AddI src1 (AndI src2 mask)));
11331 ins_cost(INSN_COST);
11332 format %{ "addw $dst, $src1, $src2, uxtb" %}
11333
11334 ins_encode %{
11335 __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
11336 as_Register($src2$$reg), ext::uxtb);
11337 %}
11338 ins_pipe(ialu_reg_reg);
11339 %}
11340
11341 instruct AddExtI_uxth_and(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_65535 mask, rFlagsReg cr)
11342 %{
11343 match(Set dst (AddI src1 (AndI src2 mask)));
11344 ins_cost(INSN_COST);
11345 format %{ "addw $dst, $src1, $src2, uxth" %}
11346
11347 ins_encode %{
11348 __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
11349 as_Register($src2$$reg), ext::uxth);
11350 %}
11351 ins_pipe(ialu_reg_reg);
11352 %}
11353
11354 instruct AddExtL_uxtb_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_255 mask, rFlagsReg cr)
11355 %{
11356 match(Set dst (AddL src1 (AndL src2 mask)));
11357 ins_cost(INSN_COST);
11358 format %{ "add $dst, $src1, $src2, uxtb" %}
11359
11360 ins_encode %{
11361 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
11362 as_Register($src2$$reg), ext::uxtb);
11363 %}
11364 ins_pipe(ialu_reg_reg);
11365 %}
11366
11367 instruct AddExtL_uxth_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_65535 mask, rFlagsReg cr)
11368 %{
11369 match(Set dst (AddL src1 (AndL src2 mask)));
11370 ins_cost(INSN_COST);
11371 format %{ "add $dst, $src1, $src2, uxth" %}
11372
11373 ins_encode %{
11374 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
11375 as_Register($src2$$reg), ext::uxth);
11376 %}
11377 ins_pipe(ialu_reg_reg);
11378 %}
11379
11380 instruct AddExtL_uxtw_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_4294967295 mask, rFlagsReg cr)
11381 %{
11382 match(Set dst (AddL src1 (AndL src2 mask)));
11383 ins_cost(INSN_COST);
11384 format %{ "add $dst, $src1, $src2, uxtw" %}
11385
11386 ins_encode %{
11387 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
11388 as_Register($src2$$reg), ext::uxtw);
11389 %}
11390 ins_pipe(ialu_reg_reg);
11391 %}
11392
11393 instruct SubExtI_uxtb_and(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_255 mask, rFlagsReg cr)
11394 %{
11395 match(Set dst (SubI src1 (AndI src2 mask)));
11396 ins_cost(INSN_COST);
11397 format %{ "subw $dst, $src1, $src2, uxtb" %}
11398
11399 ins_encode %{
11400 __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
11401 as_Register($src2$$reg), ext::uxtb);
11402 %}
11403 ins_pipe(ialu_reg_reg);
11404 %}
11405
11406 instruct SubExtI_uxth_and(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_65535 mask, rFlagsReg cr)
11407 %{
11408 match(Set dst (SubI src1 (AndI src2 mask)));
11409 ins_cost(INSN_COST);
11410 format %{ "subw $dst, $src1, $src2, uxth" %}
11411
11412 ins_encode %{
11413 __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
11414 as_Register($src2$$reg), ext::uxth);
11415 %}
11416 ins_pipe(ialu_reg_reg);
11417 %}
11418
11419 instruct SubExtL_uxtb_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_255 mask, rFlagsReg cr)
11420 %{
11421 match(Set dst (SubL src1 (AndL src2 mask)));
11422 ins_cost(INSN_COST);
11423 format %{ "sub $dst, $src1, $src2, uxtb" %}
11424
11425 ins_encode %{
11426 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
11427 as_Register($src2$$reg), ext::uxtb);
11428 %}
11429 ins_pipe(ialu_reg_reg);
11430 %}
11431
11432 instruct SubExtL_uxth_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_65535 mask, rFlagsReg cr)
11433 %{
11434 match(Set dst (SubL src1 (AndL src2 mask)));
11435 ins_cost(INSN_COST);
11436 format %{ "sub $dst, $src1, $src2, uxth" %}
11437
11438 ins_encode %{
11439 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
11440 as_Register($src2$$reg), ext::uxth);
11441 %}
11442 ins_pipe(ialu_reg_reg);
11443 %}
11444
11445 instruct SubExtL_uxtw_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_4294967295 mask, rFlagsReg cr)
11446 %{
11447 match(Set dst (SubL src1 (AndL src2 mask)));
11448 ins_cost(INSN_COST);
11449 format %{ "sub $dst, $src1, $src2, uxtw" %}
11450
11451 ins_encode %{
11452 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
11453 as_Register($src2$$reg), ext::uxtw);
11454 %}
11455 ins_pipe(ialu_reg_reg);
11456 %}
11457
11458 // END This section of the file is automatically generated. Do not edit --------------
11459
11460 // ============================================================================
11461 // Floating Point Arithmetic Instructions
11462
11463 instruct addF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
11464 match(Set dst (AddF src1 src2));
11465
11466 ins_cost(INSN_COST * 5);
11467 format %{ "fadds $dst, $src1, $src2" %}
11468
11469 ins_encode %{
11470 __ fadds(as_FloatRegister($dst$$reg),
11471 as_FloatRegister($src1$$reg),
11472 as_FloatRegister($src2$$reg));
11473 %}
11474
11475 ins_pipe(fp_dop_reg_reg_s);
11476 %}
11477
11478 instruct addD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
11479 match(Set dst (AddD src1 src2));
11480
11481 ins_cost(INSN_COST * 5);
11482 format %{ "faddd $dst, $src1, $src2" %}
11483
11484 ins_encode %{
11485 __ faddd(as_FloatRegister($dst$$reg),
11486 as_FloatRegister($src1$$reg),
11487 as_FloatRegister($src2$$reg));
11488 %}
11489
11490 ins_pipe(fp_dop_reg_reg_d);
11491 %}
11492
11493 instruct subF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
11494 match(Set dst (SubF src1 src2));
11495
11496 ins_cost(INSN_COST * 5);
11497 format %{ "fsubs $dst, $src1, $src2" %}
11498
11499 ins_encode %{
11500 __ fsubs(as_FloatRegister($dst$$reg),
11501 as_FloatRegister($src1$$reg),
11502 as_FloatRegister($src2$$reg));
11503 %}
11504
11505 ins_pipe(fp_dop_reg_reg_s);
11506 %}
11507
11508 instruct subD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
11509 match(Set dst (SubD src1 src2));
11510
11511 ins_cost(INSN_COST * 5);
11512 format %{ "fsubd $dst, $src1, $src2" %}
11513
11514 ins_encode %{
11515 __ fsubd(as_FloatRegister($dst$$reg),
11516 as_FloatRegister($src1$$reg),
11517 as_FloatRegister($src2$$reg));
11518 %}
11519
11520 ins_pipe(fp_dop_reg_reg_d);
11521 %}
11522
11523 instruct mulF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
11524 match(Set dst (MulF src1 src2));
11525
11526 ins_cost(INSN_COST * 6);
11527 format %{ "fmuls $dst, $src1, $src2" %}
11528
11529 ins_encode %{
11530 __ fmuls(as_FloatRegister($dst$$reg),
11531 as_FloatRegister($src1$$reg),
11532 as_FloatRegister($src2$$reg));
11533 %}
11534
11535 ins_pipe(fp_dop_reg_reg_s);
11536 %}
11537
11538 instruct mulD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
11539 match(Set dst (MulD src1 src2));
11540
11541 ins_cost(INSN_COST * 6);
11542 format %{ "fmuld $dst, $src1, $src2" %}
11543
11544 ins_encode %{
11545 __ fmuld(as_FloatRegister($dst$$reg),
11546 as_FloatRegister($src1$$reg),
11547 as_FloatRegister($src2$$reg));
11548 %}
11549
11550 ins_pipe(fp_dop_reg_reg_d);
11551 %}
11552
11553 // We cannot use these fused mul w add/sub ops because they don't
11554 // produce the same result as the equivalent separated ops
11555 // (essentially they don't round the intermediate result). that's a
11556 // shame. leaving them here in case we can idenitfy cases where it is
11557 // legitimate to use them
11558
11559
11560 // instruct maddF_reg_reg(vRegF dst, vRegF src1, vRegF src2, vRegF src3) %{
11561 // match(Set dst (AddF (MulF src1 src2) src3));
11562
11563 // format %{ "fmadds $dst, $src1, $src2, $src3" %}
11564
11565 // ins_encode %{
11566 // __ fmadds(as_FloatRegister($dst$$reg),
11567 // as_FloatRegister($src1$$reg),
11568 // as_FloatRegister($src2$$reg),
11569 // as_FloatRegister($src3$$reg));
11570 // %}
11571
11572 // ins_pipe(pipe_class_default);
11573 // %}
11574
11575 // instruct maddD_reg_reg(vRegD dst, vRegD src1, vRegD src2, vRegD src3) %{
11576 // match(Set dst (AddD (MulD src1 src2) src3));
11577
11578 // format %{ "fmaddd $dst, $src1, $src2, $src3" %}
11579
11580 // ins_encode %{
11581 // __ fmaddd(as_FloatRegister($dst$$reg),
11582 // as_FloatRegister($src1$$reg),
11583 // as_FloatRegister($src2$$reg),
11584 // as_FloatRegister($src3$$reg));
11585 // %}
11586
11587 // ins_pipe(pipe_class_default);
11588 // %}
11589
11590 // instruct msubF_reg_reg(vRegF dst, vRegF src1, vRegF src2, vRegF src3) %{
11591 // match(Set dst (AddF (MulF (NegF src1) src2) src3));
11592 // match(Set dst (AddF (NegF (MulF src1 src2)) src3));
11593
11594 // format %{ "fmsubs $dst, $src1, $src2, $src3" %}
11595
11596 // ins_encode %{
11597 // __ fmsubs(as_FloatRegister($dst$$reg),
11598 // as_FloatRegister($src1$$reg),
11599 // as_FloatRegister($src2$$reg),
11600 // as_FloatRegister($src3$$reg));
11601 // %}
11602
11603 // ins_pipe(pipe_class_default);
11604 // %}
11605
11606 // instruct msubD_reg_reg(vRegD dst, vRegD src1, vRegD src2, vRegD src3) %{
11607 // match(Set dst (AddD (MulD (NegD src1) src2) src3));
11608 // match(Set dst (AddD (NegD (MulD src1 src2)) src3));
11609
11610 // format %{ "fmsubd $dst, $src1, $src2, $src3" %}
11611
11612 // ins_encode %{
11613 // __ fmsubd(as_FloatRegister($dst$$reg),
11614 // as_FloatRegister($src1$$reg),
11615 // as_FloatRegister($src2$$reg),
11616 // as_FloatRegister($src3$$reg));
11617 // %}
11618
11619 // ins_pipe(pipe_class_default);
11620 // %}
11621
11622 // instruct mnaddF_reg_reg(vRegF dst, vRegF src1, vRegF src2, vRegF src3) %{
11623 // match(Set dst (SubF (MulF (NegF src1) src2) src3));
11624 // match(Set dst (SubF (NegF (MulF src1 src2)) src3));
11625
11626 // format %{ "fnmadds $dst, $src1, $src2, $src3" %}
11627
11628 // ins_encode %{
11629 // __ fnmadds(as_FloatRegister($dst$$reg),
11630 // as_FloatRegister($src1$$reg),
11631 // as_FloatRegister($src2$$reg),
11632 // as_FloatRegister($src3$$reg));
11633 // %}
11634
11635 // ins_pipe(pipe_class_default);
11636 // %}
11637
11638 // instruct mnaddD_reg_reg(vRegD dst, vRegD src1, vRegD src2, vRegD src3) %{
11639 // match(Set dst (SubD (MulD (NegD src1) src2) src3));
11640 // match(Set dst (SubD (NegD (MulD src1 src2)) src3));
11641
11642 // format %{ "fnmaddd $dst, $src1, $src2, $src3" %}
11643
11644 // ins_encode %{
11645 // __ fnmaddd(as_FloatRegister($dst$$reg),
11646 // as_FloatRegister($src1$$reg),
11647 // as_FloatRegister($src2$$reg),
11648 // as_FloatRegister($src3$$reg));
11649 // %}
11650
11651 // ins_pipe(pipe_class_default);
11652 // %}
11653
11654 // instruct mnsubF_reg_reg(vRegF dst, vRegF src1, vRegF src2, vRegF src3, immF0 zero) %{
11655 // match(Set dst (SubF (MulF src1 src2) src3));
11656
11657 // format %{ "fnmsubs $dst, $src1, $src2, $src3" %}
11658
11659 // ins_encode %{
11660 // __ fnmsubs(as_FloatRegister($dst$$reg),
11661 // as_FloatRegister($src1$$reg),
11662 // as_FloatRegister($src2$$reg),
11663 // as_FloatRegister($src3$$reg));
11664 // %}
11665
11666 // ins_pipe(pipe_class_default);
11667 // %}
11668
11669 // instruct mnsubD_reg_reg(vRegD dst, vRegD src1, vRegD src2, vRegD src3, immD0 zero) %{
11670 // match(Set dst (SubD (MulD src1 src2) src3));
11671
11672 // format %{ "fnmsubd $dst, $src1, $src2, $src3" %}
11673
11674 // ins_encode %{
11675 // // n.b. insn name should be fnmsubd
11676 // __ fnmsub(as_FloatRegister($dst$$reg),
11677 // as_FloatRegister($src1$$reg),
11678 // as_FloatRegister($src2$$reg),
11679 // as_FloatRegister($src3$$reg));
11680 // %}
11681
11682 // ins_pipe(pipe_class_default);
11683 // %}
11684
11685
11686 instruct divF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
11687 match(Set dst (DivF src1 src2));
11688
11689 ins_cost(INSN_COST * 18);
11690 format %{ "fdivs $dst, $src1, $src2" %}
11691
11692 ins_encode %{
11693 __ fdivs(as_FloatRegister($dst$$reg),
11694 as_FloatRegister($src1$$reg),
11695 as_FloatRegister($src2$$reg));
11696 %}
11697
11698 ins_pipe(fp_div_s);
11699 %}
11700
11701 instruct divD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
11702 match(Set dst (DivD src1 src2));
11703
11704 ins_cost(INSN_COST * 32);
11705 format %{ "fdivd $dst, $src1, $src2" %}
11706
11707 ins_encode %{
11708 __ fdivd(as_FloatRegister($dst$$reg),
11709 as_FloatRegister($src1$$reg),
11710 as_FloatRegister($src2$$reg));
11711 %}
11712
11713 ins_pipe(fp_div_d);
11714 %}
11715
11716 instruct negF_reg_reg(vRegF dst, vRegF src) %{
11717 match(Set dst (NegF src));
11718
11719 ins_cost(INSN_COST * 3);
11720 format %{ "fneg $dst, $src" %}
11721
11722 ins_encode %{
11723 __ fnegs(as_FloatRegister($dst$$reg),
11724 as_FloatRegister($src$$reg));
11725 %}
11726
11727 ins_pipe(fp_uop_s);
11728 %}
11729
11730 instruct negD_reg_reg(vRegD dst, vRegD src) %{
11731 match(Set dst (NegD src));
11732
11733 ins_cost(INSN_COST * 3);
11734 format %{ "fnegd $dst, $src" %}
11735
11736 ins_encode %{
11737 __ fnegd(as_FloatRegister($dst$$reg),
11738 as_FloatRegister($src$$reg));
11739 %}
11740
11741 ins_pipe(fp_uop_d);
11742 %}
11743
11744 instruct absF_reg(vRegF dst, vRegF src) %{
11745 match(Set dst (AbsF src));
11746
11747 ins_cost(INSN_COST * 3);
11748 format %{ "fabss $dst, $src" %}
11749 ins_encode %{
11750 __ fabss(as_FloatRegister($dst$$reg),
11751 as_FloatRegister($src$$reg));
11752 %}
11753
11754 ins_pipe(fp_uop_s);
11755 %}
11756
11757 instruct absD_reg(vRegD dst, vRegD src) %{
11758 match(Set dst (AbsD src));
11759
11760 ins_cost(INSN_COST * 3);
11761 format %{ "fabsd $dst, $src" %}
11762 ins_encode %{
11763 __ fabsd(as_FloatRegister($dst$$reg),
11764 as_FloatRegister($src$$reg));
11765 %}
11766
11767 ins_pipe(fp_uop_d);
11768 %}
11769
11770 instruct sqrtD_reg(vRegD dst, vRegD src) %{
11771 match(Set dst (SqrtD src));
11772
11773 ins_cost(INSN_COST * 50);
11774 format %{ "fsqrtd $dst, $src" %}
11775 ins_encode %{
11776 __ fsqrtd(as_FloatRegister($dst$$reg),
11777 as_FloatRegister($src$$reg));
11778 %}
11779
11780 ins_pipe(fp_div_s);
11781 %}
11782
11783 instruct sqrtF_reg(vRegF dst, vRegF src) %{
11784 match(Set dst (ConvD2F (SqrtD (ConvF2D src))));
11785
11786 ins_cost(INSN_COST * 50);
11787 format %{ "fsqrts $dst, $src" %}
11788 ins_encode %{
11789 __ fsqrts(as_FloatRegister($dst$$reg),
11790 as_FloatRegister($src$$reg));
11791 %}
11792
11793 ins_pipe(fp_div_d);
11794 %}
11795
11796 // ============================================================================
11797 // Logical Instructions
11798
11799 // Integer Logical Instructions
11800
11801 // And Instructions
11802
11803
11804 instruct andI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, rFlagsReg cr) %{
11805 match(Set dst (AndI src1 src2));
11806
11807 format %{ "andw $dst, $src1, $src2\t# int" %}
11808
11809 ins_cost(INSN_COST);
11810 ins_encode %{
11811 __ andw(as_Register($dst$$reg),
11812 as_Register($src1$$reg),
11813 as_Register($src2$$reg));
11814 %}
11815
11816 ins_pipe(ialu_reg_reg);
11817 %}
11818
11819 instruct andI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immILog src2, rFlagsReg cr) %{
11820 match(Set dst (AndI src1 src2));
11821
11822 format %{ "andsw $dst, $src1, $src2\t# int" %}
11823
11824 ins_cost(INSN_COST);
11825 ins_encode %{
11826 __ andw(as_Register($dst$$reg),
11827 as_Register($src1$$reg),
11828 (unsigned long)($src2$$constant));
11829 %}
11830
11831 ins_pipe(ialu_reg_imm);
11832 %}
11833
11834 // Or Instructions
11835
11836 instruct orI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
11837 match(Set dst (OrI src1 src2));
11838
11839 format %{ "orrw $dst, $src1, $src2\t# int" %}
11840
11841 ins_cost(INSN_COST);
11842 ins_encode %{
11843 __ orrw(as_Register($dst$$reg),
11844 as_Register($src1$$reg),
11845 as_Register($src2$$reg));
11846 %}
11847
11848 ins_pipe(ialu_reg_reg);
11849 %}
11850
11851 instruct orI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immILog src2) %{
11852 match(Set dst (OrI src1 src2));
11853
11854 format %{ "orrw $dst, $src1, $src2\t# int" %}
11855
11856 ins_cost(INSN_COST);
11857 ins_encode %{
11858 __ orrw(as_Register($dst$$reg),
11859 as_Register($src1$$reg),
11860 (unsigned long)($src2$$constant));
11861 %}
11862
11863 ins_pipe(ialu_reg_imm);
11864 %}
11865
11866 // Xor Instructions
11867
11868 instruct xorI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
11869 match(Set dst (XorI src1 src2));
11870
11871 format %{ "eorw $dst, $src1, $src2\t# int" %}
11872
11873 ins_cost(INSN_COST);
11874 ins_encode %{
11875 __ eorw(as_Register($dst$$reg),
11876 as_Register($src1$$reg),
11877 as_Register($src2$$reg));
11878 %}
11879
11880 ins_pipe(ialu_reg_reg);
11881 %}
11882
11883 instruct xorI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immILog src2) %{
11884 match(Set dst (XorI src1 src2));
11885
11886 format %{ "eorw $dst, $src1, $src2\t# int" %}
11887
11888 ins_cost(INSN_COST);
11889 ins_encode %{
11890 __ eorw(as_Register($dst$$reg),
11891 as_Register($src1$$reg),
11892 (unsigned long)($src2$$constant));
11893 %}
11894
11895 ins_pipe(ialu_reg_imm);
11896 %}
11897
11898 // Long Logical Instructions
11899 // TODO
11900
11901 instruct andL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2, rFlagsReg cr) %{
11902 match(Set dst (AndL src1 src2));
11903
11904 format %{ "and $dst, $src1, $src2\t# int" %}
11905
11906 ins_cost(INSN_COST);
11907 ins_encode %{
11908 __ andr(as_Register($dst$$reg),
11909 as_Register($src1$$reg),
11910 as_Register($src2$$reg));
11911 %}
11912
11913 ins_pipe(ialu_reg_reg);
11914 %}
11915
11916 instruct andL_reg_imm(iRegLNoSp dst, iRegL src1, immLLog src2, rFlagsReg cr) %{
11917 match(Set dst (AndL src1 src2));
11918
11919 format %{ "and $dst, $src1, $src2\t# int" %}
11920
11921 ins_cost(INSN_COST);
11922 ins_encode %{
11923 __ andr(as_Register($dst$$reg),
11924 as_Register($src1$$reg),
11925 (unsigned long)($src2$$constant));
11926 %}
11927
11928 ins_pipe(ialu_reg_imm);
11929 %}
11930
11931 // Or Instructions
11932
11933 instruct orL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
11934 match(Set dst (OrL src1 src2));
11935
11936 format %{ "orr $dst, $src1, $src2\t# int" %}
11937
11938 ins_cost(INSN_COST);
11939 ins_encode %{
11940 __ orr(as_Register($dst$$reg),
11941 as_Register($src1$$reg),
11942 as_Register($src2$$reg));
11943 %}
11944
11945 ins_pipe(ialu_reg_reg);
11946 %}
11947
11948 instruct orL_reg_imm(iRegLNoSp dst, iRegL src1, immLLog src2) %{
11949 match(Set dst (OrL src1 src2));
11950
11951 format %{ "orr $dst, $src1, $src2\t# int" %}
11952
11953 ins_cost(INSN_COST);
11954 ins_encode %{
11955 __ orr(as_Register($dst$$reg),
11956 as_Register($src1$$reg),
11957 (unsigned long)($src2$$constant));
11958 %}
11959
11960 ins_pipe(ialu_reg_imm);
11961 %}
11962
11963 // Xor Instructions
11964
11965 instruct xorL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
11966 match(Set dst (XorL src1 src2));
11967
11968 format %{ "eor $dst, $src1, $src2\t# int" %}
11969
11970 ins_cost(INSN_COST);
11971 ins_encode %{
11972 __ eor(as_Register($dst$$reg),
11973 as_Register($src1$$reg),
11974 as_Register($src2$$reg));
11975 %}
11976
11977 ins_pipe(ialu_reg_reg);
11978 %}
11979
11980 instruct xorL_reg_imm(iRegLNoSp dst, iRegL src1, immLLog src2) %{
11981 match(Set dst (XorL src1 src2));
11982
11983 ins_cost(INSN_COST);
11984 format %{ "eor $dst, $src1, $src2\t# int" %}
11985
11986 ins_encode %{
11987 __ eor(as_Register($dst$$reg),
11988 as_Register($src1$$reg),
11989 (unsigned long)($src2$$constant));
11990 %}
11991
11992 ins_pipe(ialu_reg_imm);
11993 %}
11994
11995 instruct convI2L_reg_reg(iRegLNoSp dst, iRegIorL2I src)
11996 %{
11997 match(Set dst (ConvI2L src));
11998
11999 ins_cost(INSN_COST);
12000 format %{ "sxtw $dst, $src\t# i2l" %}
12001 ins_encode %{
12002 __ sbfm($dst$$Register, $src$$Register, 0, 31);
12003 %}
12004 ins_pipe(ialu_reg_shift);
12005 %}
12006
12007 // this pattern occurs in bigmath arithmetic
12008 instruct convUI2L_reg_reg(iRegLNoSp dst, iRegIorL2I src, immL_32bits mask)
12009 %{
12010 match(Set dst (AndL (ConvI2L src) mask));
12011
12012 ins_cost(INSN_COST);
12013 format %{ "ubfm $dst, $src, 0, 31\t# ui2l" %}
12014 ins_encode %{
12015 __ ubfm($dst$$Register, $src$$Register, 0, 31);
12016 %}
12017
12018 ins_pipe(ialu_reg_shift);
12019 %}
12020
12021 instruct convL2I_reg(iRegINoSp dst, iRegL src) %{
12022 match(Set dst (ConvL2I src));
12023
12024 ins_cost(INSN_COST);
12025 format %{ "movw $dst, $src \t// l2i" %}
12026
12027 ins_encode %{
12028 __ movw(as_Register($dst$$reg), as_Register($src$$reg));
12029 %}
12030
12031 ins_pipe(ialu_reg);
12032 %}
12033
12034 instruct convI2B(iRegINoSp dst, iRegIorL2I src, rFlagsReg cr)
12035 %{
12036 match(Set dst (Conv2B src));
12037 effect(KILL cr);
12038
12039 format %{
12040 "cmpw $src, zr\n\t"
12041 "cset $dst, ne"
12042 %}
12043
12044 ins_encode %{
12045 __ cmpw(as_Register($src$$reg), zr);
12046 __ cset(as_Register($dst$$reg), Assembler::NE);
12047 %}
12048
12049 ins_pipe(ialu_reg);
12050 %}
12051
12052 instruct convP2B(iRegINoSp dst, iRegP src, rFlagsReg cr)
12053 %{
12054 match(Set dst (Conv2B src));
12055 effect(KILL cr);
12056
12057 format %{
12058 "cmp $src, zr\n\t"
12059 "cset $dst, ne"
12060 %}
12061
12062 ins_encode %{
12063 __ cmp(as_Register($src$$reg), zr);
12064 __ cset(as_Register($dst$$reg), Assembler::NE);
12065 %}
12066
12067 ins_pipe(ialu_reg);
12068 %}
12069
12070 instruct convD2F_reg(vRegF dst, vRegD src) %{
12071 match(Set dst (ConvD2F src));
12072
12073 ins_cost(INSN_COST * 5);
12074 format %{ "fcvtd $dst, $src \t// d2f" %}
12075
12076 ins_encode %{
12077 __ fcvtd(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg));
12078 %}
12079
12080 ins_pipe(fp_d2f);
12081 %}
12082
12083 instruct convF2D_reg(vRegD dst, vRegF src) %{
12084 match(Set dst (ConvF2D src));
12085
12086 ins_cost(INSN_COST * 5);
12087 format %{ "fcvts $dst, $src \t// f2d" %}
12088
12089 ins_encode %{
12090 __ fcvts(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg));
12091 %}
12092
12093 ins_pipe(fp_f2d);
12094 %}
12095
12096 instruct convF2I_reg_reg(iRegINoSp dst, vRegF src) %{
12097 match(Set dst (ConvF2I src));
12098
12099 ins_cost(INSN_COST * 5);
12100 format %{ "fcvtzsw $dst, $src \t// f2i" %}
12101
12102 ins_encode %{
12103 __ fcvtzsw(as_Register($dst$$reg), as_FloatRegister($src$$reg));
12104 %}
12105
12106 ins_pipe(fp_f2i);
12107 %}
12108
12109 instruct convF2L_reg_reg(iRegLNoSp dst, vRegF src) %{
12110 match(Set dst (ConvF2L src));
12111
12112 ins_cost(INSN_COST * 5);
12113 format %{ "fcvtzs $dst, $src \t// f2l" %}
12114
12115 ins_encode %{
12116 __ fcvtzs(as_Register($dst$$reg), as_FloatRegister($src$$reg));
12117 %}
12118
12119 ins_pipe(fp_f2l);
12120 %}
12121
12122 instruct convI2F_reg_reg(vRegF dst, iRegIorL2I src) %{
12123 match(Set dst (ConvI2F src));
12124
12125 ins_cost(INSN_COST * 5);
12126 format %{ "scvtfws $dst, $src \t// i2f" %}
12127
12128 ins_encode %{
12129 __ scvtfws(as_FloatRegister($dst$$reg), as_Register($src$$reg));
12130 %}
12131
12132 ins_pipe(fp_i2f);
12133 %}
12134
12135 instruct convL2F_reg_reg(vRegF dst, iRegL src) %{
12136 match(Set dst (ConvL2F src));
12137
12138 ins_cost(INSN_COST * 5);
12139 format %{ "scvtfs $dst, $src \t// l2f" %}
12140
12141 ins_encode %{
12142 __ scvtfs(as_FloatRegister($dst$$reg), as_Register($src$$reg));
12143 %}
12144
12145 ins_pipe(fp_l2f);
12146 %}
12147
12148 instruct convD2I_reg_reg(iRegINoSp dst, vRegD src) %{
12149 match(Set dst (ConvD2I src));
12150
12151 ins_cost(INSN_COST * 5);
12152 format %{ "fcvtzdw $dst, $src \t// d2i" %}
12153
12154 ins_encode %{
12155 __ fcvtzdw(as_Register($dst$$reg), as_FloatRegister($src$$reg));
12156 %}
12157
12158 ins_pipe(fp_d2i);
12159 %}
12160
12161 instruct convD2L_reg_reg(iRegLNoSp dst, vRegD src) %{
12162 match(Set dst (ConvD2L src));
12163
12164 ins_cost(INSN_COST * 5);
12165 format %{ "fcvtzd $dst, $src \t// d2l" %}
12166
12167 ins_encode %{
12168 __ fcvtzd(as_Register($dst$$reg), as_FloatRegister($src$$reg));
12169 %}
12170
12171 ins_pipe(fp_d2l);
12172 %}
12173
12174 instruct convI2D_reg_reg(vRegD dst, iRegIorL2I src) %{
12175 match(Set dst (ConvI2D src));
12176
12177 ins_cost(INSN_COST * 5);
12178 format %{ "scvtfwd $dst, $src \t// i2d" %}
12179
12180 ins_encode %{
12181 __ scvtfwd(as_FloatRegister($dst$$reg), as_Register($src$$reg));
12182 %}
12183
12184 ins_pipe(fp_i2d);
12185 %}
12186
12187 instruct convL2D_reg_reg(vRegD dst, iRegL src) %{
12188 match(Set dst (ConvL2D src));
12189
12190 ins_cost(INSN_COST * 5);
12191 format %{ "scvtfd $dst, $src \t// l2d" %}
12192
12193 ins_encode %{
12194 __ scvtfd(as_FloatRegister($dst$$reg), as_Register($src$$reg));
12195 %}
12196
12197 ins_pipe(fp_l2d);
12198 %}
12199
12200 // stack <-> reg and reg <-> reg shuffles with no conversion
12201
12202 instruct MoveF2I_stack_reg(iRegINoSp dst, stackSlotF src) %{
12203
12204 match(Set dst (MoveF2I src));
12205
12206 effect(DEF dst, USE src);
12207
12208 ins_cost(4 * INSN_COST);
12209
12210 format %{ "ldrw $dst, $src\t# MoveF2I_stack_reg" %}
12211
12212 ins_encode %{
12213 __ ldrw($dst$$Register, Address(sp, $src$$disp));
12214 %}
12215
12216 ins_pipe(iload_reg_reg);
12217
12218 %}
12219
12220 instruct MoveI2F_stack_reg(vRegF dst, stackSlotI src) %{
12221
12222 match(Set dst (MoveI2F src));
12223
12224 effect(DEF dst, USE src);
12225
12226 ins_cost(4 * INSN_COST);
12227
12228 format %{ "ldrs $dst, $src\t# MoveI2F_stack_reg" %}
12229
12230 ins_encode %{
12231 __ ldrs(as_FloatRegister($dst$$reg), Address(sp, $src$$disp));
12232 %}
12233
12234 ins_pipe(pipe_class_memory);
12235
12236 %}
12237
12238 instruct MoveD2L_stack_reg(iRegLNoSp dst, stackSlotD src) %{
12239
12240 match(Set dst (MoveD2L src));
12241
12242 effect(DEF dst, USE src);
12243
12244 ins_cost(4 * INSN_COST);
12245
12246 format %{ "ldr $dst, $src\t# MoveD2L_stack_reg" %}
12247
12248 ins_encode %{
12249 __ ldr($dst$$Register, Address(sp, $src$$disp));
12250 %}
12251
12252 ins_pipe(iload_reg_reg);
12253
12254 %}
12255
12256 instruct MoveL2D_stack_reg(vRegD dst, stackSlotL src) %{
12257
12258 match(Set dst (MoveL2D src));
12259
12260 effect(DEF dst, USE src);
12261
12262 ins_cost(4 * INSN_COST);
12263
12264 format %{ "ldrd $dst, $src\t# MoveL2D_stack_reg" %}
12265
12266 ins_encode %{
12267 __ ldrd(as_FloatRegister($dst$$reg), Address(sp, $src$$disp));
12268 %}
12269
12270 ins_pipe(pipe_class_memory);
12271
12272 %}
12273
12274 instruct MoveF2I_reg_stack(stackSlotI dst, vRegF src) %{
12275
12276 match(Set dst (MoveF2I src));
12277
12278 effect(DEF dst, USE src);
12279
12280 ins_cost(INSN_COST);
12281
12282 format %{ "strs $src, $dst\t# MoveF2I_reg_stack" %}
12283
12284 ins_encode %{
12285 __ strs(as_FloatRegister($src$$reg), Address(sp, $dst$$disp));
12286 %}
12287
12288 ins_pipe(pipe_class_memory);
12289
12290 %}
12291
12292 instruct MoveI2F_reg_stack(stackSlotF dst, iRegI src) %{
12293
12294 match(Set dst (MoveI2F src));
12295
12296 effect(DEF dst, USE src);
12297
12298 ins_cost(INSN_COST);
12299
12300 format %{ "strw $src, $dst\t# MoveI2F_reg_stack" %}
12301
12302 ins_encode %{
12303 __ strw($src$$Register, Address(sp, $dst$$disp));
12304 %}
12305
12306 ins_pipe(istore_reg_reg);
12307
12308 %}
12309
12310 instruct MoveD2L_reg_stack(stackSlotL dst, vRegD src) %{
12311
12312 match(Set dst (MoveD2L src));
12313
12314 effect(DEF dst, USE src);
12315
12316 ins_cost(INSN_COST);
12317
12318 format %{ "strd $dst, $src\t# MoveD2L_reg_stack" %}
12319
12320 ins_encode %{
12321 __ strd(as_FloatRegister($src$$reg), Address(sp, $dst$$disp));
12322 %}
12323
12324 ins_pipe(pipe_class_memory);
12325
12326 %}
12327
12328 instruct MoveL2D_reg_stack(stackSlotD dst, iRegL src) %{
12329
12330 match(Set dst (MoveL2D src));
12331
12332 effect(DEF dst, USE src);
12333
12334 ins_cost(INSN_COST);
12335
12336 format %{ "str $src, $dst\t# MoveL2D_reg_stack" %}
12337
12338 ins_encode %{
12339 __ str($src$$Register, Address(sp, $dst$$disp));
12340 %}
12341
12342 ins_pipe(istore_reg_reg);
12343
12344 %}
12345
12346 instruct MoveF2I_reg_reg(iRegINoSp dst, vRegF src) %{
12347
12348 match(Set dst (MoveF2I src));
12349
12350 effect(DEF dst, USE src);
12351
12352 ins_cost(INSN_COST);
12353
12354 format %{ "fmovs $dst, $src\t# MoveF2I_reg_reg" %}
12355
12356 ins_encode %{
12357 __ fmovs($dst$$Register, as_FloatRegister($src$$reg));
12358 %}
12359
12360 ins_pipe(fp_f2i);
12361
12362 %}
12363
12364 instruct MoveI2F_reg_reg(vRegF dst, iRegI src) %{
12365
12366 match(Set dst (MoveI2F src));
12367
12368 effect(DEF dst, USE src);
12369
12370 ins_cost(INSN_COST);
12371
12372 format %{ "fmovs $dst, $src\t# MoveI2F_reg_reg" %}
12373
12374 ins_encode %{
12375 __ fmovs(as_FloatRegister($dst$$reg), $src$$Register);
12376 %}
12377
12378 ins_pipe(fp_i2f);
12379
12380 %}
12381
12382 instruct MoveD2L_reg_reg(iRegLNoSp dst, vRegD src) %{
12383
12384 match(Set dst (MoveD2L src));
12385
12386 effect(DEF dst, USE src);
12387
12388 ins_cost(INSN_COST);
12389
12390 format %{ "fmovd $dst, $src\t# MoveD2L_reg_reg" %}
12391
12392 ins_encode %{
12393 __ fmovd($dst$$Register, as_FloatRegister($src$$reg));
12394 %}
12395
12396 ins_pipe(fp_d2l);
12397
12398 %}
12399
12400 instruct MoveL2D_reg_reg(vRegD dst, iRegL src) %{
12401
12402 match(Set dst (MoveL2D src));
12403
12404 effect(DEF dst, USE src);
12405
12406 ins_cost(INSN_COST);
12407
12408 format %{ "fmovd $dst, $src\t# MoveL2D_reg_reg" %}
12409
12410 ins_encode %{
12411 __ fmovd(as_FloatRegister($dst$$reg), $src$$Register);
12412 %}
12413
12414 ins_pipe(fp_l2d);
12415
12416 %}
12417
12418 // ============================================================================
12419 // clearing of an array
12420
12421 instruct clearArray_reg_reg(iRegL_R11 cnt, iRegP_R10 base, Universe dummy, rFlagsReg cr)
12422 %{
12423 match(Set dummy (ClearArray cnt base));
12424 effect(USE_KILL cnt, USE_KILL base, KILL cr);
12425
12426 ins_cost(4 * INSN_COST);
12427 format %{ "ClearArray $cnt, $base" %}
12428
12429 ins_encode %{
12430 __ zero_words($base$$Register, $cnt$$Register);
12431 %}
12432
12433 ins_pipe(pipe_class_memory);
12434 %}
12435
12436 instruct clearArray_imm_reg(immL cnt, iRegP_R10 base, iRegL_R11 tmp, Universe dummy, rFlagsReg cr)
12437 %{
12438 match(Set dummy (ClearArray cnt base));
12439 effect(USE_KILL base, TEMP tmp, KILL cr);
12440
12441 ins_cost(4 * INSN_COST);
12442 format %{ "ClearArray $cnt, $base" %}
12443
12444 ins_encode %{
12445 __ zero_words($base$$Register, (u_int64_t)$cnt$$constant);
12446 %}
12447
12448 ins_pipe(pipe_class_memory);
12449 %}
12450
12451 // ============================================================================
12452 // Overflow Math Instructions
12453
12454 instruct overflowAddI_reg_reg(rFlagsReg cr, iRegIorL2I op1, iRegIorL2I op2)
12455 %{
12456 match(Set cr (OverflowAddI op1 op2));
12457
12458 format %{ "cmnw $op1, $op2\t# overflow check int" %}
12459 ins_cost(INSN_COST);
12460 ins_encode %{
12461 __ cmnw($op1$$Register, $op2$$Register);
12462 %}
12463
12464 ins_pipe(icmp_reg_reg);
12465 %}
12466
12467 instruct overflowAddI_reg_imm(rFlagsReg cr, iRegIorL2I op1, immIAddSub op2)
12468 %{
12469 match(Set cr (OverflowAddI op1 op2));
12470
12471 format %{ "cmnw $op1, $op2\t# overflow check int" %}
12472 ins_cost(INSN_COST);
12473 ins_encode %{
12474 __ cmnw($op1$$Register, $op2$$constant);
12475 %}
12476
12477 ins_pipe(icmp_reg_imm);
12478 %}
12479
12480 instruct overflowAddL_reg_reg(rFlagsReg cr, iRegL op1, iRegL op2)
12481 %{
12482 match(Set cr (OverflowAddL op1 op2));
12483
12484 format %{ "cmn $op1, $op2\t# overflow check long" %}
12485 ins_cost(INSN_COST);
12486 ins_encode %{
12487 __ cmn($op1$$Register, $op2$$Register);
12488 %}
12489
12490 ins_pipe(icmp_reg_reg);
12491 %}
12492
12493 instruct overflowAddL_reg_imm(rFlagsReg cr, iRegL op1, immLAddSub op2)
12494 %{
12495 match(Set cr (OverflowAddL op1 op2));
12496
12497 format %{ "cmn $op1, $op2\t# overflow check long" %}
12498 ins_cost(INSN_COST);
12499 ins_encode %{
12500 __ cmn($op1$$Register, $op2$$constant);
12501 %}
12502
12503 ins_pipe(icmp_reg_imm);
12504 %}
12505
12506 instruct overflowSubI_reg_reg(rFlagsReg cr, iRegIorL2I op1, iRegIorL2I op2)
12507 %{
12508 match(Set cr (OverflowSubI op1 op2));
12509
12510 format %{ "cmpw $op1, $op2\t# overflow check int" %}
12511 ins_cost(INSN_COST);
12512 ins_encode %{
12513 __ cmpw($op1$$Register, $op2$$Register);
12514 %}
12515
12516 ins_pipe(icmp_reg_reg);
12517 %}
12518
12519 instruct overflowSubI_reg_imm(rFlagsReg cr, iRegIorL2I op1, immIAddSub op2)
12520 %{
12521 match(Set cr (OverflowSubI op1 op2));
12522
12523 format %{ "cmpw $op1, $op2\t# overflow check int" %}
12524 ins_cost(INSN_COST);
12525 ins_encode %{
12526 __ cmpw($op1$$Register, $op2$$constant);
12527 %}
12528
12529 ins_pipe(icmp_reg_imm);
12530 %}
12531
12532 instruct overflowSubL_reg_reg(rFlagsReg cr, iRegL op1, iRegL op2)
12533 %{
12534 match(Set cr (OverflowSubL op1 op2));
12535
12536 format %{ "cmp $op1, $op2\t# overflow check long" %}
12537 ins_cost(INSN_COST);
12538 ins_encode %{
12539 __ cmp($op1$$Register, $op2$$Register);
12540 %}
12541
12542 ins_pipe(icmp_reg_reg);
12543 %}
12544
12545 instruct overflowSubL_reg_imm(rFlagsReg cr, iRegL op1, immLAddSub op2)
12546 %{
12547 match(Set cr (OverflowSubL op1 op2));
12548
12549 format %{ "cmp $op1, $op2\t# overflow check long" %}
12550 ins_cost(INSN_COST);
12551 ins_encode %{
12552 __ cmp($op1$$Register, $op2$$constant);
12553 %}
12554
12555 ins_pipe(icmp_reg_imm);
12556 %}
12557
12558 instruct overflowNegI_reg(rFlagsReg cr, immI0 zero, iRegIorL2I op1)
12559 %{
12560 match(Set cr (OverflowSubI zero op1));
12561
12562 format %{ "cmpw zr, $op1\t# overflow check int" %}
12563 ins_cost(INSN_COST);
12564 ins_encode %{
12565 __ cmpw(zr, $op1$$Register);
12566 %}
12567
12568 ins_pipe(icmp_reg_imm);
12569 %}
12570
12571 instruct overflowNegL_reg(rFlagsReg cr, immI0 zero, iRegL op1)
12572 %{
12573 match(Set cr (OverflowSubL zero op1));
12574
12575 format %{ "cmp zr, $op1\t# overflow check long" %}
12576 ins_cost(INSN_COST);
12577 ins_encode %{
12578 __ cmp(zr, $op1$$Register);
12579 %}
12580
12581 ins_pipe(icmp_reg_imm);
12582 %}
12583
12584 instruct overflowMulI_reg(rFlagsReg cr, iRegIorL2I op1, iRegIorL2I op2)
12585 %{
12586 match(Set cr (OverflowMulI op1 op2));
12587
12588 format %{ "smull rscratch1, $op1, $op2\t# overflow check int\n\t"
12589 "cmp rscratch1, rscratch1, sxtw\n\t"
12590 "movw rscratch1, #0x80000000\n\t"
12591 "cselw rscratch1, rscratch1, zr, NE\n\t"
12592 "cmpw rscratch1, #1" %}
12593 ins_cost(5 * INSN_COST);
12594 ins_encode %{
12595 __ smull(rscratch1, $op1$$Register, $op2$$Register);
12596 __ subs(zr, rscratch1, rscratch1, ext::sxtw); // NE => overflow
12597 __ movw(rscratch1, 0x80000000); // Develop 0 (EQ),
12598 __ cselw(rscratch1, rscratch1, zr, Assembler::NE); // or 0x80000000 (NE)
12599 __ cmpw(rscratch1, 1); // 0x80000000 - 1 => VS
12600 %}
12601
12602 ins_pipe(pipe_slow);
12603 %}
12604
12605 instruct overflowMulI_reg_branch(cmpOp cmp, iRegIorL2I op1, iRegIorL2I op2, label labl, rFlagsReg cr)
12606 %{
12607 match(If cmp (OverflowMulI op1 op2));
12608 predicate(n->in(1)->as_Bool()->_test._test == BoolTest::overflow
12609 || n->in(1)->as_Bool()->_test._test == BoolTest::no_overflow);
12610 effect(USE labl, KILL cr);
12611
12612 format %{ "smull rscratch1, $op1, $op2\t# overflow check int\n\t"
12613 "cmp rscratch1, rscratch1, sxtw\n\t"
12614 "b$cmp $labl" %}
12615 ins_cost(3 * INSN_COST); // Branch is rare so treat as INSN_COST
12616 ins_encode %{
12617 Label* L = $labl$$label;
12618 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
12619 __ smull(rscratch1, $op1$$Register, $op2$$Register);
12620 __ subs(zr, rscratch1, rscratch1, ext::sxtw); // NE => overflow
12621 __ br(cond == Assembler::VS ? Assembler::NE : Assembler::EQ, *L);
12622 %}
12623
12624 ins_pipe(pipe_serial);
12625 %}
12626
12627 instruct overflowMulL_reg(rFlagsReg cr, iRegL op1, iRegL op2)
12628 %{
12629 match(Set cr (OverflowMulL op1 op2));
12630
12631 format %{ "mul rscratch1, $op1, $op2\t#overflow check long\n\t"
12632 "smulh rscratch2, $op1, $op2\n\t"
12633 "cmp rscratch2, rscratch1, ASR #63\n\t"
12634 "movw rscratch1, #0x80000000\n\t"
12635 "cselw rscratch1, rscratch1, zr, NE\n\t"
12636 "cmpw rscratch1, #1" %}
12637 ins_cost(6 * INSN_COST);
12638 ins_encode %{
12639 __ mul(rscratch1, $op1$$Register, $op2$$Register); // Result bits 0..63
12640 __ smulh(rscratch2, $op1$$Register, $op2$$Register); // Result bits 64..127
12641 __ cmp(rscratch2, rscratch1, Assembler::ASR, 63); // Top is pure sign ext
12642 __ movw(rscratch1, 0x80000000); // Develop 0 (EQ),
12643 __ cselw(rscratch1, rscratch1, zr, Assembler::NE); // or 0x80000000 (NE)
12644 __ cmpw(rscratch1, 1); // 0x80000000 - 1 => VS
12645 %}
12646
12647 ins_pipe(pipe_slow);
12648 %}
12649
12650 instruct overflowMulL_reg_branch(cmpOp cmp, iRegL op1, iRegL op2, label labl, rFlagsReg cr)
12651 %{
12652 match(If cmp (OverflowMulL op1 op2));
12653 predicate(n->in(1)->as_Bool()->_test._test == BoolTest::overflow
12654 || n->in(1)->as_Bool()->_test._test == BoolTest::no_overflow);
12655 effect(USE labl, KILL cr);
12656
12657 format %{ "mul rscratch1, $op1, $op2\t#overflow check long\n\t"
12658 "smulh rscratch2, $op1, $op2\n\t"
12659 "cmp rscratch2, rscratch1, ASR #63\n\t"
12660 "b$cmp $labl" %}
12661 ins_cost(4 * INSN_COST); // Branch is rare so treat as INSN_COST
12662 ins_encode %{
12663 Label* L = $labl$$label;
12664 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
12665 __ mul(rscratch1, $op1$$Register, $op2$$Register); // Result bits 0..63
12666 __ smulh(rscratch2, $op1$$Register, $op2$$Register); // Result bits 64..127
12667 __ cmp(rscratch2, rscratch1, Assembler::ASR, 63); // Top is pure sign ext
12668 __ br(cond == Assembler::VS ? Assembler::NE : Assembler::EQ, *L);
12669 %}
12670
12671 ins_pipe(pipe_serial);
12672 %}
12673
12674 // ============================================================================
12675 // Compare Instructions
12676
12677 instruct compI_reg_reg(rFlagsReg cr, iRegI op1, iRegI op2)
12678 %{
12679 match(Set cr (CmpI op1 op2));
12680
12681 effect(DEF cr, USE op1, USE op2);
12682
12683 ins_cost(INSN_COST);
12684 format %{ "cmpw $op1, $op2" %}
12685
12686 ins_encode(aarch64_enc_cmpw(op1, op2));
12687
12688 ins_pipe(icmp_reg_reg);
12689 %}
12690
12691 instruct compI_reg_immI0(rFlagsReg cr, iRegI op1, immI0 zero)
12692 %{
12693 match(Set cr (CmpI op1 zero));
12694
12695 effect(DEF cr, USE op1);
12696
12697 ins_cost(INSN_COST);
12698 format %{ "cmpw $op1, 0" %}
12699
12700 ins_encode(aarch64_enc_cmpw_imm_addsub(op1, zero));
12701
12702 ins_pipe(icmp_reg_imm);
12703 %}
12704
12705 instruct compI_reg_immIAddSub(rFlagsReg cr, iRegI op1, immIAddSub op2)
12706 %{
12707 match(Set cr (CmpI op1 op2));
12708
12709 effect(DEF cr, USE op1);
12710
12711 ins_cost(INSN_COST);
12712 format %{ "cmpw $op1, $op2" %}
12713
12714 ins_encode(aarch64_enc_cmpw_imm_addsub(op1, op2));
12715
12716 ins_pipe(icmp_reg_imm);
12717 %}
12718
12719 instruct compI_reg_immI(rFlagsReg cr, iRegI op1, immI op2)
12720 %{
12721 match(Set cr (CmpI op1 op2));
12722
12723 effect(DEF cr, USE op1);
12724
12725 ins_cost(INSN_COST * 2);
12726 format %{ "cmpw $op1, $op2" %}
12727
12728 ins_encode(aarch64_enc_cmpw_imm(op1, op2));
12729
12730 ins_pipe(icmp_reg_imm);
12731 %}
12732
12733 // Unsigned compare Instructions; really, same as signed compare
12734 // except it should only be used to feed an If or a CMovI which takes a
12735 // cmpOpU.
12736
12737 instruct compU_reg_reg(rFlagsRegU cr, iRegI op1, iRegI op2)
12738 %{
12739 match(Set cr (CmpU op1 op2));
12740
12741 effect(DEF cr, USE op1, USE op2);
12742
12743 ins_cost(INSN_COST);
12744 format %{ "cmpw $op1, $op2\t# unsigned" %}
12745
12746 ins_encode(aarch64_enc_cmpw(op1, op2));
12747
12748 ins_pipe(icmp_reg_reg);
12749 %}
12750
12751 instruct compU_reg_immI0(rFlagsRegU cr, iRegI op1, immI0 zero)
12752 %{
12753 match(Set cr (CmpU op1 zero));
12754
12755 effect(DEF cr, USE op1);
12756
12757 ins_cost(INSN_COST);
12758 format %{ "cmpw $op1, #0\t# unsigned" %}
12759
12760 ins_encode(aarch64_enc_cmpw_imm_addsub(op1, zero));
12761
12762 ins_pipe(icmp_reg_imm);
12763 %}
12764
12765 instruct compU_reg_immIAddSub(rFlagsRegU cr, iRegI op1, immIAddSub op2)
12766 %{
12767 match(Set cr (CmpU op1 op2));
12768
12769 effect(DEF cr, USE op1);
12770
12771 ins_cost(INSN_COST);
12772 format %{ "cmpw $op1, $op2\t# unsigned" %}
12773
12774 ins_encode(aarch64_enc_cmpw_imm_addsub(op1, op2));
12775
12776 ins_pipe(icmp_reg_imm);
12777 %}
12778
12779 instruct compU_reg_immI(rFlagsRegU cr, iRegI op1, immI op2)
12780 %{
12781 match(Set cr (CmpU op1 op2));
12782
12783 effect(DEF cr, USE op1);
12784
12785 ins_cost(INSN_COST * 2);
12786 format %{ "cmpw $op1, $op2\t# unsigned" %}
12787
12788 ins_encode(aarch64_enc_cmpw_imm(op1, op2));
12789
12790 ins_pipe(icmp_reg_imm);
12791 %}
12792
12793 instruct compL_reg_reg(rFlagsReg cr, iRegL op1, iRegL op2)
12794 %{
12795 match(Set cr (CmpL op1 op2));
12796
12797 effect(DEF cr, USE op1, USE op2);
12798
12799 ins_cost(INSN_COST);
12800 format %{ "cmp $op1, $op2" %}
12801
12802 ins_encode(aarch64_enc_cmp(op1, op2));
12803
12804 ins_pipe(icmp_reg_reg);
12805 %}
12806
12807 instruct compL_reg_immL0(rFlagsReg cr, iRegL op1, immL0 zero)
12808 %{
12809 match(Set cr (CmpL op1 zero));
12810
12811 effect(DEF cr, USE op1);
12812
12813 ins_cost(INSN_COST);
12814 format %{ "tst $op1" %}
12815
12816 ins_encode(aarch64_enc_cmp_imm_addsub(op1, zero));
12817
12818 ins_pipe(icmp_reg_imm);
12819 %}
12820
12821 instruct compL_reg_immLAddSub(rFlagsReg cr, iRegL op1, immLAddSub op2)
12822 %{
12823 match(Set cr (CmpL op1 op2));
12824
12825 effect(DEF cr, USE op1);
12826
12827 ins_cost(INSN_COST);
12828 format %{ "cmp $op1, $op2" %}
12829
12830 ins_encode(aarch64_enc_cmp_imm_addsub(op1, op2));
12831
12832 ins_pipe(icmp_reg_imm);
12833 %}
12834
12835 instruct compL_reg_immL(rFlagsReg cr, iRegL op1, immL op2)
12836 %{
12837 match(Set cr (CmpL op1 op2));
12838
12839 effect(DEF cr, USE op1);
12840
12841 ins_cost(INSN_COST * 2);
12842 format %{ "cmp $op1, $op2" %}
12843
12844 ins_encode(aarch64_enc_cmp_imm(op1, op2));
12845
12846 ins_pipe(icmp_reg_imm);
12847 %}
12848
12849 instruct compUL_reg_reg(rFlagsRegU cr, iRegL op1, iRegL op2)
12850 %{
12851 match(Set cr (CmpUL op1 op2));
12852
12853 effect(DEF cr, USE op1, USE op2);
12854
12855 ins_cost(INSN_COST);
12856 format %{ "cmp $op1, $op2" %}
12857
12858 ins_encode(aarch64_enc_cmp(op1, op2));
12859
12860 ins_pipe(icmp_reg_reg);
12861 %}
12862
12863 instruct compUL_reg_immL0(rFlagsRegU cr, iRegL op1, immL0 zero)
12864 %{
12865 match(Set cr (CmpUL op1 zero));
12866
12867 effect(DEF cr, USE op1);
12868
12869 ins_cost(INSN_COST);
12870 format %{ "tst $op1" %}
12871
12872 ins_encode(aarch64_enc_cmp_imm_addsub(op1, zero));
12873
12874 ins_pipe(icmp_reg_imm);
12875 %}
12876
12877 instruct compUL_reg_immLAddSub(rFlagsRegU cr, iRegL op1, immLAddSub op2)
12878 %{
12879 match(Set cr (CmpUL op1 op2));
12880
12881 effect(DEF cr, USE op1);
12882
12883 ins_cost(INSN_COST);
12884 format %{ "cmp $op1, $op2" %}
12885
12886 ins_encode(aarch64_enc_cmp_imm_addsub(op1, op2));
12887
12888 ins_pipe(icmp_reg_imm);
12889 %}
12890
12891 instruct compUL_reg_immL(rFlagsRegU cr, iRegL op1, immL op2)
12892 %{
12893 match(Set cr (CmpUL op1 op2));
12894
12895 effect(DEF cr, USE op1);
12896
12897 ins_cost(INSN_COST * 2);
12898 format %{ "cmp $op1, $op2" %}
12899
12900 ins_encode(aarch64_enc_cmp_imm(op1, op2));
12901
12902 ins_pipe(icmp_reg_imm);
12903 %}
12904
12905 instruct compP_reg_reg(rFlagsRegU cr, iRegP op1, iRegP op2)
12906 %{
12907 match(Set cr (CmpP op1 op2));
12908
12909 effect(DEF cr, USE op1, USE op2);
12910
12911 ins_cost(INSN_COST);
12912 format %{ "cmp $op1, $op2\t // ptr" %}
12913
12914 ins_encode(aarch64_enc_cmpp(op1, op2));
12915
12916 ins_pipe(icmp_reg_reg);
12917 %}
12918
12919 instruct compN_reg_reg(rFlagsRegU cr, iRegN op1, iRegN op2)
12920 %{
12921 match(Set cr (CmpN op1 op2));
12922
12923 effect(DEF cr, USE op1, USE op2);
12924
12925 ins_cost(INSN_COST);
12926 format %{ "cmp $op1, $op2\t // compressed ptr" %}
12927
12928 ins_encode(aarch64_enc_cmpn(op1, op2));
12929
12930 ins_pipe(icmp_reg_reg);
12931 %}
12932
12933 instruct testP_reg(rFlagsRegU cr, iRegP op1, immP0 zero)
12934 %{
12935 match(Set cr (CmpP op1 zero));
12936
12937 effect(DEF cr, USE op1, USE zero);
12938
12939 ins_cost(INSN_COST);
12940 format %{ "cmp $op1, 0\t // ptr" %}
12941
12942 ins_encode(aarch64_enc_testp(op1));
12943
12944 ins_pipe(icmp_reg_imm);
12945 %}
12946
12947 instruct testN_reg(rFlagsRegU cr, iRegN op1, immN0 zero)
12948 %{
12949 match(Set cr (CmpN op1 zero));
12950
12951 effect(DEF cr, USE op1, USE zero);
12952
12953 ins_cost(INSN_COST);
12954 format %{ "cmp $op1, 0\t // compressed ptr" %}
12955
12956 ins_encode(aarch64_enc_testn(op1));
12957
12958 ins_pipe(icmp_reg_imm);
12959 %}
12960
12961 // FP comparisons
12962 //
12963 // n.b. CmpF/CmpD set a normal flags reg which then gets compared
12964 // using normal cmpOp. See declaration of rFlagsReg for details.
12965
12966 instruct compF_reg_reg(rFlagsReg cr, vRegF src1, vRegF src2)
12967 %{
12968 match(Set cr (CmpF src1 src2));
12969
12970 ins_cost(3 * INSN_COST);
12971 format %{ "fcmps $src1, $src2" %}
12972
12973 ins_encode %{
12974 __ fcmps(as_FloatRegister($src1$$reg), as_FloatRegister($src2$$reg));
12975 %}
12976
12977 ins_pipe(pipe_class_compare);
12978 %}
12979
12980 instruct compF_reg_zero(rFlagsReg cr, vRegF src1, immF0 src2)
12981 %{
12982 match(Set cr (CmpF src1 src2));
12983
12984 ins_cost(3 * INSN_COST);
12985 format %{ "fcmps $src1, 0.0" %}
12986
12987 ins_encode %{
12988 __ fcmps(as_FloatRegister($src1$$reg), 0.0);
12989 %}
12990
12991 ins_pipe(pipe_class_compare);
12992 %}
12993 // FROM HERE
12994
12995 instruct compD_reg_reg(rFlagsReg cr, vRegD src1, vRegD src2)
12996 %{
12997 match(Set cr (CmpD src1 src2));
12998
12999 ins_cost(3 * INSN_COST);
13000 format %{ "fcmpd $src1, $src2" %}
13001
13002 ins_encode %{
13003 __ fcmpd(as_FloatRegister($src1$$reg), as_FloatRegister($src2$$reg));
13004 %}
13005
13006 ins_pipe(pipe_class_compare);
13007 %}
13008
13009 instruct compD_reg_zero(rFlagsReg cr, vRegD src1, immD0 src2)
13010 %{
13011 match(Set cr (CmpD src1 src2));
13012
13013 ins_cost(3 * INSN_COST);
13014 format %{ "fcmpd $src1, 0.0" %}
13015
13016 ins_encode %{
13017 __ fcmpd(as_FloatRegister($src1$$reg), 0.0);
13018 %}
13019
13020 ins_pipe(pipe_class_compare);
13021 %}
13022
13023 instruct compF3_reg_reg(iRegINoSp dst, vRegF src1, vRegF src2, rFlagsReg cr)
13024 %{
13025 match(Set dst (CmpF3 src1 src2));
13026 effect(KILL cr);
13027
13028 ins_cost(5 * INSN_COST);
13029 format %{ "fcmps $src1, $src2\n\t"
13030 "csinvw($dst, zr, zr, eq\n\t"
13031 "csnegw($dst, $dst, $dst, lt)"
13032 %}
13033
13034 ins_encode %{
13035 Label done;
13036 FloatRegister s1 = as_FloatRegister($src1$$reg);
13037 FloatRegister s2 = as_FloatRegister($src2$$reg);
13038 Register d = as_Register($dst$$reg);
13039 __ fcmps(s1, s2);
13040 // installs 0 if EQ else -1
13041 __ csinvw(d, zr, zr, Assembler::EQ);
13042 // keeps -1 if less or unordered else installs 1
13043 __ csnegw(d, d, d, Assembler::LT);
13044 __ bind(done);
13045 %}
13046
13047 ins_pipe(pipe_class_default);
13048
13049 %}
13050
13051 instruct compD3_reg_reg(iRegINoSp dst, vRegD src1, vRegD src2, rFlagsReg cr)
13052 %{
13053 match(Set dst (CmpD3 src1 src2));
13054 effect(KILL cr);
13055
13056 ins_cost(5 * INSN_COST);
13057 format %{ "fcmpd $src1, $src2\n\t"
13058 "csinvw($dst, zr, zr, eq\n\t"
13059 "csnegw($dst, $dst, $dst, lt)"
13060 %}
13061
13062 ins_encode %{
13063 Label done;
13064 FloatRegister s1 = as_FloatRegister($src1$$reg);
13065 FloatRegister s2 = as_FloatRegister($src2$$reg);
13066 Register d = as_Register($dst$$reg);
13067 __ fcmpd(s1, s2);
13068 // installs 0 if EQ else -1
13069 __ csinvw(d, zr, zr, Assembler::EQ);
13070 // keeps -1 if less or unordered else installs 1
13071 __ csnegw(d, d, d, Assembler::LT);
13072 __ bind(done);
13073 %}
13074 ins_pipe(pipe_class_default);
13075
13076 %}
13077
13078 instruct compF3_reg_immF0(iRegINoSp dst, vRegF src1, immF0 zero, rFlagsReg cr)
13079 %{
13080 match(Set dst (CmpF3 src1 zero));
13081 effect(KILL cr);
13082
13083 ins_cost(5 * INSN_COST);
13084 format %{ "fcmps $src1, 0.0\n\t"
13085 "csinvw($dst, zr, zr, eq\n\t"
13086 "csnegw($dst, $dst, $dst, lt)"
13087 %}
13088
13089 ins_encode %{
13090 Label done;
13091 FloatRegister s1 = as_FloatRegister($src1$$reg);
13092 Register d = as_Register($dst$$reg);
13093 __ fcmps(s1, 0.0);
13094 // installs 0 if EQ else -1
13095 __ csinvw(d, zr, zr, Assembler::EQ);
13096 // keeps -1 if less or unordered else installs 1
13097 __ csnegw(d, d, d, Assembler::LT);
13098 __ bind(done);
13099 %}
13100
13101 ins_pipe(pipe_class_default);
13102
13103 %}
13104
13105 instruct compD3_reg_immD0(iRegINoSp dst, vRegD src1, immD0 zero, rFlagsReg cr)
13106 %{
13107 match(Set dst (CmpD3 src1 zero));
13108 effect(KILL cr);
13109
13110 ins_cost(5 * INSN_COST);
13111 format %{ "fcmpd $src1, 0.0\n\t"
13112 "csinvw($dst, zr, zr, eq\n\t"
13113 "csnegw($dst, $dst, $dst, lt)"
13114 %}
13115
13116 ins_encode %{
13117 Label done;
13118 FloatRegister s1 = as_FloatRegister($src1$$reg);
13119 Register d = as_Register($dst$$reg);
13120 __ fcmpd(s1, 0.0);
13121 // installs 0 if EQ else -1
13122 __ csinvw(d, zr, zr, Assembler::EQ);
13123 // keeps -1 if less or unordered else installs 1
13124 __ csnegw(d, d, d, Assembler::LT);
13125 __ bind(done);
13126 %}
13127 ins_pipe(pipe_class_default);
13128
13129 %}
13130
13131 // Manifest a CmpL result in an integer register.
13132 // (src1 < src2) ? -1 : ((src1 > src2) ? 1 : 0)
13133 instruct cmpL3_reg_reg(iRegINoSp dst, iRegL src1, iRegL src2, rFlagsReg flags)
13134 %{
13135 match(Set dst (CmpL3 src1 src2));
13136 effect(KILL flags);
13137
13138 ins_cost(INSN_COST * 6);
13139 format %{
13140 "cmp $src1, $src2"
13141 "csetw $dst, ne"
13142 "cnegw $dst, lt"
13143 %}
13144 // format %{ "CmpL3 $dst, $src1, $src2" %}
13145 ins_encode %{
13146 __ cmp($src1$$Register, $src2$$Register);
13147 __ csetw($dst$$Register, Assembler::NE);
13148 __ cnegw($dst$$Register, $dst$$Register, Assembler::LT);
13149 %}
13150
13151 ins_pipe(ialu_reg_reg);
13152 %}
13153
13154 instruct cmpLTMask_reg_reg(iRegINoSp dst, iRegIorL2I p, iRegIorL2I q, rFlagsReg cr)
13155 %{
13156 match(Set dst (CmpLTMask p q));
13157 effect(KILL cr);
13158
13159 ins_cost(3 * INSN_COST);
13160
13161 format %{ "cmpw $p, $q\t# cmpLTMask\n\t"
13162 "csetw $dst, lt\n\t"
13163 "subw $dst, zr, $dst"
13164 %}
13165
13166 ins_encode %{
13167 __ cmpw(as_Register($p$$reg), as_Register($q$$reg));
13168 __ csetw(as_Register($dst$$reg), Assembler::LT);
13169 __ subw(as_Register($dst$$reg), zr, as_Register($dst$$reg));
13170 %}
13171
13172 ins_pipe(ialu_reg_reg);
13173 %}
13174
13175 instruct cmpLTMask_reg_zero(iRegINoSp dst, iRegIorL2I src, immI0 zero, rFlagsReg cr)
13176 %{
13177 match(Set dst (CmpLTMask src zero));
13178 effect(KILL cr);
13179
13180 ins_cost(INSN_COST);
13181
13182 format %{ "asrw $dst, $src, #31\t# cmpLTMask0" %}
13183
13184 ins_encode %{
13185 __ asrw(as_Register($dst$$reg), as_Register($src$$reg), 31);
13186 %}
13187
13188 ins_pipe(ialu_reg_shift);
13189 %}
13190
13191 // ============================================================================
13192 // Max and Min
13193
13194 instruct minI_rReg(iRegINoSp dst, iRegI src1, iRegI src2, rFlagsReg cr)
13195 %{
13196 match(Set dst (MinI src1 src2));
13197
13198 effect(DEF dst, USE src1, USE src2, KILL cr);
13199 size(8);
13200
13201 ins_cost(INSN_COST * 3);
13202 format %{
13203 "cmpw $src1 $src2\t signed int\n\t"
13204 "cselw $dst, $src1, $src2 lt\t"
13205 %}
13206
13207 ins_encode %{
13208 __ cmpw(as_Register($src1$$reg),
13209 as_Register($src2$$reg));
13210 __ cselw(as_Register($dst$$reg),
13211 as_Register($src1$$reg),
13212 as_Register($src2$$reg),
13213 Assembler::LT);
13214 %}
13215
13216 ins_pipe(ialu_reg_reg);
13217 %}
13218 // FROM HERE
13219
13220 instruct maxI_rReg(iRegINoSp dst, iRegI src1, iRegI src2, rFlagsReg cr)
13221 %{
13222 match(Set dst (MaxI src1 src2));
13223
13224 effect(DEF dst, USE src1, USE src2, KILL cr);
13225 size(8);
13226
13227 ins_cost(INSN_COST * 3);
13228 format %{
13229 "cmpw $src1 $src2\t signed int\n\t"
13230 "cselw $dst, $src1, $src2 gt\t"
13231 %}
13232
13233 ins_encode %{
13234 __ cmpw(as_Register($src1$$reg),
13235 as_Register($src2$$reg));
13236 __ cselw(as_Register($dst$$reg),
13237 as_Register($src1$$reg),
13238 as_Register($src2$$reg),
13239 Assembler::GT);
13240 %}
13241
13242 ins_pipe(ialu_reg_reg);
13243 %}
13244
13245 // ============================================================================
13246 // Branch Instructions
13247
13248 // Direct Branch.
13249 instruct branch(label lbl)
13250 %{
13251 match(Goto);
13252
13253 effect(USE lbl);
13254
13255 ins_cost(BRANCH_COST);
13256 format %{ "b $lbl" %}
13257
13258 ins_encode(aarch64_enc_b(lbl));
13259
13260 ins_pipe(pipe_branch);
13261 %}
13262
13263 // Conditional Near Branch
13264 instruct branchCon(cmpOp cmp, rFlagsReg cr, label lbl)
13265 %{
13266 // Same match rule as `branchConFar'.
13267 match(If cmp cr);
13268
13269 effect(USE lbl);
13270
13271 ins_cost(BRANCH_COST);
13272 // If set to 1 this indicates that the current instruction is a
13273 // short variant of a long branch. This avoids using this
13274 // instruction in first-pass matching. It will then only be used in
13275 // the `Shorten_branches' pass.
13276 // ins_short_branch(1);
13277 format %{ "b$cmp $lbl" %}
13278
13279 ins_encode(aarch64_enc_br_con(cmp, lbl));
13280
13281 ins_pipe(pipe_branch_cond);
13282 %}
13283
13284 // Conditional Near Branch Unsigned
13285 instruct branchConU(cmpOpU cmp, rFlagsRegU cr, label lbl)
13286 %{
13287 // Same match rule as `branchConFar'.
13288 match(If cmp cr);
13289
13290 effect(USE lbl);
13291
13292 ins_cost(BRANCH_COST);
13293 // If set to 1 this indicates that the current instruction is a
13294 // short variant of a long branch. This avoids using this
13295 // instruction in first-pass matching. It will then only be used in
13296 // the `Shorten_branches' pass.
13297 // ins_short_branch(1);
13298 format %{ "b$cmp $lbl\t# unsigned" %}
13299
13300 ins_encode(aarch64_enc_br_conU(cmp, lbl));
13301
13302 ins_pipe(pipe_branch_cond);
13303 %}
13304
13305 // Make use of CBZ and CBNZ. These instructions, as well as being
13306 // shorter than (cmp; branch), have the additional benefit of not
13307 // killing the flags.
13308
13309 instruct cmpI_imm0_branch(cmpOp cmp, iRegIorL2I op1, immI0 op2, label labl, rFlagsReg cr) %{
13310 match(If cmp (CmpI op1 op2));
13311 predicate(n->in(1)->as_Bool()->_test._test == BoolTest::ne
13312 || n->in(1)->as_Bool()->_test._test == BoolTest::eq);
13313 effect(USE labl);
13314
13315 ins_cost(BRANCH_COST);
13316 format %{ "cbw$cmp $op1, $labl" %}
13317 ins_encode %{
13318 Label* L = $labl$$label;
13319 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
13320 if (cond == Assembler::EQ)
13321 __ cbzw($op1$$Register, *L);
13322 else
13323 __ cbnzw($op1$$Register, *L);
13324 %}
13325 ins_pipe(pipe_cmp_branch);
13326 %}
13327
13328 instruct cmpL_imm0_branch(cmpOp cmp, iRegL op1, immL0 op2, label labl, rFlagsReg cr) %{
13329 match(If cmp (CmpL op1 op2));
13330 predicate(n->in(1)->as_Bool()->_test._test == BoolTest::ne
13331 || n->in(1)->as_Bool()->_test._test == BoolTest::eq);
13332 effect(USE labl);
13333
13334 ins_cost(BRANCH_COST);
13335 format %{ "cb$cmp $op1, $labl" %}
13336 ins_encode %{
13337 Label* L = $labl$$label;
13338 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
13339 if (cond == Assembler::EQ)
13340 __ cbz($op1$$Register, *L);
13341 else
13342 __ cbnz($op1$$Register, *L);
13343 %}
13344 ins_pipe(pipe_cmp_branch);
13345 %}
13346
13347 instruct cmpP_imm0_branch(cmpOp cmp, iRegP op1, immP0 op2, label labl, rFlagsReg cr) %{
13348 match(If cmp (CmpP op1 op2));
13349 predicate(n->in(1)->as_Bool()->_test._test == BoolTest::ne
13350 || n->in(1)->as_Bool()->_test._test == BoolTest::eq);
13351 effect(USE labl);
13352
13353 ins_cost(BRANCH_COST);
13354 format %{ "cb$cmp $op1, $labl" %}
13355 ins_encode %{
13356 Label* L = $labl$$label;
13357 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
13358 if (cond == Assembler::EQ)
13359 __ cbz($op1$$Register, *L);
13360 else
13361 __ cbnz($op1$$Register, *L);
13362 %}
13363 ins_pipe(pipe_cmp_branch);
13364 %}
13365
13366 instruct cmpN_imm0_branch(cmpOp cmp, iRegN op1, immN0 op2, label labl, rFlagsReg cr) %{
13367 match(If cmp (CmpN op1 op2));
13368 predicate(n->in(1)->as_Bool()->_test._test == BoolTest::ne
13369 || n->in(1)->as_Bool()->_test._test == BoolTest::eq);
13370 effect(USE labl);
13371
13372 ins_cost(BRANCH_COST);
13373 format %{ "cbw$cmp $op1, $labl" %}
13374 ins_encode %{
13375 Label* L = $labl$$label;
13376 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
13377 if (cond == Assembler::EQ)
13378 __ cbzw($op1$$Register, *L);
13379 else
13380 __ cbnzw($op1$$Register, *L);
13381 %}
13382 ins_pipe(pipe_cmp_branch);
13383 %}
13384
13385 instruct cmpP_narrowOop_imm0_branch(cmpOp cmp, iRegN oop, immP0 zero, label labl, rFlagsReg cr) %{
13386 match(If cmp (CmpP (DecodeN oop) zero));
13387 predicate(n->in(1)->as_Bool()->_test._test == BoolTest::ne
13388 || n->in(1)->as_Bool()->_test._test == BoolTest::eq);
13389 effect(USE labl);
13390
13391 ins_cost(BRANCH_COST);
13392 format %{ "cb$cmp $oop, $labl" %}
13393 ins_encode %{
13394 Label* L = $labl$$label;
13395 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
13396 if (cond == Assembler::EQ)
13397 __ cbzw($oop$$Register, *L);
13398 else
13399 __ cbnzw($oop$$Register, *L);
13400 %}
13401 ins_pipe(pipe_cmp_branch);
13402 %}
13403
13404 instruct cmpUI_imm0_branch(cmpOpU cmp, iRegIorL2I op1, immI0 op2, label labl, rFlagsRegU cr) %{
13405 match(If cmp (CmpU op1 op2));
13406 predicate(n->in(1)->as_Bool()->_test._test == BoolTest::ne
13407 || n->in(1)->as_Bool()->_test._test == BoolTest::eq
13408 || n->in(1)->as_Bool()->_test._test == BoolTest::gt
13409 || n->in(1)->as_Bool()->_test._test == BoolTest::le);
13410 effect(USE labl);
13411
13412 ins_cost(BRANCH_COST);
13413 format %{ "cbw$cmp $op1, $labl" %}
13414 ins_encode %{
13415 Label* L = $labl$$label;
13416 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
13417 if (cond == Assembler::EQ || cond == Assembler::LS)
13418 __ cbzw($op1$$Register, *L);
13419 else
13420 __ cbnzw($op1$$Register, *L);
13421 %}
13422 ins_pipe(pipe_cmp_branch);
13423 %}
13424
13425 instruct cmpUL_imm0_branch(cmpOpU cmp, iRegL op1, immL0 op2, label labl, rFlagsRegU cr) %{
13426 match(If cmp (CmpUL op1 op2));
13427 predicate(n->in(1)->as_Bool()->_test._test == BoolTest::ne
13428 || n->in(1)->as_Bool()->_test._test == BoolTest::eq
13429 || n->in(1)->as_Bool()->_test._test == BoolTest::gt
13430 || n->in(1)->as_Bool()->_test._test == BoolTest::le);
13431 effect(USE labl);
13432
13433 ins_cost(BRANCH_COST);
13434 format %{ "cb$cmp $op1, $labl" %}
13435 ins_encode %{
13436 Label* L = $labl$$label;
13437 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
13438 if (cond == Assembler::EQ || cond == Assembler::LS)
13439 __ cbz($op1$$Register, *L);
13440 else
13441 __ cbnz($op1$$Register, *L);
13442 %}
13443 ins_pipe(pipe_cmp_branch);
13444 %}
13445
13446 // Test bit and Branch
13447
13448 // Patterns for short (< 32KiB) variants
13449 instruct cmpL_branch_sign(cmpOp cmp, iRegL op1, immL0 op2, label labl) %{
13450 match(If cmp (CmpL op1 op2));
13451 predicate(n->in(1)->as_Bool()->_test._test == BoolTest::lt
13452 || n->in(1)->as_Bool()->_test._test == BoolTest::ge);
13453 effect(USE labl);
13454
13455 ins_cost(BRANCH_COST);
13456 format %{ "cb$cmp $op1, $labl # long" %}
13457 ins_encode %{
13458 Label* L = $labl$$label;
13459 Assembler::Condition cond =
13460 ((Assembler::Condition)$cmp$$cmpcode == Assembler::LT) ? Assembler::NE : Assembler::EQ;
13461 __ tbr(cond, $op1$$Register, 63, *L);
13462 %}
13463 ins_pipe(pipe_cmp_branch);
13464 ins_short_branch(1);
13465 %}
13466
13467 instruct cmpI_branch_sign(cmpOp cmp, iRegIorL2I op1, immI0 op2, label labl) %{
13468 match(If cmp (CmpI op1 op2));
13469 predicate(n->in(1)->as_Bool()->_test._test == BoolTest::lt
13470 || n->in(1)->as_Bool()->_test._test == BoolTest::ge);
13471 effect(USE labl);
13472
13473 ins_cost(BRANCH_COST);
13474 format %{ "cb$cmp $op1, $labl # int" %}
13475 ins_encode %{
13476 Label* L = $labl$$label;
13477 Assembler::Condition cond =
13478 ((Assembler::Condition)$cmp$$cmpcode == Assembler::LT) ? Assembler::NE : Assembler::EQ;
13479 __ tbr(cond, $op1$$Register, 31, *L);
13480 %}
13481 ins_pipe(pipe_cmp_branch);
13482 ins_short_branch(1);
13483 %}
13484
13485 instruct cmpL_branch_bit(cmpOp cmp, iRegL op1, immL op2, immL0 op3, label labl) %{
13486 match(If cmp (CmpL (AndL op1 op2) op3));
13487 predicate((n->in(1)->as_Bool()->_test._test == BoolTest::ne
13488 || n->in(1)->as_Bool()->_test._test == BoolTest::eq)
13489 && is_power_of_2(n->in(2)->in(1)->in(2)->get_long()));
13490 effect(USE labl);
13491
13492 ins_cost(BRANCH_COST);
13493 format %{ "tb$cmp $op1, $op2, $labl" %}
13494 ins_encode %{
13495 Label* L = $labl$$label;
13496 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
13497 int bit = exact_log2($op2$$constant);
13498 __ tbr(cond, $op1$$Register, bit, *L);
13499 %}
13500 ins_pipe(pipe_cmp_branch);
13501 ins_short_branch(1);
13502 %}
13503
13504 instruct cmpI_branch_bit(cmpOp cmp, iRegIorL2I op1, immI op2, immI0 op3, label labl) %{
13505 match(If cmp (CmpI (AndI op1 op2) op3));
13506 predicate((n->in(1)->as_Bool()->_test._test == BoolTest::ne
13507 || n->in(1)->as_Bool()->_test._test == BoolTest::eq)
13508 && is_power_of_2(n->in(2)->in(1)->in(2)->get_int()));
13509 effect(USE labl);
13510
13511 ins_cost(BRANCH_COST);
13512 format %{ "tb$cmp $op1, $op2, $labl" %}
13513 ins_encode %{
13514 Label* L = $labl$$label;
13515 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
13516 int bit = exact_log2($op2$$constant);
13517 __ tbr(cond, $op1$$Register, bit, *L);
13518 %}
13519 ins_pipe(pipe_cmp_branch);
13520 ins_short_branch(1);
13521 %}
13522
13523 // And far variants
13524 instruct far_cmpL_branch_sign(cmpOp cmp, iRegL op1, immL0 op2, label labl) %{
13525 match(If cmp (CmpL op1 op2));
13526 predicate(n->in(1)->as_Bool()->_test._test == BoolTest::lt
13527 || n->in(1)->as_Bool()->_test._test == BoolTest::ge);
13528 effect(USE labl);
13529
13530 ins_cost(BRANCH_COST);
13531 format %{ "cb$cmp $op1, $labl # long" %}
13532 ins_encode %{
13533 Label* L = $labl$$label;
13534 Assembler::Condition cond =
13535 ((Assembler::Condition)$cmp$$cmpcode == Assembler::LT) ? Assembler::NE : Assembler::EQ;
13536 __ tbr(cond, $op1$$Register, 63, *L, /*far*/true);
13537 %}
13538 ins_pipe(pipe_cmp_branch);
13539 %}
13540
13541 instruct far_cmpI_branch_sign(cmpOp cmp, iRegIorL2I op1, immI0 op2, label labl) %{
13542 match(If cmp (CmpI op1 op2));
13543 predicate(n->in(1)->as_Bool()->_test._test == BoolTest::lt
13544 || n->in(1)->as_Bool()->_test._test == BoolTest::ge);
13545 effect(USE labl);
13546
13547 ins_cost(BRANCH_COST);
13548 format %{ "cb$cmp $op1, $labl # int" %}
13549 ins_encode %{
13550 Label* L = $labl$$label;
13551 Assembler::Condition cond =
13552 ((Assembler::Condition)$cmp$$cmpcode == Assembler::LT) ? Assembler::NE : Assembler::EQ;
13553 __ tbr(cond, $op1$$Register, 31, *L, /*far*/true);
13554 %}
13555 ins_pipe(pipe_cmp_branch);
13556 %}
13557
13558 instruct far_cmpL_branch_bit(cmpOp cmp, iRegL op1, immL op2, immL0 op3, label labl) %{
13559 match(If cmp (CmpL (AndL op1 op2) op3));
13560 predicate((n->in(1)->as_Bool()->_test._test == BoolTest::ne
13561 || n->in(1)->as_Bool()->_test._test == BoolTest::eq)
13562 && is_power_of_2(n->in(2)->in(1)->in(2)->get_long()));
13563 effect(USE labl);
13564
13565 ins_cost(BRANCH_COST);
13566 format %{ "tb$cmp $op1, $op2, $labl" %}
13567 ins_encode %{
13568 Label* L = $labl$$label;
13569 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
13570 int bit = exact_log2($op2$$constant);
13571 __ tbr(cond, $op1$$Register, bit, *L, /*far*/true);
13572 %}
13573 ins_pipe(pipe_cmp_branch);
13574 %}
13575
13576 instruct far_cmpI_branch_bit(cmpOp cmp, iRegIorL2I op1, immI op2, immI0 op3, label labl) %{
13577 match(If cmp (CmpI (AndI op1 op2) op3));
13578 predicate((n->in(1)->as_Bool()->_test._test == BoolTest::ne
13579 || n->in(1)->as_Bool()->_test._test == BoolTest::eq)
13580 && is_power_of_2(n->in(2)->in(1)->in(2)->get_int()));
13581 effect(USE labl);
13582
13583 ins_cost(BRANCH_COST);
13584 format %{ "tb$cmp $op1, $op2, $labl" %}
13585 ins_encode %{
13586 Label* L = $labl$$label;
13587 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
13588 int bit = exact_log2($op2$$constant);
13589 __ tbr(cond, $op1$$Register, bit, *L, /*far*/true);
13590 %}
13591 ins_pipe(pipe_cmp_branch);
13592 %}
13593
13594 // Test bits
13595
13596 instruct cmpL_and(cmpOp cmp, iRegL op1, immL op2, immL0 op3, rFlagsReg cr) %{
13597 match(Set cr (CmpL (AndL op1 op2) op3));
13598 predicate(Assembler::operand_valid_for_logical_immediate
13599 (/*is_32*/false, n->in(1)->in(2)->get_long()));
13600
13601 ins_cost(INSN_COST);
13602 format %{ "tst $op1, $op2 # long" %}
13603 ins_encode %{
13604 __ tst($op1$$Register, $op2$$constant);
13605 %}
13606 ins_pipe(ialu_reg_reg);
13607 %}
13608
13609 instruct cmpI_and(cmpOp cmp, iRegIorL2I op1, immI op2, immI0 op3, rFlagsReg cr) %{
13610 match(Set cr (CmpI (AndI op1 op2) op3));
13611 predicate(Assembler::operand_valid_for_logical_immediate
13612 (/*is_32*/true, n->in(1)->in(2)->get_int()));
13613
13614 ins_cost(INSN_COST);
13615 format %{ "tst $op1, $op2 # int" %}
13616 ins_encode %{
13617 __ tstw($op1$$Register, $op2$$constant);
13618 %}
13619 ins_pipe(ialu_reg_reg);
13620 %}
13621
13622 instruct cmpL_and_reg(cmpOp cmp, iRegL op1, iRegL op2, immL0 op3, rFlagsReg cr) %{
13623 match(Set cr (CmpL (AndL op1 op2) op3));
13624
13625 ins_cost(INSN_COST);
13626 format %{ "tst $op1, $op2 # long" %}
13627 ins_encode %{
13628 __ tst($op1$$Register, $op2$$Register);
13629 %}
13630 ins_pipe(ialu_reg_reg);
13631 %}
13632
13633 instruct cmpI_and_reg(cmpOp cmp, iRegIorL2I op1, iRegIorL2I op2, immI0 op3, rFlagsReg cr) %{
13634 match(Set cr (CmpI (AndI op1 op2) op3));
13635
13636 ins_cost(INSN_COST);
13637 format %{ "tstw $op1, $op2 # int" %}
13638 ins_encode %{
13639 __ tstw($op1$$Register, $op2$$Register);
13640 %}
13641 ins_pipe(ialu_reg_reg);
13642 %}
13643
13644
13645 // Conditional Far Branch
13646 // Conditional Far Branch Unsigned
13647 // TODO: fixme
13648
13649 // counted loop end branch near
13650 instruct branchLoopEnd(cmpOp cmp, rFlagsReg cr, label lbl)
13651 %{
13652 match(CountedLoopEnd cmp cr);
13653
13654 effect(USE lbl);
13655
13656 ins_cost(BRANCH_COST);
13657 // short variant.
13658 // ins_short_branch(1);
13659 format %{ "b$cmp $lbl \t// counted loop end" %}
13660
13661 ins_encode(aarch64_enc_br_con(cmp, lbl));
13662
13663 ins_pipe(pipe_branch);
13664 %}
13665
13666 // counted loop end branch near Unsigned
13667 instruct branchLoopEndU(cmpOpU cmp, rFlagsRegU cr, label lbl)
13668 %{
13669 match(CountedLoopEnd cmp cr);
13670
13671 effect(USE lbl);
13672
13673 ins_cost(BRANCH_COST);
13674 // short variant.
13675 // ins_short_branch(1);
13676 format %{ "b$cmp $lbl \t// counted loop end unsigned" %}
13677
13678 ins_encode(aarch64_enc_br_conU(cmp, lbl));
13679
13680 ins_pipe(pipe_branch);
13681 %}
13682
13683 // counted loop end branch far
13684 // counted loop end branch far unsigned
13685 // TODO: fixme
13686
13687 // ============================================================================
13688 // inlined locking and unlocking
13689
13690 instruct cmpFastLock(rFlagsReg cr, iRegP object, iRegP box, iRegPNoSp tmp, iRegPNoSp tmp2)
13691 %{
13692 match(Set cr (FastLock object box));
13693 effect(TEMP tmp, TEMP tmp2);
13694
13695 // TODO
13696 // identify correct cost
13697 ins_cost(5 * INSN_COST);
13698 format %{ "fastlock $object,$box\t! kills $tmp,$tmp2" %}
13699
13700 ins_encode(aarch64_enc_fast_lock(object, box, tmp, tmp2));
13701
13702 ins_pipe(pipe_serial);
13703 %}
13704
13705 instruct cmpFastUnlock(rFlagsReg cr, iRegP object, iRegP box, iRegPNoSp tmp, iRegPNoSp tmp2)
13706 %{
13707 match(Set cr (FastUnlock object box));
13708 effect(TEMP tmp, TEMP tmp2);
13709
13710 ins_cost(5 * INSN_COST);
13711 format %{ "fastunlock $object,$box\t! kills $tmp, $tmp2" %}
13712
13713 ins_encode(aarch64_enc_fast_unlock(object, box, tmp, tmp2));
13714
13715 ins_pipe(pipe_serial);
13716 %}
13717
13718
13719 // ============================================================================
13720 // Safepoint Instructions
13721
13722 // TODO
13723 // provide a near and far version of this code
13724
13725 instruct safePoint(rFlagsReg cr, iRegP poll)
13726 %{
13727 match(SafePoint poll);
13728 effect(KILL cr);
13729
13730 format %{
13731 "ldrw zr, [$poll]\t# Safepoint: poll for GC"
13732 %}
13733 ins_encode %{
13734 __ read_polling_page(as_Register($poll$$reg), relocInfo::poll_type);
13735 %}
13736 ins_pipe(pipe_serial); // ins_pipe(iload_reg_mem);
13737 %}
13738
13739
13740 // ============================================================================
13741 // Procedure Call/Return Instructions
13742
13743 // Call Java Static Instruction
13744
13745 instruct CallStaticJavaDirect(method meth)
13746 %{
13747 match(CallStaticJava);
13748
13749 effect(USE meth);
13750
13751 predicate(!((CallStaticJavaNode*)n)->is_method_handle_invoke());
13752
13753 ins_cost(CALL_COST);
13754
13755 format %{ "call,static $meth \t// ==> " %}
13756
13757 ins_encode( aarch64_enc_java_static_call(meth),
13758 aarch64_enc_call_epilog );
13759
13760 ins_pipe(pipe_class_call);
13761 %}
13762
13763 // TO HERE
13764
13765 // Call Java Static Instruction (method handle version)
13766
13767 instruct CallStaticJavaDirectHandle(method meth, iRegP_FP reg_mh_save)
13768 %{
13769 match(CallStaticJava);
13770
13771 effect(USE meth);
13772
13773 predicate(((CallStaticJavaNode*)n)->is_method_handle_invoke());
13774
13775 ins_cost(CALL_COST);
13776
13777 format %{ "call,static $meth \t// (methodhandle) ==> " %}
13778
13779 ins_encode( aarch64_enc_java_handle_call(meth),
13780 aarch64_enc_call_epilog );
13781
13782 ins_pipe(pipe_class_call);
13783 %}
13784
13785 // Call Java Dynamic Instruction
13786 instruct CallDynamicJavaDirect(method meth)
13787 %{
13788 match(CallDynamicJava);
13789
13790 effect(USE meth);
13791
13792 ins_cost(CALL_COST);
13793
13794 format %{ "CALL,dynamic $meth \t// ==> " %}
13795
13796 ins_encode( aarch64_enc_java_dynamic_call(meth),
13797 aarch64_enc_call_epilog );
13798
13799 ins_pipe(pipe_class_call);
13800 %}
13801
13802 // Call Runtime Instruction
13803
13804 instruct CallRuntimeDirect(method meth)
13805 %{
13806 match(CallRuntime);
13807
13808 effect(USE meth);
13809
13810 ins_cost(CALL_COST);
13811
13812 format %{ "CALL, runtime $meth" %}
13813
13814 ins_encode( aarch64_enc_java_to_runtime(meth) );
13815
13816 ins_pipe(pipe_class_call);
13817 %}
13818
13819 // Call Runtime Instruction
13820
13821 instruct CallLeafDirect(method meth)
13822 %{
13823 match(CallLeaf);
13824
13825 effect(USE meth);
13826
13827 ins_cost(CALL_COST);
13828
13829 format %{ "CALL, runtime leaf $meth" %}
13830
13831 ins_encode( aarch64_enc_java_to_runtime(meth) );
13832
13833 ins_pipe(pipe_class_call);
13834 %}
13835
13836 // Call Runtime Instruction
13837
13838 instruct CallLeafNoFPDirect(method meth)
13839 %{
13840 match(CallLeafNoFP);
13841
13842 effect(USE meth);
13843
13844 ins_cost(CALL_COST);
13845
13846 format %{ "CALL, runtime leaf nofp $meth" %}
13847
13848 ins_encode( aarch64_enc_java_to_runtime(meth) );
13849
13850 ins_pipe(pipe_class_call);
13851 %}
13852
13853 // Tail Call; Jump from runtime stub to Java code.
13854 // Also known as an 'interprocedural jump'.
13855 // Target of jump will eventually return to caller.
13856 // TailJump below removes the return address.
13857 instruct TailCalljmpInd(iRegPNoSp jump_target, inline_cache_RegP method_oop)
13858 %{
13859 match(TailCall jump_target method_oop);
13860
13861 ins_cost(CALL_COST);
13862
13863 format %{ "br $jump_target\t# $method_oop holds method oop" %}
13864
13865 ins_encode(aarch64_enc_tail_call(jump_target));
13866
13867 ins_pipe(pipe_class_call);
13868 %}
13869
13870 instruct TailjmpInd(iRegPNoSp jump_target, iRegP_R0 ex_oop)
13871 %{
13872 match(TailJump jump_target ex_oop);
13873
13874 ins_cost(CALL_COST);
13875
13876 format %{ "br $jump_target\t# $ex_oop holds exception oop" %}
13877
13878 ins_encode(aarch64_enc_tail_jmp(jump_target));
13879
13880 ins_pipe(pipe_class_call);
13881 %}
13882
13883 // Create exception oop: created by stack-crawling runtime code.
13884 // Created exception is now available to this handler, and is setup
13885 // just prior to jumping to this handler. No code emitted.
13886 // TODO check
13887 // should ex_oop be in r0? intel uses rax, ppc cannot use r0 so uses rarg1
13888 instruct CreateException(iRegP_R0 ex_oop)
13889 %{
13890 match(Set ex_oop (CreateEx));
13891
13892 format %{ " -- \t// exception oop; no code emitted" %}
13893
13894 size(0);
13895
13896 ins_encode( /*empty*/ );
13897
13898 ins_pipe(pipe_class_empty);
13899 %}
13900
13901 // Rethrow exception: The exception oop will come in the first
13902 // argument position. Then JUMP (not call) to the rethrow stub code.
13903 instruct RethrowException() %{
13904 match(Rethrow);
13905 ins_cost(CALL_COST);
13906
13907 format %{ "b rethrow_stub" %}
13908
13909 ins_encode( aarch64_enc_rethrow() );
13910
13911 ins_pipe(pipe_class_call);
13912 %}
13913
13914
13915 // Return Instruction
13916 // epilog node loads ret address into lr as part of frame pop
13917 instruct Ret()
13918 %{
13919 match(Return);
13920
13921 format %{ "ret\t// return register" %}
13922
13923 ins_encode( aarch64_enc_ret() );
13924
13925 ins_pipe(pipe_branch);
13926 %}
13927
13928 // Die now.
13929 instruct ShouldNotReachHere() %{
13930 match(Halt);
13931
13932 ins_cost(CALL_COST);
13933 format %{ "ShouldNotReachHere" %}
13934
13935 ins_encode %{
13936 // TODO
13937 // implement proper trap call here
13938 __ brk(999);
13939 %}
13940
13941 ins_pipe(pipe_class_default);
13942 %}
13943
13944 // ============================================================================
13945 // Partial Subtype Check
13946 //
13947 // superklass array for an instance of the superklass. Set a hidden
13948 // internal cache on a hit (cache is checked with exposed code in
13949 // gen_subtype_check()). Return NZ for a miss or zero for a hit. The
13950 // encoding ALSO sets flags.
13951
13952 instruct partialSubtypeCheck(iRegP_R4 sub, iRegP_R0 super, iRegP_R2 temp, iRegP_R5 result, rFlagsReg cr)
13953 %{
13954 match(Set result (PartialSubtypeCheck sub super));
13955 effect(KILL cr, KILL temp);
13956
13957 ins_cost(1100); // slightly larger than the next version
13958 format %{ "partialSubtypeCheck $result, $sub, $super" %}
13959
13960 ins_encode(aarch64_enc_partial_subtype_check(sub, super, temp, result));
13961
13962 opcode(0x1); // Force zero of result reg on hit
13963
13964 ins_pipe(pipe_class_memory);
13965 %}
13966
13967 instruct partialSubtypeCheckVsZero(iRegP_R4 sub, iRegP_R0 super, iRegP_R2 temp, iRegP_R5 result, immP0 zero, rFlagsReg cr)
13968 %{
13969 match(Set cr (CmpP (PartialSubtypeCheck sub super) zero));
13970 effect(KILL temp, KILL result);
13971
13972 ins_cost(1100); // slightly larger than the next version
13973 format %{ "partialSubtypeCheck $result, $sub, $super == 0" %}
13974
13975 ins_encode(aarch64_enc_partial_subtype_check(sub, super, temp, result));
13976
13977 opcode(0x0); // Don't zero result reg on hit
13978
13979 ins_pipe(pipe_class_memory);
13980 %}
13981
13982 instruct string_compare(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
13983 iRegI_R0 result, iRegP_R10 tmp1, rFlagsReg cr)
13984 %{
13985 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
13986 effect(KILL tmp1, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
13987
13988 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # KILL $tmp1" %}
13989 ins_encode %{
13990 __ string_compare($str1$$Register, $str2$$Register,
13991 $cnt1$$Register, $cnt2$$Register, $result$$Register,
13992 $tmp1$$Register);
13993 %}
13994 ins_pipe(pipe_class_memory);
13995 %}
13996
13997 instruct string_indexof(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2, iRegI_R2 cnt2,
13998 iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2, iRegINoSp tmp3, iRegINoSp tmp4, rFlagsReg cr)
13999 %{
14000 match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 cnt2)));
14001 effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2,
14002 TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, KILL cr);
14003 format %{ "String IndexOf $str1,$cnt1,$str2,$cnt2 -> $result" %}
14004
14005 ins_encode %{
14006 __ string_indexof($str1$$Register, $str2$$Register,
14007 $cnt1$$Register, $cnt2$$Register,
14008 $tmp1$$Register, $tmp2$$Register,
14009 $tmp3$$Register, $tmp4$$Register,
14010 -1, $result$$Register);
14011 %}
14012 ins_pipe(pipe_class_memory);
14013 %}
14014
14015 instruct string_indexof_con(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2,
14016 immI_le_4 int_cnt2, iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2,
14017 iRegINoSp tmp3, iRegINoSp tmp4, rFlagsReg cr)
14018 %{
14019 match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 int_cnt2)));
14020 effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1,
14021 TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, KILL cr);
14022 format %{ "String IndexOf $str1,$cnt1,$str2,$int_cnt2 -> $result" %}
14023
14024 ins_encode %{
14025 int icnt2 = (int)$int_cnt2$$constant;
14026 __ string_indexof($str1$$Register, $str2$$Register,
14027 $cnt1$$Register, zr,
14028 $tmp1$$Register, $tmp2$$Register,
14029 $tmp3$$Register, $tmp4$$Register,
14030 icnt2, $result$$Register);
14031 %}
14032 ins_pipe(pipe_class_memory);
14033 %}
14034
14035 instruct string_equals(iRegP_R1 str1, iRegP_R3 str2, iRegI_R4 cnt,
14036 iRegI_R0 result, iRegP_R10 tmp, rFlagsReg cr)
14037 %{
14038 match(Set result (StrEquals (Binary str1 str2) cnt));
14039 effect(KILL tmp, USE_KILL str1, USE_KILL str2, USE_KILL cnt, KILL cr);
14040
14041 format %{ "String Equals $str1,$str2,$cnt -> $result // KILL $tmp" %}
14042 ins_encode %{
14043 __ string_equals($str1$$Register, $str2$$Register,
14044 $cnt$$Register, $result$$Register,
14045 $tmp$$Register);
14046 %}
14047 ins_pipe(pipe_class_memory);
14048 %}
14049
14050 instruct array_equals(iRegP_R1 ary1, iRegP_R2 ary2, iRegI_R0 result,
14051 iRegP_R10 tmp, rFlagsReg cr)
14052 %{
14053 match(Set result (AryEq ary1 ary2));
14054 effect(KILL tmp, USE_KILL ary1, USE_KILL ary2, KILL cr);
14055
14056 format %{ "Array Equals $ary1,ary2 -> $result // KILL $tmp" %}
14057 ins_encode %{
14058 __ char_arrays_equals($ary1$$Register, $ary2$$Register,
14059 $result$$Register, $tmp$$Register);
14060 %}
14061 ins_pipe(pipe_class_memory);
14062 %}
14063
14064 // encode char[] to byte[] in ISO_8859_1
14065 instruct encode_iso_array(iRegP_R2 src, iRegP_R1 dst, iRegI_R3 len,
14066 vRegD_V0 Vtmp1, vRegD_V1 Vtmp2,
14067 vRegD_V2 Vtmp3, vRegD_V3 Vtmp4,
14068 iRegI_R0 result, rFlagsReg cr)
14069 %{
14070 match(Set result (EncodeISOArray src (Binary dst len)));
14071 effect(USE_KILL src, USE_KILL dst, USE_KILL len,
14072 KILL Vtmp1, KILL Vtmp2, KILL Vtmp3, KILL Vtmp4, KILL cr);
14073
14074 format %{ "Encode array $src,$dst,$len -> $result" %}
14075 ins_encode %{
14076 __ encode_iso_array($src$$Register, $dst$$Register, $len$$Register,
14077 $result$$Register, $Vtmp1$$FloatRegister, $Vtmp2$$FloatRegister,
14078 $Vtmp3$$FloatRegister, $Vtmp4$$FloatRegister);
14079 %}
14080 ins_pipe( pipe_class_memory );
14081 %}
14082
14083 // ============================================================================
14084 // This name is KNOWN by the ADLC and cannot be changed.
14085 // The ADLC forces a 'TypeRawPtr::BOTTOM' output type
14086 // for this guy.
14087 instruct tlsLoadP(thread_RegP dst)
14088 %{
14089 match(Set dst (ThreadLocal));
14090
14091 ins_cost(0);
14092
14093 format %{ " -- \t// $dst=Thread::current(), empty" %}
14094
14095 size(0);
14096
14097 ins_encode( /*empty*/ );
14098
14099 ins_pipe(pipe_class_empty);
14100 %}
14101
14102 // ====================VECTOR INSTRUCTIONS=====================================
14103
14104 // Load vector (32 bits)
14105 instruct loadV4(vecD dst, vmem4 mem)
14106 %{
14107 predicate(n->as_LoadVector()->memory_size() == 4);
14108 match(Set dst (LoadVector mem));
14109 ins_cost(4 * INSN_COST);
14110 format %{ "ldrs $dst,$mem\t# vector (32 bits)" %}
14111 ins_encode( aarch64_enc_ldrvS(dst, mem) );
14112 ins_pipe(vload_reg_mem64);
14113 %}
14114
14115 // Load vector (64 bits)
14116 instruct loadV8(vecD dst, vmem8 mem)
14117 %{
14118 predicate(n->as_LoadVector()->memory_size() == 8);
14119 match(Set dst (LoadVector mem));
14120 ins_cost(4 * INSN_COST);
14121 format %{ "ldrd $dst,$mem\t# vector (64 bits)" %}
14122 ins_encode( aarch64_enc_ldrvD(dst, mem) );
14123 ins_pipe(vload_reg_mem64);
14124 %}
14125
14126 // Load Vector (128 bits)
14127 instruct loadV16(vecX dst, vmem16 mem)
14128 %{
14129 predicate(n->as_LoadVector()->memory_size() == 16);
14130 match(Set dst (LoadVector mem));
14131 ins_cost(4 * INSN_COST);
14132 format %{ "ldrq $dst,$mem\t# vector (128 bits)" %}
14133 ins_encode( aarch64_enc_ldrvQ(dst, mem) );
14134 ins_pipe(vload_reg_mem128);
14135 %}
14136
14137 // Store Vector (32 bits)
14138 instruct storeV4(vecD src, vmem4 mem)
14139 %{
14140 predicate(n->as_StoreVector()->memory_size() == 4);
14141 match(Set mem (StoreVector mem src));
14142 ins_cost(4 * INSN_COST);
14143 format %{ "strs $mem,$src\t# vector (32 bits)" %}
14144 ins_encode( aarch64_enc_strvS(src, mem) );
14145 ins_pipe(vstore_reg_mem64);
14146 %}
14147
14148 // Store Vector (64 bits)
14149 instruct storeV8(vecD src, vmem8 mem)
14150 %{
14151 predicate(n->as_StoreVector()->memory_size() == 8);
14152 match(Set mem (StoreVector mem src));
14153 ins_cost(4 * INSN_COST);
14154 format %{ "strd $mem,$src\t# vector (64 bits)" %}
14155 ins_encode( aarch64_enc_strvD(src, mem) );
14156 ins_pipe(vstore_reg_mem64);
14157 %}
14158
14159 // Store Vector (128 bits)
14160 instruct storeV16(vecX src, vmem16 mem)
14161 %{
14162 predicate(n->as_StoreVector()->memory_size() == 16);
14163 match(Set mem (StoreVector mem src));
14164 ins_cost(4 * INSN_COST);
14165 format %{ "strq $mem,$src\t# vector (128 bits)" %}
14166 ins_encode( aarch64_enc_strvQ(src, mem) );
14167 ins_pipe(vstore_reg_mem128);
14168 %}
14169
14170 instruct replicate8B(vecD dst, iRegIorL2I src)
14171 %{
14172 predicate(n->as_Vector()->length() == 4 ||
14173 n->as_Vector()->length() == 8);
14174 match(Set dst (ReplicateB src));
14175 ins_cost(INSN_COST);
14176 format %{ "dup $dst, $src\t# vector (8B)" %}
14177 ins_encode %{
14178 __ dup(as_FloatRegister($dst$$reg), __ T8B, as_Register($src$$reg));
14179 %}
14180 ins_pipe(vdup_reg_reg64);
14181 %}
14182
14183 instruct replicate16B(vecX dst, iRegIorL2I src)
14184 %{
14185 predicate(n->as_Vector()->length() == 16);
14186 match(Set dst (ReplicateB src));
14187 ins_cost(INSN_COST);
14188 format %{ "dup $dst, $src\t# vector (16B)" %}
14189 ins_encode %{
14190 __ dup(as_FloatRegister($dst$$reg), __ T16B, as_Register($src$$reg));
14191 %}
14192 ins_pipe(vdup_reg_reg128);
14193 %}
14194
14195 instruct replicate8B_imm(vecD dst, immI con)
14196 %{
14197 predicate(n->as_Vector()->length() == 4 ||
14198 n->as_Vector()->length() == 8);
14199 match(Set dst (ReplicateB con));
14200 ins_cost(INSN_COST);
14201 format %{ "movi $dst, $con\t# vector(8B)" %}
14202 ins_encode %{
14203 __ mov(as_FloatRegister($dst$$reg), __ T8B, $con$$constant & 0xff);
14204 %}
14205 ins_pipe(vmovi_reg_imm64);
14206 %}
14207
14208 instruct replicate16B_imm(vecX dst, immI con)
14209 %{
14210 predicate(n->as_Vector()->length() == 16);
14211 match(Set dst (ReplicateB con));
14212 ins_cost(INSN_COST);
14213 format %{ "movi $dst, $con\t# vector(16B)" %}
14214 ins_encode %{
14215 __ mov(as_FloatRegister($dst$$reg), __ T16B, $con$$constant & 0xff);
14216 %}
14217 ins_pipe(vmovi_reg_imm128);
14218 %}
14219
14220 instruct replicate4S(vecD dst, iRegIorL2I src)
14221 %{
14222 predicate(n->as_Vector()->length() == 2 ||
14223 n->as_Vector()->length() == 4);
14224 match(Set dst (ReplicateS src));
14225 ins_cost(INSN_COST);
14226 format %{ "dup $dst, $src\t# vector (4S)" %}
14227 ins_encode %{
14228 __ dup(as_FloatRegister($dst$$reg), __ T4H, as_Register($src$$reg));
14229 %}
14230 ins_pipe(vdup_reg_reg64);
14231 %}
14232
14233 instruct replicate8S(vecX dst, iRegIorL2I src)
14234 %{
14235 predicate(n->as_Vector()->length() == 8);
14236 match(Set dst (ReplicateS src));
14237 ins_cost(INSN_COST);
14238 format %{ "dup $dst, $src\t# vector (8S)" %}
14239 ins_encode %{
14240 __ dup(as_FloatRegister($dst$$reg), __ T8H, as_Register($src$$reg));
14241 %}
14242 ins_pipe(vdup_reg_reg128);
14243 %}
14244
14245 instruct replicate4S_imm(vecD dst, immI con)
14246 %{
14247 predicate(n->as_Vector()->length() == 2 ||
14248 n->as_Vector()->length() == 4);
14249 match(Set dst (ReplicateS con));
14250 ins_cost(INSN_COST);
14251 format %{ "movi $dst, $con\t# vector(4H)" %}
14252 ins_encode %{
14253 __ mov(as_FloatRegister($dst$$reg), __ T4H, $con$$constant & 0xffff);
14254 %}
14255 ins_pipe(vmovi_reg_imm64);
14256 %}
14257
14258 instruct replicate8S_imm(vecX dst, immI con)
14259 %{
14260 predicate(n->as_Vector()->length() == 8);
14261 match(Set dst (ReplicateS con));
14262 ins_cost(INSN_COST);
14263 format %{ "movi $dst, $con\t# vector(8H)" %}
14264 ins_encode %{
14265 __ mov(as_FloatRegister($dst$$reg), __ T8H, $con$$constant & 0xffff);
14266 %}
14267 ins_pipe(vmovi_reg_imm128);
14268 %}
14269
14270 instruct replicate2I(vecD dst, iRegIorL2I src)
14271 %{
14272 predicate(n->as_Vector()->length() == 2);
14273 match(Set dst (ReplicateI src));
14274 ins_cost(INSN_COST);
14275 format %{ "dup $dst, $src\t# vector (2I)" %}
14276 ins_encode %{
14277 __ dup(as_FloatRegister($dst$$reg), __ T2S, as_Register($src$$reg));
14278 %}
14279 ins_pipe(vdup_reg_reg64);
14280 %}
14281
14282 instruct replicate4I(vecX dst, iRegIorL2I src)
14283 %{
14284 predicate(n->as_Vector()->length() == 4);
14285 match(Set dst (ReplicateI src));
14286 ins_cost(INSN_COST);
14287 format %{ "dup $dst, $src\t# vector (4I)" %}
14288 ins_encode %{
14289 __ dup(as_FloatRegister($dst$$reg), __ T4S, as_Register($src$$reg));
14290 %}
14291 ins_pipe(vdup_reg_reg128);
14292 %}
14293
14294 instruct replicate2I_imm(vecD dst, immI con)
14295 %{
14296 predicate(n->as_Vector()->length() == 2);
14297 match(Set dst (ReplicateI con));
14298 ins_cost(INSN_COST);
14299 format %{ "movi $dst, $con\t# vector(2I)" %}
14300 ins_encode %{
14301 __ mov(as_FloatRegister($dst$$reg), __ T2S, $con$$constant);
14302 %}
14303 ins_pipe(vmovi_reg_imm64);
14304 %}
14305
14306 instruct replicate4I_imm(vecX dst, immI con)
14307 %{
14308 predicate(n->as_Vector()->length() == 4);
14309 match(Set dst (ReplicateI con));
14310 ins_cost(INSN_COST);
14311 format %{ "movi $dst, $con\t# vector(4I)" %}
14312 ins_encode %{
14313 __ mov(as_FloatRegister($dst$$reg), __ T4S, $con$$constant);
14314 %}
14315 ins_pipe(vmovi_reg_imm128);
14316 %}
14317
14318 instruct replicate2L(vecX dst, iRegL src)
14319 %{
14320 predicate(n->as_Vector()->length() == 2);
14321 match(Set dst (ReplicateL src));
14322 ins_cost(INSN_COST);
14323 format %{ "dup $dst, $src\t# vector (2L)" %}
14324 ins_encode %{
14325 __ dup(as_FloatRegister($dst$$reg), __ T2D, as_Register($src$$reg));
14326 %}
14327 ins_pipe(vdup_reg_reg128);
14328 %}
14329
14330 instruct replicate2L_zero(vecX dst, immI0 zero)
14331 %{
14332 predicate(n->as_Vector()->length() == 2);
14333 match(Set dst (ReplicateI zero));
14334 ins_cost(INSN_COST);
14335 format %{ "movi $dst, $zero\t# vector(4I)" %}
14336 ins_encode %{
14337 __ eor(as_FloatRegister($dst$$reg), __ T16B,
14338 as_FloatRegister($dst$$reg),
14339 as_FloatRegister($dst$$reg));
14340 %}
14341 ins_pipe(vmovi_reg_imm128);
14342 %}
14343
14344 instruct replicate2F(vecD dst, vRegF src)
14345 %{
14346 predicate(n->as_Vector()->length() == 2);
14347 match(Set dst (ReplicateF src));
14348 ins_cost(INSN_COST);
14349 format %{ "dup $dst, $src\t# vector (2F)" %}
14350 ins_encode %{
14351 __ dup(as_FloatRegister($dst$$reg), __ T2S,
14352 as_FloatRegister($src$$reg));
14353 %}
14354 ins_pipe(vdup_reg_freg64);
14355 %}
14356
14357 instruct replicate4F(vecX dst, vRegF src)
14358 %{
14359 predicate(n->as_Vector()->length() == 4);
14360 match(Set dst (ReplicateF src));
14361 ins_cost(INSN_COST);
14362 format %{ "dup $dst, $src\t# vector (4F)" %}
14363 ins_encode %{
14364 __ dup(as_FloatRegister($dst$$reg), __ T4S,
14365 as_FloatRegister($src$$reg));
14366 %}
14367 ins_pipe(vdup_reg_freg128);
14368 %}
14369
14370 instruct replicate2D(vecX dst, vRegD src)
14371 %{
14372 predicate(n->as_Vector()->length() == 2);
14373 match(Set dst (ReplicateD src));
14374 ins_cost(INSN_COST);
14375 format %{ "dup $dst, $src\t# vector (2D)" %}
14376 ins_encode %{
14377 __ dup(as_FloatRegister($dst$$reg), __ T2D,
14378 as_FloatRegister($src$$reg));
14379 %}
14380 ins_pipe(vdup_reg_dreg128);
14381 %}
14382
14383 // ====================VECTOR ARITHMETIC=======================================
14384
14385 // --------------------------------- ADD --------------------------------------
14386
14387 instruct vadd8B(vecD dst, vecD src1, vecD src2)
14388 %{
14389 predicate(n->as_Vector()->length() == 4 ||
14390 n->as_Vector()->length() == 8);
14391 match(Set dst (AddVB src1 src2));
14392 ins_cost(INSN_COST);
14393 format %{ "addv $dst,$src1,$src2\t# vector (8B)" %}
14394 ins_encode %{
14395 __ addv(as_FloatRegister($dst$$reg), __ T8B,
14396 as_FloatRegister($src1$$reg),
14397 as_FloatRegister($src2$$reg));
14398 %}
14399 ins_pipe(vdop64);
14400 %}
14401
14402 instruct vadd16B(vecX dst, vecX src1, vecX src2)
14403 %{
14404 predicate(n->as_Vector()->length() == 16);
14405 match(Set dst (AddVB src1 src2));
14406 ins_cost(INSN_COST);
14407 format %{ "addv $dst,$src1,$src2\t# vector (16B)" %}
14408 ins_encode %{
14409 __ addv(as_FloatRegister($dst$$reg), __ T16B,
14410 as_FloatRegister($src1$$reg),
14411 as_FloatRegister($src2$$reg));
14412 %}
14413 ins_pipe(vdop128);
14414 %}
14415
14416 instruct vadd4S(vecD dst, vecD src1, vecD src2)
14417 %{
14418 predicate(n->as_Vector()->length() == 2 ||
14419 n->as_Vector()->length() == 4);
14420 match(Set dst (AddVS src1 src2));
14421 ins_cost(INSN_COST);
14422 format %{ "addv $dst,$src1,$src2\t# vector (4H)" %}
14423 ins_encode %{
14424 __ addv(as_FloatRegister($dst$$reg), __ T4H,
14425 as_FloatRegister($src1$$reg),
14426 as_FloatRegister($src2$$reg));
14427 %}
14428 ins_pipe(vdop64);
14429 %}
14430
14431 instruct vadd8S(vecX dst, vecX src1, vecX src2)
14432 %{
14433 predicate(n->as_Vector()->length() == 8);
14434 match(Set dst (AddVS src1 src2));
14435 ins_cost(INSN_COST);
14436 format %{ "addv $dst,$src1,$src2\t# vector (8H)" %}
14437 ins_encode %{
14438 __ addv(as_FloatRegister($dst$$reg), __ T8H,
14439 as_FloatRegister($src1$$reg),
14440 as_FloatRegister($src2$$reg));
14441 %}
14442 ins_pipe(vdop128);
14443 %}
14444
14445 instruct vadd2I(vecD dst, vecD src1, vecD src2)
14446 %{
14447 predicate(n->as_Vector()->length() == 2);
14448 match(Set dst (AddVI src1 src2));
14449 ins_cost(INSN_COST);
14450 format %{ "addv $dst,$src1,$src2\t# vector (2S)" %}
14451 ins_encode %{
14452 __ addv(as_FloatRegister($dst$$reg), __ T2S,
14453 as_FloatRegister($src1$$reg),
14454 as_FloatRegister($src2$$reg));
14455 %}
14456 ins_pipe(vdop64);
14457 %}
14458
14459 instruct vadd4I(vecX dst, vecX src1, vecX src2)
14460 %{
14461 predicate(n->as_Vector()->length() == 4);
14462 match(Set dst (AddVI src1 src2));
14463 ins_cost(INSN_COST);
14464 format %{ "addv $dst,$src1,$src2\t# vector (4S)" %}
14465 ins_encode %{
14466 __ addv(as_FloatRegister($dst$$reg), __ T4S,
14467 as_FloatRegister($src1$$reg),
14468 as_FloatRegister($src2$$reg));
14469 %}
14470 ins_pipe(vdop128);
14471 %}
14472
14473 instruct vadd2L(vecX dst, vecX src1, vecX src2)
14474 %{
14475 predicate(n->as_Vector()->length() == 2);
14476 match(Set dst (AddVL src1 src2));
14477 ins_cost(INSN_COST);
14478 format %{ "addv $dst,$src1,$src2\t# vector (2L)" %}
14479 ins_encode %{
14480 __ addv(as_FloatRegister($dst$$reg), __ T2D,
14481 as_FloatRegister($src1$$reg),
14482 as_FloatRegister($src2$$reg));
14483 %}
14484 ins_pipe(vdop128);
14485 %}
14486
14487 instruct vadd2F(vecD dst, vecD src1, vecD src2)
14488 %{
14489 predicate(n->as_Vector()->length() == 2);
14490 match(Set dst (AddVF src1 src2));
14491 ins_cost(INSN_COST);
14492 format %{ "fadd $dst,$src1,$src2\t# vector (2S)" %}
14493 ins_encode %{
14494 __ fadd(as_FloatRegister($dst$$reg), __ T2S,
14495 as_FloatRegister($src1$$reg),
14496 as_FloatRegister($src2$$reg));
14497 %}
14498 ins_pipe(vdop_fp64);
14499 %}
14500
14501 instruct vadd4F(vecX dst, vecX src1, vecX src2)
14502 %{
14503 predicate(n->as_Vector()->length() == 4);
14504 match(Set dst (AddVF src1 src2));
14505 ins_cost(INSN_COST);
14506 format %{ "fadd $dst,$src1,$src2\t# vector (4S)" %}
14507 ins_encode %{
14508 __ fadd(as_FloatRegister($dst$$reg), __ T4S,
14509 as_FloatRegister($src1$$reg),
14510 as_FloatRegister($src2$$reg));
14511 %}
14512 ins_pipe(vdop_fp128);
14513 %}
14514
14515 instruct vadd2D(vecX dst, vecX src1, vecX src2)
14516 %{
14517 match(Set dst (AddVD src1 src2));
14518 ins_cost(INSN_COST);
14519 format %{ "fadd $dst,$src1,$src2\t# vector (2D)" %}
14520 ins_encode %{
14521 __ fadd(as_FloatRegister($dst$$reg), __ T2D,
14522 as_FloatRegister($src1$$reg),
14523 as_FloatRegister($src2$$reg));
14524 %}
14525 ins_pipe(vdop_fp128);
14526 %}
14527
14528 // --------------------------------- SUB --------------------------------------
14529
14530 instruct vsub8B(vecD dst, vecD src1, vecD src2)
14531 %{
14532 predicate(n->as_Vector()->length() == 4 ||
14533 n->as_Vector()->length() == 8);
14534 match(Set dst (SubVB src1 src2));
14535 ins_cost(INSN_COST);
14536 format %{ "subv $dst,$src1,$src2\t# vector (8B)" %}
14537 ins_encode %{
14538 __ subv(as_FloatRegister($dst$$reg), __ T8B,
14539 as_FloatRegister($src1$$reg),
14540 as_FloatRegister($src2$$reg));
14541 %}
14542 ins_pipe(vdop64);
14543 %}
14544
14545 instruct vsub16B(vecX dst, vecX src1, vecX src2)
14546 %{
14547 predicate(n->as_Vector()->length() == 16);
14548 match(Set dst (SubVB src1 src2));
14549 ins_cost(INSN_COST);
14550 format %{ "subv $dst,$src1,$src2\t# vector (16B)" %}
14551 ins_encode %{
14552 __ subv(as_FloatRegister($dst$$reg), __ T16B,
14553 as_FloatRegister($src1$$reg),
14554 as_FloatRegister($src2$$reg));
14555 %}
14556 ins_pipe(vdop128);
14557 %}
14558
14559 instruct vsub4S(vecD dst, vecD src1, vecD src2)
14560 %{
14561 predicate(n->as_Vector()->length() == 2 ||
14562 n->as_Vector()->length() == 4);
14563 match(Set dst (SubVS src1 src2));
14564 ins_cost(INSN_COST);
14565 format %{ "subv $dst,$src1,$src2\t# vector (4H)" %}
14566 ins_encode %{
14567 __ subv(as_FloatRegister($dst$$reg), __ T4H,
14568 as_FloatRegister($src1$$reg),
14569 as_FloatRegister($src2$$reg));
14570 %}
14571 ins_pipe(vdop64);
14572 %}
14573
14574 instruct vsub8S(vecX dst, vecX src1, vecX src2)
14575 %{
14576 predicate(n->as_Vector()->length() == 8);
14577 match(Set dst (SubVS src1 src2));
14578 ins_cost(INSN_COST);
14579 format %{ "subv $dst,$src1,$src2\t# vector (8H)" %}
14580 ins_encode %{
14581 __ subv(as_FloatRegister($dst$$reg), __ T8H,
14582 as_FloatRegister($src1$$reg),
14583 as_FloatRegister($src2$$reg));
14584 %}
14585 ins_pipe(vdop128);
14586 %}
14587
14588 instruct vsub2I(vecD dst, vecD src1, vecD src2)
14589 %{
14590 predicate(n->as_Vector()->length() == 2);
14591 match(Set dst (SubVI src1 src2));
14592 ins_cost(INSN_COST);
14593 format %{ "subv $dst,$src1,$src2\t# vector (2S)" %}
14594 ins_encode %{
14595 __ subv(as_FloatRegister($dst$$reg), __ T2S,
14596 as_FloatRegister($src1$$reg),
14597 as_FloatRegister($src2$$reg));
14598 %}
14599 ins_pipe(vdop64);
14600 %}
14601
14602 instruct vsub4I(vecX dst, vecX src1, vecX src2)
14603 %{
14604 predicate(n->as_Vector()->length() == 4);
14605 match(Set dst (SubVI src1 src2));
14606 ins_cost(INSN_COST);
14607 format %{ "subv $dst,$src1,$src2\t# vector (4S)" %}
14608 ins_encode %{
14609 __ subv(as_FloatRegister($dst$$reg), __ T4S,
14610 as_FloatRegister($src1$$reg),
14611 as_FloatRegister($src2$$reg));
14612 %}
14613 ins_pipe(vdop128);
14614 %}
14615
14616 instruct vsub2L(vecX dst, vecX src1, vecX src2)
14617 %{
14618 predicate(n->as_Vector()->length() == 2);
14619 match(Set dst (SubVL src1 src2));
14620 ins_cost(INSN_COST);
14621 format %{ "subv $dst,$src1,$src2\t# vector (2L)" %}
14622 ins_encode %{
14623 __ subv(as_FloatRegister($dst$$reg), __ T2D,
14624 as_FloatRegister($src1$$reg),
14625 as_FloatRegister($src2$$reg));
14626 %}
14627 ins_pipe(vdop128);
14628 %}
14629
14630 instruct vsub2F(vecD dst, vecD src1, vecD src2)
14631 %{
14632 predicate(n->as_Vector()->length() == 2);
14633 match(Set dst (SubVF src1 src2));
14634 ins_cost(INSN_COST);
14635 format %{ "fsub $dst,$src1,$src2\t# vector (2S)" %}
14636 ins_encode %{
14637 __ fsub(as_FloatRegister($dst$$reg), __ T2S,
14638 as_FloatRegister($src1$$reg),
14639 as_FloatRegister($src2$$reg));
14640 %}
14641 ins_pipe(vdop_fp64);
14642 %}
14643
14644 instruct vsub4F(vecX dst, vecX src1, vecX src2)
14645 %{
14646 predicate(n->as_Vector()->length() == 4);
14647 match(Set dst (SubVF src1 src2));
14648 ins_cost(INSN_COST);
14649 format %{ "fsub $dst,$src1,$src2\t# vector (4S)" %}
14650 ins_encode %{
14651 __ fsub(as_FloatRegister($dst$$reg), __ T4S,
14652 as_FloatRegister($src1$$reg),
14653 as_FloatRegister($src2$$reg));
14654 %}
14655 ins_pipe(vdop_fp128);
14656 %}
14657
14658 instruct vsub2D(vecX dst, vecX src1, vecX src2)
14659 %{
14660 predicate(n->as_Vector()->length() == 2);
14661 match(Set dst (SubVD src1 src2));
14662 ins_cost(INSN_COST);
14663 format %{ "fsub $dst,$src1,$src2\t# vector (2D)" %}
14664 ins_encode %{
14665 __ fsub(as_FloatRegister($dst$$reg), __ T2D,
14666 as_FloatRegister($src1$$reg),
14667 as_FloatRegister($src2$$reg));
14668 %}
14669 ins_pipe(vdop_fp128);
14670 %}
14671
14672 // --------------------------------- MUL --------------------------------------
14673
14674 instruct vmul4S(vecD dst, vecD src1, vecD src2)
14675 %{
14676 predicate(n->as_Vector()->length() == 2 ||
14677 n->as_Vector()->length() == 4);
14678 match(Set dst (MulVS src1 src2));
14679 ins_cost(INSN_COST);
14680 format %{ "mulv $dst,$src1,$src2\t# vector (4H)" %}
14681 ins_encode %{
14682 __ mulv(as_FloatRegister($dst$$reg), __ T4H,
14683 as_FloatRegister($src1$$reg),
14684 as_FloatRegister($src2$$reg));
14685 %}
14686 ins_pipe(vmul64);
14687 %}
14688
14689 instruct vmul8S(vecX dst, vecX src1, vecX src2)
14690 %{
14691 predicate(n->as_Vector()->length() == 8);
14692 match(Set dst (MulVS src1 src2));
14693 ins_cost(INSN_COST);
14694 format %{ "mulv $dst,$src1,$src2\t# vector (8H)" %}
14695 ins_encode %{
14696 __ mulv(as_FloatRegister($dst$$reg), __ T8H,
14697 as_FloatRegister($src1$$reg),
14698 as_FloatRegister($src2$$reg));
14699 %}
14700 ins_pipe(vmul128);
14701 %}
14702
14703 instruct vmul2I(vecD dst, vecD src1, vecD src2)
14704 %{
14705 predicate(n->as_Vector()->length() == 2);
14706 match(Set dst (MulVI src1 src2));
14707 ins_cost(INSN_COST);
14708 format %{ "mulv $dst,$src1,$src2\t# vector (2S)" %}
14709 ins_encode %{
14710 __ mulv(as_FloatRegister($dst$$reg), __ T2S,
14711 as_FloatRegister($src1$$reg),
14712 as_FloatRegister($src2$$reg));
14713 %}
14714 ins_pipe(vmul64);
14715 %}
14716
14717 instruct vmul4I(vecX dst, vecX src1, vecX src2)
14718 %{
14719 predicate(n->as_Vector()->length() == 4);
14720 match(Set dst (MulVI src1 src2));
14721 ins_cost(INSN_COST);
14722 format %{ "mulv $dst,$src1,$src2\t# vector (4S)" %}
14723 ins_encode %{
14724 __ mulv(as_FloatRegister($dst$$reg), __ T4S,
14725 as_FloatRegister($src1$$reg),
14726 as_FloatRegister($src2$$reg));
14727 %}
14728 ins_pipe(vmul128);
14729 %}
14730
14731 instruct vmul2F(vecD dst, vecD src1, vecD src2)
14732 %{
14733 predicate(n->as_Vector()->length() == 2);
14734 match(Set dst (MulVF src1 src2));
14735 ins_cost(INSN_COST);
14736 format %{ "fmul $dst,$src1,$src2\t# vector (2S)" %}
14737 ins_encode %{
14738 __ fmul(as_FloatRegister($dst$$reg), __ T2S,
14739 as_FloatRegister($src1$$reg),
14740 as_FloatRegister($src2$$reg));
14741 %}
14742 ins_pipe(vmuldiv_fp64);
14743 %}
14744
14745 instruct vmul4F(vecX dst, vecX src1, vecX src2)
14746 %{
14747 predicate(n->as_Vector()->length() == 4);
14748 match(Set dst (MulVF src1 src2));
14749 ins_cost(INSN_COST);
14750 format %{ "fmul $dst,$src1,$src2\t# vector (4S)" %}
14751 ins_encode %{
14752 __ fmul(as_FloatRegister($dst$$reg), __ T4S,
14753 as_FloatRegister($src1$$reg),
14754 as_FloatRegister($src2$$reg));
14755 %}
14756 ins_pipe(vmuldiv_fp128);
14757 %}
14758
14759 instruct vmul2D(vecX dst, vecX src1, vecX src2)
14760 %{
14761 predicate(n->as_Vector()->length() == 2);
14762 match(Set dst (MulVD src1 src2));
14763 ins_cost(INSN_COST);
14764 format %{ "fmul $dst,$src1,$src2\t# vector (2D)" %}
14765 ins_encode %{
14766 __ fmul(as_FloatRegister($dst$$reg), __ T2D,
14767 as_FloatRegister($src1$$reg),
14768 as_FloatRegister($src2$$reg));
14769 %}
14770 ins_pipe(vmuldiv_fp128);
14771 %}
14772
14773 // --------------------------------- MLA --------------------------------------
14774
14775 instruct vmla4S(vecD dst, vecD src1, vecD src2)
14776 %{
14777 predicate(n->as_Vector()->length() == 2 ||
14778 n->as_Vector()->length() == 4);
14779 match(Set dst (AddVS dst (MulVS src1 src2)));
14780 ins_cost(INSN_COST);
14781 format %{ "mlav $dst,$src1,$src2\t# vector (4H)" %}
14782 ins_encode %{
14783 __ mlav(as_FloatRegister($dst$$reg), __ T4H,
14784 as_FloatRegister($src1$$reg),
14785 as_FloatRegister($src2$$reg));
14786 %}
14787 ins_pipe(vmla64);
14788 %}
14789
14790 instruct vmla8S(vecX dst, vecX src1, vecX src2)
14791 %{
14792 predicate(n->as_Vector()->length() == 8);
14793 match(Set dst (AddVS dst (MulVS src1 src2)));
14794 ins_cost(INSN_COST);
14795 format %{ "mlav $dst,$src1,$src2\t# vector (8H)" %}
14796 ins_encode %{
14797 __ mlav(as_FloatRegister($dst$$reg), __ T8H,
14798 as_FloatRegister($src1$$reg),
14799 as_FloatRegister($src2$$reg));
14800 %}
14801 ins_pipe(vmla128);
14802 %}
14803
14804 instruct vmla2I(vecD dst, vecD src1, vecD src2)
14805 %{
14806 predicate(n->as_Vector()->length() == 2);
14807 match(Set dst (AddVI dst (MulVI src1 src2)));
14808 ins_cost(INSN_COST);
14809 format %{ "mlav $dst,$src1,$src2\t# vector (2S)" %}
14810 ins_encode %{
14811 __ mlav(as_FloatRegister($dst$$reg), __ T2S,
14812 as_FloatRegister($src1$$reg),
14813 as_FloatRegister($src2$$reg));
14814 %}
14815 ins_pipe(vmla64);
14816 %}
14817
14818 instruct vmla4I(vecX dst, vecX src1, vecX src2)
14819 %{
14820 predicate(n->as_Vector()->length() == 4);
14821 match(Set dst (AddVI dst (MulVI src1 src2)));
14822 ins_cost(INSN_COST);
14823 format %{ "mlav $dst,$src1,$src2\t# vector (4S)" %}
14824 ins_encode %{
14825 __ mlav(as_FloatRegister($dst$$reg), __ T4S,
14826 as_FloatRegister($src1$$reg),
14827 as_FloatRegister($src2$$reg));
14828 %}
14829 ins_pipe(vmla128);
14830 %}
14831
14832 // --------------------------------- MLS --------------------------------------
14833
14834 instruct vmls4S(vecD dst, vecD src1, vecD src2)
14835 %{
14836 predicate(n->as_Vector()->length() == 2 ||
14837 n->as_Vector()->length() == 4);
14838 match(Set dst (SubVS dst (MulVS src1 src2)));
14839 ins_cost(INSN_COST);
14840 format %{ "mlsv $dst,$src1,$src2\t# vector (4H)" %}
14841 ins_encode %{
14842 __ mlsv(as_FloatRegister($dst$$reg), __ T4H,
14843 as_FloatRegister($src1$$reg),
14844 as_FloatRegister($src2$$reg));
14845 %}
14846 ins_pipe(vmla64);
14847 %}
14848
14849 instruct vmls8S(vecX dst, vecX src1, vecX src2)
14850 %{
14851 predicate(n->as_Vector()->length() == 8);
14852 match(Set dst (SubVS dst (MulVS src1 src2)));
14853 ins_cost(INSN_COST);
14854 format %{ "mlsv $dst,$src1,$src2\t# vector (8H)" %}
14855 ins_encode %{
14856 __ mlsv(as_FloatRegister($dst$$reg), __ T8H,
14857 as_FloatRegister($src1$$reg),
14858 as_FloatRegister($src2$$reg));
14859 %}
14860 ins_pipe(vmla128);
14861 %}
14862
14863 instruct vmls2I(vecD dst, vecD src1, vecD src2)
14864 %{
14865 predicate(n->as_Vector()->length() == 2);
14866 match(Set dst (SubVI dst (MulVI src1 src2)));
14867 ins_cost(INSN_COST);
14868 format %{ "mlsv $dst,$src1,$src2\t# vector (2S)" %}
14869 ins_encode %{
14870 __ mlsv(as_FloatRegister($dst$$reg), __ T2S,
14871 as_FloatRegister($src1$$reg),
14872 as_FloatRegister($src2$$reg));
14873 %}
14874 ins_pipe(vmla64);
14875 %}
14876
14877 instruct vmls4I(vecX dst, vecX src1, vecX src2)
14878 %{
14879 predicate(n->as_Vector()->length() == 4);
14880 match(Set dst (SubVI dst (MulVI src1 src2)));
14881 ins_cost(INSN_COST);
14882 format %{ "mlsv $dst,$src1,$src2\t# vector (4S)" %}
14883 ins_encode %{
14884 __ mlsv(as_FloatRegister($dst$$reg), __ T4S,
14885 as_FloatRegister($src1$$reg),
14886 as_FloatRegister($src2$$reg));
14887 %}
14888 ins_pipe(vmla128);
14889 %}
14890
14891 // --------------------------------- DIV --------------------------------------
14892
14893 instruct vdiv2F(vecD dst, vecD src1, vecD src2)
14894 %{
14895 predicate(n->as_Vector()->length() == 2);
14896 match(Set dst (DivVF src1 src2));
14897 ins_cost(INSN_COST);
14898 format %{ "fdiv $dst,$src1,$src2\t# vector (2S)" %}
14899 ins_encode %{
14900 __ fdiv(as_FloatRegister($dst$$reg), __ T2S,
14901 as_FloatRegister($src1$$reg),
14902 as_FloatRegister($src2$$reg));
14903 %}
14904 ins_pipe(vmuldiv_fp64);
14905 %}
14906
14907 instruct vdiv4F(vecX dst, vecX src1, vecX src2)
14908 %{
14909 predicate(n->as_Vector()->length() == 4);
14910 match(Set dst (DivVF src1 src2));
14911 ins_cost(INSN_COST);
14912 format %{ "fdiv $dst,$src1,$src2\t# vector (4S)" %}
14913 ins_encode %{
14914 __ fdiv(as_FloatRegister($dst$$reg), __ T4S,
14915 as_FloatRegister($src1$$reg),
14916 as_FloatRegister($src2$$reg));
14917 %}
14918 ins_pipe(vmuldiv_fp128);
14919 %}
14920
14921 instruct vdiv2D(vecX dst, vecX src1, vecX src2)
14922 %{
14923 predicate(n->as_Vector()->length() == 2);
14924 match(Set dst (DivVD src1 src2));
14925 ins_cost(INSN_COST);
14926 format %{ "fdiv $dst,$src1,$src2\t# vector (2D)" %}
14927 ins_encode %{
14928 __ fdiv(as_FloatRegister($dst$$reg), __ T2D,
14929 as_FloatRegister($src1$$reg),
14930 as_FloatRegister($src2$$reg));
14931 %}
14932 ins_pipe(vmuldiv_fp128);
14933 %}
14934
14935 // --------------------------------- AND --------------------------------------
14936
14937 instruct vand8B(vecD dst, vecD src1, vecD src2)
14938 %{
14939 predicate(n->as_Vector()->length_in_bytes() == 4 ||
14940 n->as_Vector()->length_in_bytes() == 8);
14941 match(Set dst (AndV src1 src2));
14942 ins_cost(INSN_COST);
14943 format %{ "and $dst,$src1,$src2\t# vector (8B)" %}
14944 ins_encode %{
14945 __ andr(as_FloatRegister($dst$$reg), __ T8B,
14946 as_FloatRegister($src1$$reg),
14947 as_FloatRegister($src2$$reg));
14948 %}
14949 ins_pipe(vlogical64);
14950 %}
14951
14952 instruct vand16B(vecX dst, vecX src1, vecX src2)
14953 %{
14954 predicate(n->as_Vector()->length_in_bytes() == 16);
14955 match(Set dst (AndV src1 src2));
14956 ins_cost(INSN_COST);
14957 format %{ "and $dst,$src1,$src2\t# vector (16B)" %}
14958 ins_encode %{
14959 __ andr(as_FloatRegister($dst$$reg), __ T16B,
14960 as_FloatRegister($src1$$reg),
14961 as_FloatRegister($src2$$reg));
14962 %}
14963 ins_pipe(vlogical128);
14964 %}
14965
14966 // --------------------------------- OR ---------------------------------------
14967
14968 instruct vor8B(vecD dst, vecD src1, vecD src2)
14969 %{
14970 predicate(n->as_Vector()->length_in_bytes() == 4 ||
14971 n->as_Vector()->length_in_bytes() == 8);
14972 match(Set dst (OrV src1 src2));
14973 ins_cost(INSN_COST);
14974 format %{ "and $dst,$src1,$src2\t# vector (8B)" %}
14975 ins_encode %{
14976 __ orr(as_FloatRegister($dst$$reg), __ T8B,
14977 as_FloatRegister($src1$$reg),
14978 as_FloatRegister($src2$$reg));
14979 %}
14980 ins_pipe(vlogical64);
14981 %}
14982
14983 instruct vor16B(vecX dst, vecX src1, vecX src2)
14984 %{
14985 predicate(n->as_Vector()->length_in_bytes() == 16);
14986 match(Set dst (OrV src1 src2));
14987 ins_cost(INSN_COST);
14988 format %{ "orr $dst,$src1,$src2\t# vector (16B)" %}
14989 ins_encode %{
14990 __ orr(as_FloatRegister($dst$$reg), __ T16B,
14991 as_FloatRegister($src1$$reg),
14992 as_FloatRegister($src2$$reg));
14993 %}
14994 ins_pipe(vlogical128);
14995 %}
14996
14997 // --------------------------------- XOR --------------------------------------
14998
14999 instruct vxor8B(vecD dst, vecD src1, vecD src2)
15000 %{
15001 predicate(n->as_Vector()->length_in_bytes() == 4 ||
15002 n->as_Vector()->length_in_bytes() == 8);
15003 match(Set dst (XorV src1 src2));
15004 ins_cost(INSN_COST);
15005 format %{ "xor $dst,$src1,$src2\t# vector (8B)" %}
15006 ins_encode %{
15007 __ eor(as_FloatRegister($dst$$reg), __ T8B,
15008 as_FloatRegister($src1$$reg),
15009 as_FloatRegister($src2$$reg));
15010 %}
15011 ins_pipe(vlogical64);
15012 %}
15013
15014 instruct vxor16B(vecX dst, vecX src1, vecX src2)
15015 %{
15016 predicate(n->as_Vector()->length_in_bytes() == 16);
15017 match(Set dst (XorV src1 src2));
15018 ins_cost(INSN_COST);
15019 format %{ "xor $dst,$src1,$src2\t# vector (16B)" %}
15020 ins_encode %{
15021 __ eor(as_FloatRegister($dst$$reg), __ T16B,
15022 as_FloatRegister($src1$$reg),
15023 as_FloatRegister($src2$$reg));
15024 %}
15025 ins_pipe(vlogical128);
15026 %}
15027
15028 // ------------------------------ Shift ---------------------------------------
15029 instruct vshiftcnt8B(vecD dst, iRegIorL2I cnt) %{
15030 predicate(n->as_Vector()->length_in_bytes() == 8);
15031 match(Set dst (LShiftCntV cnt));
15032 match(Set dst (RShiftCntV cnt));
15033 format %{ "dup $dst, $cnt\t# shift count vector (8B)" %}
15034 ins_encode %{
15035 __ dup(as_FloatRegister($dst$$reg), __ T8B, as_Register($cnt$$reg));
15036 %}
15037 ins_pipe(vdup_reg_reg64);
15038 %}
15039
15040 instruct vshiftcnt16B(vecX dst, iRegIorL2I cnt) %{
15041 predicate(n->as_Vector()->length_in_bytes() == 16);
15042 match(Set dst (LShiftCntV cnt));
15043 match(Set dst (RShiftCntV cnt));
15044 format %{ "dup $dst, $cnt\t# shift count vector (16B)" %}
15045 ins_encode %{
15046 __ dup(as_FloatRegister($dst$$reg), __ T16B, as_Register($cnt$$reg));
15047 %}
15048 ins_pipe(vdup_reg_reg128);
15049 %}
15050
15051 instruct vsll8B(vecD dst, vecD src, vecD shift) %{
15052 predicate(n->as_Vector()->length() == 4 ||
15053 n->as_Vector()->length() == 8);
15054 match(Set dst (LShiftVB src shift));
15055 ins_cost(INSN_COST);
15056 format %{ "sshl $dst,$src,$shift\t# vector (8B)" %}
15057 ins_encode %{
15058 __ sshl(as_FloatRegister($dst$$reg), __ T8B,
15059 as_FloatRegister($src$$reg),
15060 as_FloatRegister($shift$$reg));
15061 %}
15062 ins_pipe(vshift64);
15063 %}
15064
15065 instruct vsll16B(vecX dst, vecX src, vecX shift) %{
15066 predicate(n->as_Vector()->length() == 16);
15067 match(Set dst (LShiftVB src shift));
15068 ins_cost(INSN_COST);
15069 format %{ "sshl $dst,$src,$shift\t# vector (16B)" %}
15070 ins_encode %{
15071 __ sshl(as_FloatRegister($dst$$reg), __ T16B,
15072 as_FloatRegister($src$$reg),
15073 as_FloatRegister($shift$$reg));
15074 %}
15075 ins_pipe(vshift128);
15076 %}
15077
15078 // Right shifts with vector shift count on aarch64 SIMD are implemented
15079 // as left shift by negative shift count.
15080 // There are two cases for vector shift count.
15081 //
15082 // Case 1: The vector shift count is from replication.
15083 // | |
15084 // LoadVector RShiftCntV
15085 // | /
15086 // RShiftVI
15087 // Note: In inner loop, multiple neg instructions are used, which can be
15088 // moved to outer loop and merge into one neg instruction.
15089 //
15090 // Case 2: The vector shift count is from loading.
15091 // This case isn't supported by middle-end now. But it's supported by
15092 // panama/vectorIntrinsics(JEP 338: Vector API).
15093 // | |
15094 // LoadVector LoadVector
15095 // | /
15096 // RShiftVI
15097 //
15098
15099 instruct vsra8B(vecD dst, vecD src, vecD shift, vecD tmp) %{
15100 predicate(n->as_Vector()->length() == 4 ||
15101 n->as_Vector()->length() == 8);
15102 match(Set dst (RShiftVB src shift));
15103 ins_cost(INSN_COST);
15104 effect(TEMP tmp);
15105 format %{ "negr $tmp,$shift\t"
15106 "sshl $dst,$src,$tmp\t# vector (8B)" %}
15107 ins_encode %{
15108 __ negr(as_FloatRegister($tmp$$reg), __ T8B,
15109 as_FloatRegister($shift$$reg));
15110 __ sshl(as_FloatRegister($dst$$reg), __ T8B,
15111 as_FloatRegister($src$$reg),
15112 as_FloatRegister($tmp$$reg));
15113 %}
15114 ins_pipe(vshift64);
15115 %}
15116
15117 instruct vsra16B(vecX dst, vecX src, vecX shift, vecX tmp) %{
15118 predicate(n->as_Vector()->length() == 16);
15119 match(Set dst (RShiftVB src shift));
15120 ins_cost(INSN_COST);
15121 effect(TEMP tmp);
15122 format %{ "negr $tmp,$shift\t"
15123 "sshl $dst,$src,$tmp\t# vector (16B)" %}
15124 ins_encode %{
15125 __ negr(as_FloatRegister($tmp$$reg), __ T16B,
15126 as_FloatRegister($shift$$reg));
15127 __ sshl(as_FloatRegister($dst$$reg), __ T16B,
15128 as_FloatRegister($src$$reg),
15129 as_FloatRegister($tmp$$reg));
15130 %}
15131 ins_pipe(vshift128);
15132 %}
15133
15134 instruct vsrl8B(vecD dst, vecD src, vecD shift, vecD tmp) %{
15135 predicate(n->as_Vector()->length() == 4 ||
15136 n->as_Vector()->length() == 8);
15137 match(Set dst (URShiftVB src shift));
15138 ins_cost(INSN_COST);
15139 effect(TEMP tmp);
15140 format %{ "negr $tmp,$shift\t"
15141 "ushl $dst,$src,$tmp\t# vector (8B)" %}
15142 ins_encode %{
15143 __ negr(as_FloatRegister($tmp$$reg), __ T8B,
15144 as_FloatRegister($shift$$reg));
15145 __ ushl(as_FloatRegister($dst$$reg), __ T8B,
15146 as_FloatRegister($src$$reg),
15147 as_FloatRegister($tmp$$reg));
15148 %}
15149 ins_pipe(vshift64);
15150 %}
15151
15152 instruct vsrl16B(vecX dst, vecX src, vecX shift, vecX tmp) %{
15153 predicate(n->as_Vector()->length() == 16);
15154 match(Set dst (URShiftVB src shift));
15155 ins_cost(INSN_COST);
15156 effect(TEMP tmp);
15157 format %{ "negr $tmp,$shift\t"
15158 "ushl $dst,$src,$tmp\t# vector (16B)" %}
15159 ins_encode %{
15160 __ negr(as_FloatRegister($tmp$$reg), __ T16B,
15161 as_FloatRegister($shift$$reg));
15162 __ ushl(as_FloatRegister($dst$$reg), __ T16B,
15163 as_FloatRegister($src$$reg),
15164 as_FloatRegister($tmp$$reg));
15165 %}
15166 ins_pipe(vshift128);
15167 %}
15168
15169 instruct vsll8B_imm(vecD dst, vecD src, immI shift) %{
15170 predicate(n->as_Vector()->length() == 4 ||
15171 n->as_Vector()->length() == 8);
15172 match(Set dst (LShiftVB src shift));
15173 ins_cost(INSN_COST);
15174 format %{ "shl $dst, $src, $shift\t# vector (8B)" %}
15175 ins_encode %{
15176 int sh = (int)$shift$$constant & 31;
15177 if (sh >= 8) {
15178 __ eor(as_FloatRegister($dst$$reg), __ T8B,
15179 as_FloatRegister($src$$reg),
15180 as_FloatRegister($src$$reg));
15181 } else {
15182 __ shl(as_FloatRegister($dst$$reg), __ T8B,
15183 as_FloatRegister($src$$reg), sh);
15184 }
15185 %}
15186 ins_pipe(vshift64_imm);
15187 %}
15188
15189 instruct vsll16B_imm(vecX dst, vecX src, immI shift) %{
15190 predicate(n->as_Vector()->length() == 16);
15191 match(Set dst (LShiftVB src shift));
15192 ins_cost(INSN_COST);
15193 format %{ "shl $dst, $src, $shift\t# vector (16B)" %}
15194 ins_encode %{
15195 int sh = (int)$shift$$constant & 31;
15196 if (sh >= 8) {
15197 __ eor(as_FloatRegister($dst$$reg), __ T16B,
15198 as_FloatRegister($src$$reg),
15199 as_FloatRegister($src$$reg));
15200 } else {
15201 __ shl(as_FloatRegister($dst$$reg), __ T16B,
15202 as_FloatRegister($src$$reg), sh);
15203 }
15204 %}
15205 ins_pipe(vshift128_imm);
15206 %}
15207
15208 instruct vsra8B_imm(vecD dst, vecD src, immI shift) %{
15209 predicate(n->as_Vector()->length() == 4 ||
15210 n->as_Vector()->length() == 8);
15211 match(Set dst (RShiftVB src shift));
15212 ins_cost(INSN_COST);
15213 format %{ "sshr $dst, $src, $shift\t# vector (8B)" %}
15214 ins_encode %{
15215 int sh = (int)$shift$$constant & 31;
15216 if (sh >= 8) sh = 7;
15217 sh = -sh & 7;
15218 __ sshr(as_FloatRegister($dst$$reg), __ T8B,
15219 as_FloatRegister($src$$reg), sh);
15220 %}
15221 ins_pipe(vshift64_imm);
15222 %}
15223
15224 instruct vsra16B_imm(vecX dst, vecX src, immI shift) %{
15225 predicate(n->as_Vector()->length() == 16);
15226 match(Set dst (RShiftVB src shift));
15227 ins_cost(INSN_COST);
15228 format %{ "sshr $dst, $src, $shift\t# vector (16B)" %}
15229 ins_encode %{
15230 int sh = (int)$shift$$constant & 31;
15231 if (sh >= 8) sh = 7;
15232 sh = -sh & 7;
15233 __ sshr(as_FloatRegister($dst$$reg), __ T16B,
15234 as_FloatRegister($src$$reg), sh);
15235 %}
15236 ins_pipe(vshift128_imm);
15237 %}
15238
15239 instruct vsrl8B_imm(vecD dst, vecD src, immI shift) %{
15240 predicate(n->as_Vector()->length() == 4 ||
15241 n->as_Vector()->length() == 8);
15242 match(Set dst (URShiftVB src shift));
15243 ins_cost(INSN_COST);
15244 format %{ "ushr $dst, $src, $shift\t# vector (8B)" %}
15245 ins_encode %{
15246 int sh = (int)$shift$$constant & 31;
15247 if (sh >= 8) {
15248 __ eor(as_FloatRegister($dst$$reg), __ T8B,
15249 as_FloatRegister($src$$reg),
15250 as_FloatRegister($src$$reg));
15251 } else {
15252 __ ushr(as_FloatRegister($dst$$reg), __ T8B,
15253 as_FloatRegister($src$$reg), -sh & 7);
15254 }
15255 %}
15256 ins_pipe(vshift64_imm);
15257 %}
15258
15259 instruct vsrl16B_imm(vecX dst, vecX src, immI shift) %{
15260 predicate(n->as_Vector()->length() == 16);
15261 match(Set dst (URShiftVB src shift));
15262 ins_cost(INSN_COST);
15263 format %{ "ushr $dst, $src, $shift\t# vector (16B)" %}
15264 ins_encode %{
15265 int sh = (int)$shift$$constant & 31;
15266 if (sh >= 8) {
15267 __ eor(as_FloatRegister($dst$$reg), __ T16B,
15268 as_FloatRegister($src$$reg),
15269 as_FloatRegister($src$$reg));
15270 } else {
15271 __ ushr(as_FloatRegister($dst$$reg), __ T16B,
15272 as_FloatRegister($src$$reg), -sh & 7);
15273 }
15274 %}
15275 ins_pipe(vshift128_imm);
15276 %}
15277
15278 instruct vsll4S(vecD dst, vecD src, vecD shift) %{
15279 predicate(n->as_Vector()->length() == 2 ||
15280 n->as_Vector()->length() == 4);
15281 match(Set dst (LShiftVS src shift));
15282 ins_cost(INSN_COST);
15283 format %{ "sshl $dst,$src,$shift\t# vector (4H)" %}
15284 ins_encode %{
15285 __ sshl(as_FloatRegister($dst$$reg), __ T4H,
15286 as_FloatRegister($src$$reg),
15287 as_FloatRegister($shift$$reg));
15288 %}
15289 ins_pipe(vshift64);
15290 %}
15291
15292 instruct vsll8S(vecX dst, vecX src, vecX shift) %{
15293 predicate(n->as_Vector()->length() == 8);
15294 match(Set dst (LShiftVS src shift));
15295 ins_cost(INSN_COST);
15296 format %{ "sshl $dst,$src,$shift\t# vector (8H)" %}
15297 ins_encode %{
15298 __ sshl(as_FloatRegister($dst$$reg), __ T8H,
15299 as_FloatRegister($src$$reg),
15300 as_FloatRegister($shift$$reg));
15301 %}
15302 ins_pipe(vshift128);
15303 %}
15304
15305 instruct vsra4S(vecD dst, vecD src, vecD shift, vecD tmp) %{
15306 predicate(n->as_Vector()->length() == 2 ||
15307 n->as_Vector()->length() == 4);
15308 match(Set dst (RShiftVS src shift));
15309 ins_cost(INSN_COST);
15310 effect(TEMP tmp);
15311 format %{ "negr $tmp,$shift\t"
15312 "sshl $dst,$src,$tmp\t# vector (4H)" %}
15313 ins_encode %{
15314 __ negr(as_FloatRegister($tmp$$reg), __ T8B,
15315 as_FloatRegister($shift$$reg));
15316 __ sshl(as_FloatRegister($dst$$reg), __ T4H,
15317 as_FloatRegister($src$$reg),
15318 as_FloatRegister($tmp$$reg));
15319 %}
15320 ins_pipe(vshift64);
15321 %}
15322
15323 instruct vsra8S(vecX dst, vecX src, vecX shift, vecX tmp) %{
15324 predicate(n->as_Vector()->length() == 8);
15325 match(Set dst (RShiftVS src shift));
15326 ins_cost(INSN_COST);
15327 effect(TEMP tmp);
15328 format %{ "negr $tmp,$shift\t"
15329 "sshl $dst,$src,$tmp\t# vector (8H)" %}
15330 ins_encode %{
15331 __ negr(as_FloatRegister($tmp$$reg), __ T16B,
15332 as_FloatRegister($shift$$reg));
15333 __ sshl(as_FloatRegister($dst$$reg), __ T8H,
15334 as_FloatRegister($src$$reg),
15335 as_FloatRegister($tmp$$reg));
15336 %}
15337 ins_pipe(vshift128);
15338 %}
15339
15340 instruct vsrl4S(vecD dst, vecD src, vecD shift, vecD tmp) %{
15341 predicate(n->as_Vector()->length() == 2 ||
15342 n->as_Vector()->length() == 4);
15343 match(Set dst (URShiftVS src shift));
15344 ins_cost(INSN_COST);
15345 effect(TEMP tmp);
15346 format %{ "negr $tmp,$shift\t"
15347 "ushl $dst,$src,$tmp\t# vector (4H)" %}
15348 ins_encode %{
15349 __ negr(as_FloatRegister($tmp$$reg), __ T8B,
15350 as_FloatRegister($shift$$reg));
15351 __ ushl(as_FloatRegister($dst$$reg), __ T4H,
15352 as_FloatRegister($src$$reg),
15353 as_FloatRegister($tmp$$reg));
15354 %}
15355 ins_pipe(vshift64);
15356 %}
15357
15358 instruct vsrl8S(vecX dst, vecX src, vecX shift, vecX tmp) %{
15359 predicate(n->as_Vector()->length() == 8);
15360 match(Set dst (URShiftVS src shift));
15361 ins_cost(INSN_COST);
15362 effect(TEMP tmp);
15363 format %{ "negr $tmp,$shift\t"
15364 "ushl $dst,$src,$tmp\t# vector (8H)" %}
15365 ins_encode %{
15366 __ negr(as_FloatRegister($tmp$$reg), __ T16B,
15367 as_FloatRegister($shift$$reg));
15368 __ ushl(as_FloatRegister($dst$$reg), __ T8H,
15369 as_FloatRegister($src$$reg),
15370 as_FloatRegister($tmp$$reg));
15371 %}
15372 ins_pipe(vshift128);
15373 %}
15374
15375 instruct vsll4S_imm(vecD dst, vecD src, immI shift) %{
15376 predicate(n->as_Vector()->length() == 2 ||
15377 n->as_Vector()->length() == 4);
15378 match(Set dst (LShiftVS src shift));
15379 ins_cost(INSN_COST);
15380 format %{ "shl $dst, $src, $shift\t# vector (4H)" %}
15381 ins_encode %{
15382 int sh = (int)$shift$$constant & 31;
15383 if (sh >= 16) {
15384 __ eor(as_FloatRegister($dst$$reg), __ T8B,
15385 as_FloatRegister($src$$reg),
15386 as_FloatRegister($src$$reg));
15387 } else {
15388 __ shl(as_FloatRegister($dst$$reg), __ T4H,
15389 as_FloatRegister($src$$reg), sh);
15390 }
15391 %}
15392 ins_pipe(vshift64_imm);
15393 %}
15394
15395 instruct vsll8S_imm(vecX dst, vecX src, immI shift) %{
15396 predicate(n->as_Vector()->length() == 8);
15397 match(Set dst (LShiftVS src shift));
15398 ins_cost(INSN_COST);
15399 format %{ "shl $dst, $src, $shift\t# vector (8H)" %}
15400 ins_encode %{
15401 int sh = (int)$shift$$constant & 31;
15402 if (sh >= 16) {
15403 __ eor(as_FloatRegister($dst$$reg), __ T16B,
15404 as_FloatRegister($src$$reg),
15405 as_FloatRegister($src$$reg));
15406 } else {
15407 __ shl(as_FloatRegister($dst$$reg), __ T8H,
15408 as_FloatRegister($src$$reg), sh);
15409 }
15410 %}
15411 ins_pipe(vshift128_imm);
15412 %}
15413
15414 instruct vsra4S_imm(vecD dst, vecD src, immI shift) %{
15415 predicate(n->as_Vector()->length() == 2 ||
15416 n->as_Vector()->length() == 4);
15417 match(Set dst (RShiftVS src shift));
15418 ins_cost(INSN_COST);
15419 format %{ "sshr $dst, $src, $shift\t# vector (4H)" %}
15420 ins_encode %{
15421 int sh = (int)$shift$$constant & 31;
15422 if (sh >= 16) sh = 15;
15423 sh = -sh & 15;
15424 __ sshr(as_FloatRegister($dst$$reg), __ T4H,
15425 as_FloatRegister($src$$reg), sh);
15426 %}
15427 ins_pipe(vshift64_imm);
15428 %}
15429
15430 instruct vsra8S_imm(vecX dst, vecX src, immI shift) %{
15431 predicate(n->as_Vector()->length() == 8);
15432 match(Set dst (RShiftVS src shift));
15433 ins_cost(INSN_COST);
15434 format %{ "sshr $dst, $src, $shift\t# vector (8H)" %}
15435 ins_encode %{
15436 int sh = (int)$shift$$constant & 31;
15437 if (sh >= 16) sh = 15;
15438 sh = -sh & 15;
15439 __ sshr(as_FloatRegister($dst$$reg), __ T8H,
15440 as_FloatRegister($src$$reg), sh);
15441 %}
15442 ins_pipe(vshift128_imm);
15443 %}
15444
15445 instruct vsrl4S_imm(vecD dst, vecD src, immI shift) %{
15446 predicate(n->as_Vector()->length() == 2 ||
15447 n->as_Vector()->length() == 4);
15448 match(Set dst (URShiftVS src shift));
15449 ins_cost(INSN_COST);
15450 format %{ "ushr $dst, $src, $shift\t# vector (4H)" %}
15451 ins_encode %{
15452 int sh = (int)$shift$$constant & 31;
15453 if (sh >= 16) {
15454 __ eor(as_FloatRegister($dst$$reg), __ T8B,
15455 as_FloatRegister($src$$reg),
15456 as_FloatRegister($src$$reg));
15457 } else {
15458 __ ushr(as_FloatRegister($dst$$reg), __ T4H,
15459 as_FloatRegister($src$$reg), -sh & 15);
15460 }
15461 %}
15462 ins_pipe(vshift64_imm);
15463 %}
15464
15465 instruct vsrl8S_imm(vecX dst, vecX src, immI shift) %{
15466 predicate(n->as_Vector()->length() == 8);
15467 match(Set dst (URShiftVS src shift));
15468 ins_cost(INSN_COST);
15469 format %{ "ushr $dst, $src, $shift\t# vector (8H)" %}
15470 ins_encode %{
15471 int sh = (int)$shift$$constant & 31;
15472 if (sh >= 16) {
15473 __ eor(as_FloatRegister($dst$$reg), __ T16B,
15474 as_FloatRegister($src$$reg),
15475 as_FloatRegister($src$$reg));
15476 } else {
15477 __ ushr(as_FloatRegister($dst$$reg), __ T8H,
15478 as_FloatRegister($src$$reg), -sh & 15);
15479 }
15480 %}
15481 ins_pipe(vshift128_imm);
15482 %}
15483
15484 instruct vsll2I(vecD dst, vecD src, vecD shift) %{
15485 predicate(n->as_Vector()->length() == 2);
15486 match(Set dst (LShiftVI src shift));
15487 ins_cost(INSN_COST);
15488 format %{ "sshl $dst,$src,$shift\t# vector (2S)" %}
15489 ins_encode %{
15490 __ sshl(as_FloatRegister($dst$$reg), __ T2S,
15491 as_FloatRegister($src$$reg),
15492 as_FloatRegister($shift$$reg));
15493 %}
15494 ins_pipe(vshift64);
15495 %}
15496
15497 instruct vsll4I(vecX dst, vecX src, vecX shift) %{
15498 predicate(n->as_Vector()->length() == 4);
15499 match(Set dst (LShiftVI src shift));
15500 ins_cost(INSN_COST);
15501 format %{ "sshl $dst,$src,$shift\t# vector (4S)" %}
15502 ins_encode %{
15503 __ sshl(as_FloatRegister($dst$$reg), __ T4S,
15504 as_FloatRegister($src$$reg),
15505 as_FloatRegister($shift$$reg));
15506 %}
15507 ins_pipe(vshift128);
15508 %}
15509
15510 instruct vsra2I(vecD dst, vecD src, vecD shift, vecD tmp) %{
15511 predicate(n->as_Vector()->length() == 2);
15512 match(Set dst (RShiftVI src shift));
15513 ins_cost(INSN_COST);
15514 effect(TEMP tmp);
15515 format %{ "negr $tmp,$shift\t"
15516 "sshl $dst,$src,$tmp\t# vector (2S)" %}
15517 ins_encode %{
15518 __ negr(as_FloatRegister($tmp$$reg), __ T8B,
15519 as_FloatRegister($shift$$reg));
15520 __ sshl(as_FloatRegister($dst$$reg), __ T2S,
15521 as_FloatRegister($src$$reg),
15522 as_FloatRegister($tmp$$reg));
15523 %}
15524 ins_pipe(vshift64);
15525 %}
15526
15527 instruct vsra4I(vecX dst, vecX src, vecX shift, vecX tmp) %{
15528 predicate(n->as_Vector()->length() == 4);
15529 match(Set dst (RShiftVI src shift));
15530 ins_cost(INSN_COST);
15531 effect(TEMP tmp);
15532 format %{ "negr $tmp,$shift\t"
15533 "sshl $dst,$src,$tmp\t# vector (4S)" %}
15534 ins_encode %{
15535 __ negr(as_FloatRegister($tmp$$reg), __ T16B,
15536 as_FloatRegister($shift$$reg));
15537 __ sshl(as_FloatRegister($dst$$reg), __ T4S,
15538 as_FloatRegister($src$$reg),
15539 as_FloatRegister($tmp$$reg));
15540 %}
15541 ins_pipe(vshift128);
15542 %}
15543
15544 instruct vsrl2I(vecD dst, vecD src, vecD shift, vecD tmp) %{
15545 predicate(n->as_Vector()->length() == 2);
15546 match(Set dst (URShiftVI src shift));
15547 ins_cost(INSN_COST);
15548 effect(TEMP tmp);
15549 format %{ "negr $tmp,$shift\t"
15550 "ushl $dst,$src,$tmp\t# vector (2S)" %}
15551 ins_encode %{
15552 __ negr(as_FloatRegister($tmp$$reg), __ T8B,
15553 as_FloatRegister($shift$$reg));
15554 __ ushl(as_FloatRegister($dst$$reg), __ T2S,
15555 as_FloatRegister($src$$reg),
15556 as_FloatRegister($tmp$$reg));
15557 %}
15558 ins_pipe(vshift64);
15559 %}
15560
15561 instruct vsrl4I(vecX dst, vecX src, vecX shift, vecX tmp) %{
15562 predicate(n->as_Vector()->length() == 4);
15563 match(Set dst (URShiftVI src shift));
15564 ins_cost(INSN_COST);
15565 effect(TEMP tmp);
15566 format %{ "negr $tmp,$shift\t"
15567 "ushl $dst,$src,$tmp\t# vector (4S)" %}
15568 ins_encode %{
15569 __ negr(as_FloatRegister($tmp$$reg), __ T16B,
15570 as_FloatRegister($shift$$reg));
15571 __ ushl(as_FloatRegister($dst$$reg), __ T4S,
15572 as_FloatRegister($src$$reg),
15573 as_FloatRegister($tmp$$reg));
15574 %}
15575 ins_pipe(vshift128);
15576 %}
15577
15578 instruct vsll2I_imm(vecD dst, vecD src, immI shift) %{
15579 predicate(n->as_Vector()->length() == 2);
15580 match(Set dst (LShiftVI src shift));
15581 ins_cost(INSN_COST);
15582 format %{ "shl $dst, $src, $shift\t# vector (2S)" %}
15583 ins_encode %{
15584 __ shl(as_FloatRegister($dst$$reg), __ T2S,
15585 as_FloatRegister($src$$reg),
15586 (int)$shift$$constant & 31);
15587 %}
15588 ins_pipe(vshift64_imm);
15589 %}
15590
15591 instruct vsll4I_imm(vecX dst, vecX src, immI shift) %{
15592 predicate(n->as_Vector()->length() == 4);
15593 match(Set dst (LShiftVI src shift));
15594 ins_cost(INSN_COST);
15595 format %{ "shl $dst, $src, $shift\t# vector (4S)" %}
15596 ins_encode %{
15597 __ shl(as_FloatRegister($dst$$reg), __ T4S,
15598 as_FloatRegister($src$$reg),
15599 (int)$shift$$constant & 31);
15600 %}
15601 ins_pipe(vshift128_imm);
15602 %}
15603
15604 instruct vsra2I_imm(vecD dst, vecD src, immI shift) %{
15605 predicate(n->as_Vector()->length() == 2);
15606 match(Set dst (RShiftVI src shift));
15607 ins_cost(INSN_COST);
15608 format %{ "sshr $dst, $src, $shift\t# vector (2S)" %}
15609 ins_encode %{
15610 __ sshr(as_FloatRegister($dst$$reg), __ T2S,
15611 as_FloatRegister($src$$reg),
15612 -(int)$shift$$constant & 31);
15613 %}
15614 ins_pipe(vshift64_imm);
15615 %}
15616
15617 instruct vsra4I_imm(vecX dst, vecX src, immI shift) %{
15618 predicate(n->as_Vector()->length() == 4);
15619 match(Set dst (RShiftVI src shift));
15620 ins_cost(INSN_COST);
15621 format %{ "sshr $dst, $src, $shift\t# vector (4S)" %}
15622 ins_encode %{
15623 __ sshr(as_FloatRegister($dst$$reg), __ T4S,
15624 as_FloatRegister($src$$reg),
15625 -(int)$shift$$constant & 31);
15626 %}
15627 ins_pipe(vshift128_imm);
15628 %}
15629
15630 instruct vsrl2I_imm(vecD dst, vecD src, immI shift) %{
15631 predicate(n->as_Vector()->length() == 2);
15632 match(Set dst (URShiftVI src shift));
15633 ins_cost(INSN_COST);
15634 format %{ "ushr $dst, $src, $shift\t# vector (2S)" %}
15635 ins_encode %{
15636 __ ushr(as_FloatRegister($dst$$reg), __ T2S,
15637 as_FloatRegister($src$$reg),
15638 -(int)$shift$$constant & 31);
15639 %}
15640 ins_pipe(vshift64_imm);
15641 %}
15642
15643 instruct vsrl4I_imm(vecX dst, vecX src, immI shift) %{
15644 predicate(n->as_Vector()->length() == 4);
15645 match(Set dst (URShiftVI src shift));
15646 ins_cost(INSN_COST);
15647 format %{ "ushr $dst, $src, $shift\t# vector (4S)" %}
15648 ins_encode %{
15649 __ ushr(as_FloatRegister($dst$$reg), __ T4S,
15650 as_FloatRegister($src$$reg),
15651 -(int)$shift$$constant & 31);
15652 %}
15653 ins_pipe(vshift128_imm);
15654 %}
15655
15656 instruct vsll2L(vecX dst, vecX src, vecX shift) %{
15657 predicate(n->as_Vector()->length() == 2);
15658 match(Set dst (LShiftVL src shift));
15659 ins_cost(INSN_COST);
15660 format %{ "sshl $dst,$src,$shift\t# vector (2D)" %}
15661 ins_encode %{
15662 __ sshl(as_FloatRegister($dst$$reg), __ T2D,
15663 as_FloatRegister($src$$reg),
15664 as_FloatRegister($shift$$reg));
15665 %}
15666 ins_pipe(vshift128);
15667 %}
15668
15669 instruct vsra2L(vecX dst, vecX src, vecX shift, vecX tmp) %{
15670 predicate(n->as_Vector()->length() == 2);
15671 match(Set dst (RShiftVL src shift));
15672 ins_cost(INSN_COST);
15673 effect(TEMP tmp);
15674 format %{ "negr $tmp,$shift\t"
15675 "sshl $dst,$src,$tmp\t# vector (2D)" %}
15676 ins_encode %{
15677 __ negr(as_FloatRegister($tmp$$reg), __ T16B,
15678 as_FloatRegister($shift$$reg));
15679 __ sshl(as_FloatRegister($dst$$reg), __ T2D,
15680 as_FloatRegister($src$$reg),
15681 as_FloatRegister($tmp$$reg));
15682 %}
15683 ins_pipe(vshift128);
15684 %}
15685
15686 instruct vsrl2L(vecX dst, vecX src, vecX shift, vecX tmp) %{
15687 predicate(n->as_Vector()->length() == 2);
15688 match(Set dst (URShiftVL src shift));
15689 ins_cost(INSN_COST);
15690 effect(TEMP tmp);
15691 format %{ "negr $tmp,$shift\t"
15692 "ushl $dst,$src,$tmp\t# vector (2D)" %}
15693 ins_encode %{
15694 __ negr(as_FloatRegister($tmp$$reg), __ T16B,
15695 as_FloatRegister($shift$$reg));
15696 __ ushl(as_FloatRegister($dst$$reg), __ T2D,
15697 as_FloatRegister($src$$reg),
15698 as_FloatRegister($tmp$$reg));
15699 %}
15700 ins_pipe(vshift128);
15701 %}
15702
15703 instruct vsll2L_imm(vecX dst, vecX src, immI shift) %{
15704 predicate(n->as_Vector()->length() == 2);
15705 match(Set dst (LShiftVL src shift));
15706 ins_cost(INSN_COST);
15707 format %{ "shl $dst, $src, $shift\t# vector (2D)" %}
15708 ins_encode %{
15709 __ shl(as_FloatRegister($dst$$reg), __ T2D,
15710 as_FloatRegister($src$$reg),
15711 (int)$shift$$constant & 63);
15712 %}
15713 ins_pipe(vshift128_imm);
15714 %}
15715
15716 instruct vsra2L_imm(vecX dst, vecX src, immI shift) %{
15717 predicate(n->as_Vector()->length() == 2);
15718 match(Set dst (RShiftVL src shift));
15719 ins_cost(INSN_COST);
15720 format %{ "sshr $dst, $src, $shift\t# vector (2D)" %}
15721 ins_encode %{
15722 __ sshr(as_FloatRegister($dst$$reg), __ T2D,
15723 as_FloatRegister($src$$reg),
15724 -(int)$shift$$constant & 63);
15725 %}
15726 ins_pipe(vshift128_imm);
15727 %}
15728
15729 instruct vsrl2L_imm(vecX dst, vecX src, immI shift) %{
15730 predicate(n->as_Vector()->length() == 2);
15731 match(Set dst (URShiftVL src shift));
15732 ins_cost(INSN_COST);
15733 format %{ "ushr $dst, $src, $shift\t# vector (2D)" %}
15734 ins_encode %{
15735 __ ushr(as_FloatRegister($dst$$reg), __ T2D,
15736 as_FloatRegister($src$$reg),
15737 -(int)$shift$$constant & 63);
15738 %}
15739 ins_pipe(vshift128_imm);
15740 %}
15741
15742 //----------PEEPHOLE RULES-----------------------------------------------------
15743 // These must follow all instruction definitions as they use the names
15744 // defined in the instructions definitions.
15745 //
15746 // peepmatch ( root_instr_name [preceding_instruction]* );
15747 //
15748 // peepconstraint %{
15749 // (instruction_number.operand_name relational_op instruction_number.operand_name
15750 // [, ...] );
15751 // // instruction numbers are zero-based using left to right order in peepmatch
15752 //
15753 // peepreplace ( instr_name ( [instruction_number.operand_name]* ) );
15754 // // provide an instruction_number.operand_name for each operand that appears
15755 // // in the replacement instruction's match rule
15756 //
15757 // ---------VM FLAGS---------------------------------------------------------
15758 //
15759 // All peephole optimizations can be turned off using -XX:-OptoPeephole
15760 //
15761 // Each peephole rule is given an identifying number starting with zero and
15762 // increasing by one in the order seen by the parser. An individual peephole
15763 // can be enabled, and all others disabled, by using -XX:OptoPeepholeAt=#
15764 // on the command-line.
15765 //
15766 // ---------CURRENT LIMITATIONS----------------------------------------------
15767 //
15768 // Only match adjacent instructions in same basic block
15769 // Only equality constraints
15770 // Only constraints between operands, not (0.dest_reg == RAX_enc)
15771 // Only one replacement instruction
15772 //
15773 // ---------EXAMPLE----------------------------------------------------------
15774 //
15775 // // pertinent parts of existing instructions in architecture description
15776 // instruct movI(iRegINoSp dst, iRegI src)
15777 // %{
15778 // match(Set dst (CopyI src));
15779 // %}
15780 //
15781 // instruct incI_iReg(iRegINoSp dst, immI1 src, rFlagsReg cr)
15782 // %{
15783 // match(Set dst (AddI dst src));
15784 // effect(KILL cr);
15785 // %}
15786 //
15787 // // Change (inc mov) to lea
15788 // peephole %{
15789 // // increment preceeded by register-register move
15790 // peepmatch ( incI_iReg movI );
15791 // // require that the destination register of the increment
15792 // // match the destination register of the move
15793 // peepconstraint ( 0.dst == 1.dst );
15794 // // construct a replacement instruction that sets
15795 // // the destination to ( move's source register + one )
15796 // peepreplace ( leaI_iReg_immI( 0.dst 1.src 0.src ) );
15797 // %}
15798 //
15799
15800 // Implementation no longer uses movX instructions since
15801 // machine-independent system no longer uses CopyX nodes.
15802 //
15803 // peephole
15804 // %{
15805 // peepmatch (incI_iReg movI);
15806 // peepconstraint (0.dst == 1.dst);
15807 // peepreplace (leaI_iReg_immI(0.dst 1.src 0.src));
15808 // %}
15809
15810 // peephole
15811 // %{
15812 // peepmatch (decI_iReg movI);
15813 // peepconstraint (0.dst == 1.dst);
15814 // peepreplace (leaI_iReg_immI(0.dst 1.src 0.src));
15815 // %}
15816
15817 // peephole
15818 // %{
15819 // peepmatch (addI_iReg_imm movI);
15820 // peepconstraint (0.dst == 1.dst);
15821 // peepreplace (leaI_iReg_immI(0.dst 1.src 0.src));
15822 // %}
15823
15824 // peephole
15825 // %{
15826 // peepmatch (incL_iReg movL);
15827 // peepconstraint (0.dst == 1.dst);
15828 // peepreplace (leaL_iReg_immL(0.dst 1.src 0.src));
15829 // %}
15830
15831 // peephole
15832 // %{
15833 // peepmatch (decL_iReg movL);
15834 // peepconstraint (0.dst == 1.dst);
15835 // peepreplace (leaL_iReg_immL(0.dst 1.src 0.src));
15836 // %}
15837
15838 // peephole
15839 // %{
15840 // peepmatch (addL_iReg_imm movL);
15841 // peepconstraint (0.dst == 1.dst);
15842 // peepreplace (leaL_iReg_immL(0.dst 1.src 0.src));
15843 // %}
15844
15845 // peephole
15846 // %{
15847 // peepmatch (addP_iReg_imm movP);
15848 // peepconstraint (0.dst == 1.dst);
15849 // peepreplace (leaP_iReg_imm(0.dst 1.src 0.src));
15850 // %}
15851
15852 // // Change load of spilled value to only a spill
15853 // instruct storeI(memory mem, iRegI src)
15854 // %{
15855 // match(Set mem (StoreI mem src));
15856 // %}
15857 //
15858 // instruct loadI(iRegINoSp dst, memory mem)
15859 // %{
15860 // match(Set dst (LoadI mem));
15861 // %}
15862 //
15863
15864 //----------SMARTSPILL RULES---------------------------------------------------
15865 // These must follow all instruction definitions as they use the names
15866 // defined in the instructions definitions.
15867
15868 // Local Variables:
15869 // mode: c++
15870 // End: