1 //
2 // Copyright (c) 2003, 2022, Oracle and/or its affiliates. All rights reserved.
3 // Copyright (c) 2014, 2021, Red Hat, Inc. All rights reserved.
4 // DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
5 //
6 // This code is free software; you can redistribute it and/or modify it
7 // under the terms of the GNU General Public License version 2 only, as
8 // published by the Free Software Foundation.
9 //
10 // This code is distributed in the hope that it will be useful, but WITHOUT
11 // ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
12 // FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
13 // version 2 for more details (a copy is included in the LICENSE file that
14 // accompanied this code).
15 //
16 // You should have received a copy of the GNU General Public License version
17 // 2 along with this work; if not, write to the Free Software Foundation,
18 // Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
19 //
20 // Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
21 // or visit www.oracle.com if you need additional information or have any
22 // questions.
23 //
24 //
25
26 // AArch64 Architecture Description File
27
28 //----------REGISTER DEFINITION BLOCK------------------------------------------
29 // This information is used by the matcher and the register allocator to
30 // describe individual registers and classes of registers within the target
31 // architecture.
32
33 register %{
34 //----------Architecture Description Register Definitions----------------------
35 // General Registers
36 // "reg_def" name ( register save type, C convention save type,
37 // ideal register type, encoding );
38 // Register Save Types:
39 //
40 // NS = No-Save: The register allocator assumes that these registers
41 // can be used without saving upon entry to the method, &
42 // that they do not need to be saved at call sites.
43 //
44 // SOC = Save-On-Call: The register allocator assumes that these registers
45 // can be used without saving upon entry to the method,
46 // but that they must be saved at call sites.
47 //
48 // SOE = Save-On-Entry: The register allocator assumes that these registers
49 // must be saved before using them upon entry to the
50 // method, but they do not need to be saved at call
51 // sites.
52 //
53 // AS = Always-Save: The register allocator assumes that these registers
54 // must be saved before using them upon entry to the
55 // method, & that they must be saved at call sites.
56 //
57 // Ideal Register Type is used to determine how to save & restore a
58 // register. Op_RegI will get spilled with LoadI/StoreI, Op_RegP will get
59 // spilled with LoadP/StoreP. If the register supports both, use Op_RegI.
60 //
61 // The encoding number is the actual bit-pattern placed into the opcodes.
62
63 // We must define the 64 bit int registers in two 32 bit halves, the
64 // real lower register and a virtual upper half register. upper halves
65 // are used by the register allocator but are not actually supplied as
66 // operands to memory ops.
67 //
68 // follow the C1 compiler in making registers
69 //
70 // r0-r7,r10-r26 volatile (caller save)
71 // r27-r32 system (no save, no allocate)
72 // r8-r9 non-allocatable (so we can use them as scratch regs)
73 //
74 // as regards Java usage. we don't use any callee save registers
75 // because this makes it difficult to de-optimise a frame (see comment
76 // in x86 implementation of Deoptimization::unwind_callee_save_values)
77 //
78
79 // General Registers
80
81 reg_def R0 ( SOC, SOC, Op_RegI, 0, r0->as_VMReg() );
82 reg_def R0_H ( SOC, SOC, Op_RegI, 0, r0->as_VMReg()->next() );
83 reg_def R1 ( SOC, SOC, Op_RegI, 1, r1->as_VMReg() );
84 reg_def R1_H ( SOC, SOC, Op_RegI, 1, r1->as_VMReg()->next() );
85 reg_def R2 ( SOC, SOC, Op_RegI, 2, r2->as_VMReg() );
86 reg_def R2_H ( SOC, SOC, Op_RegI, 2, r2->as_VMReg()->next() );
87 reg_def R3 ( SOC, SOC, Op_RegI, 3, r3->as_VMReg() );
88 reg_def R3_H ( SOC, SOC, Op_RegI, 3, r3->as_VMReg()->next() );
89 reg_def R4 ( SOC, SOC, Op_RegI, 4, r4->as_VMReg() );
90 reg_def R4_H ( SOC, SOC, Op_RegI, 4, r4->as_VMReg()->next() );
91 reg_def R5 ( SOC, SOC, Op_RegI, 5, r5->as_VMReg() );
92 reg_def R5_H ( SOC, SOC, Op_RegI, 5, r5->as_VMReg()->next() );
93 reg_def R6 ( SOC, SOC, Op_RegI, 6, r6->as_VMReg() );
94 reg_def R6_H ( SOC, SOC, Op_RegI, 6, r6->as_VMReg()->next() );
95 reg_def R7 ( SOC, SOC, Op_RegI, 7, r7->as_VMReg() );
96 reg_def R7_H ( SOC, SOC, Op_RegI, 7, r7->as_VMReg()->next() );
97 reg_def R8 ( NS, SOC, Op_RegI, 8, r8->as_VMReg() ); // rscratch1, non-allocatable
98 reg_def R8_H ( NS, SOC, Op_RegI, 8, r8->as_VMReg()->next() );
99 reg_def R9 ( NS, SOC, Op_RegI, 9, r9->as_VMReg() ); // rscratch2, non-allocatable
100 reg_def R9_H ( NS, SOC, Op_RegI, 9, r9->as_VMReg()->next() );
101 reg_def R10 ( SOC, SOC, Op_RegI, 10, r10->as_VMReg() );
102 reg_def R10_H ( SOC, SOC, Op_RegI, 10, r10->as_VMReg()->next());
103 reg_def R11 ( SOC, SOC, Op_RegI, 11, r11->as_VMReg() );
104 reg_def R11_H ( SOC, SOC, Op_RegI, 11, r11->as_VMReg()->next());
105 reg_def R12 ( SOC, SOC, Op_RegI, 12, r12->as_VMReg() );
106 reg_def R12_H ( SOC, SOC, Op_RegI, 12, r12->as_VMReg()->next());
107 reg_def R13 ( SOC, SOC, Op_RegI, 13, r13->as_VMReg() );
108 reg_def R13_H ( SOC, SOC, Op_RegI, 13, r13->as_VMReg()->next());
109 reg_def R14 ( SOC, SOC, Op_RegI, 14, r14->as_VMReg() );
110 reg_def R14_H ( SOC, SOC, Op_RegI, 14, r14->as_VMReg()->next());
111 reg_def R15 ( SOC, SOC, Op_RegI, 15, r15->as_VMReg() );
112 reg_def R15_H ( SOC, SOC, Op_RegI, 15, r15->as_VMReg()->next());
113 reg_def R16 ( SOC, SOC, Op_RegI, 16, r16->as_VMReg() );
114 reg_def R16_H ( SOC, SOC, Op_RegI, 16, r16->as_VMReg()->next());
115 reg_def R17 ( SOC, SOC, Op_RegI, 17, r17->as_VMReg() );
116 reg_def R17_H ( SOC, SOC, Op_RegI, 17, r17->as_VMReg()->next());
117 reg_def R18 ( SOC, SOC, Op_RegI, 18, r18_tls->as_VMReg() );
118 reg_def R18_H ( SOC, SOC, Op_RegI, 18, r18_tls->as_VMReg()->next());
119 reg_def R19 ( SOC, SOE, Op_RegI, 19, r19->as_VMReg() );
120 reg_def R19_H ( SOC, SOE, Op_RegI, 19, r19->as_VMReg()->next());
121 reg_def R20 ( SOC, SOE, Op_RegI, 20, r20->as_VMReg() ); // caller esp
122 reg_def R20_H ( SOC, SOE, Op_RegI, 20, r20->as_VMReg()->next());
123 reg_def R21 ( SOC, SOE, Op_RegI, 21, r21->as_VMReg() );
124 reg_def R21_H ( SOC, SOE, Op_RegI, 21, r21->as_VMReg()->next());
125 reg_def R22 ( SOC, SOE, Op_RegI, 22, r22->as_VMReg() );
126 reg_def R22_H ( SOC, SOE, Op_RegI, 22, r22->as_VMReg()->next());
127 reg_def R23 ( SOC, SOE, Op_RegI, 23, r23->as_VMReg() );
128 reg_def R23_H ( SOC, SOE, Op_RegI, 23, r23->as_VMReg()->next());
129 reg_def R24 ( SOC, SOE, Op_RegI, 24, r24->as_VMReg() );
130 reg_def R24_H ( SOC, SOE, Op_RegI, 24, r24->as_VMReg()->next());
131 reg_def R25 ( SOC, SOE, Op_RegI, 25, r25->as_VMReg() );
132 reg_def R25_H ( SOC, SOE, Op_RegI, 25, r25->as_VMReg()->next());
133 reg_def R26 ( SOC, SOE, Op_RegI, 26, r26->as_VMReg() );
134 reg_def R26_H ( SOC, SOE, Op_RegI, 26, r26->as_VMReg()->next());
135 reg_def R27 ( SOC, SOE, Op_RegI, 27, r27->as_VMReg() ); // heapbase
136 reg_def R27_H ( SOC, SOE, Op_RegI, 27, r27->as_VMReg()->next());
137 reg_def R28 ( NS, SOE, Op_RegI, 28, r28->as_VMReg() ); // thread
138 reg_def R28_H ( NS, SOE, Op_RegI, 28, r28->as_VMReg()->next());
139 reg_def R29 ( NS, NS, Op_RegI, 29, r29->as_VMReg() ); // fp
140 reg_def R29_H ( NS, NS, Op_RegI, 29, r29->as_VMReg()->next());
141 reg_def R30 ( NS, NS, Op_RegI, 30, r30->as_VMReg() ); // lr
142 reg_def R30_H ( NS, NS, Op_RegI, 30, r30->as_VMReg()->next());
143 reg_def R31 ( NS, NS, Op_RegI, 31, r31_sp->as_VMReg() ); // sp
144 reg_def R31_H ( NS, NS, Op_RegI, 31, r31_sp->as_VMReg()->next());
145
146 // ----------------------------
147 // Float/Double/Vector Registers
148 // ----------------------------
149
150 // Double Registers
151
152 // The rules of ADL require that double registers be defined in pairs.
153 // Each pair must be two 32-bit values, but not necessarily a pair of
154 // single float registers. In each pair, ADLC-assigned register numbers
155 // must be adjacent, with the lower number even. Finally, when the
156 // CPU stores such a register pair to memory, the word associated with
157 // the lower ADLC-assigned number must be stored to the lower address.
158
159 // AArch64 has 32 floating-point registers. Each can store a vector of
160 // single or double precision floating-point values up to 8 * 32
161 // floats, 4 * 64 bit floats or 2 * 128 bit floats. We currently only
162 // use the first float or double element of the vector.
163
164 // for Java use float registers v0-v15 are always save on call whereas
165 // the platform ABI treats v8-v15 as callee save). float registers
166 // v16-v31 are SOC as per the platform spec
167
168 // For SVE vector registers, we simply extend vector register size to 8
169 // 'logical' slots. This is nominally 256 bits but it actually covers
170 // all possible 'physical' SVE vector register lengths from 128 ~ 2048
171 // bits. The 'physical' SVE vector register length is detected during
172 // startup, so the register allocator is able to identify the correct
173 // number of bytes needed for an SVE spill/unspill.
174 // Note that a vector register with 4 slots denotes a 128-bit NEON
175 // register allowing it to be distinguished from the corresponding SVE
176 // vector register when the SVE vector length is 128 bits.
177
178 reg_def V0 ( SOC, SOC, Op_RegF, 0, v0->as_VMReg() );
179 reg_def V0_H ( SOC, SOC, Op_RegF, 0, v0->as_VMReg()->next() );
180 reg_def V0_J ( SOC, SOC, Op_RegF, 0, v0->as_VMReg()->next(2) );
181 reg_def V0_K ( SOC, SOC, Op_RegF, 0, v0->as_VMReg()->next(3) );
182 reg_def V0_L ( SOC, SOC, Op_RegF, 0, v0->as_VMReg()->next(4) );
183 reg_def V0_M ( SOC, SOC, Op_RegF, 0, v0->as_VMReg()->next(5) );
184 reg_def V0_N ( SOC, SOC, Op_RegF, 0, v0->as_VMReg()->next(6) );
185 reg_def V0_O ( SOC, SOC, Op_RegF, 0, v0->as_VMReg()->next(7) );
186
187 reg_def V1 ( SOC, SOC, Op_RegF, 1, v1->as_VMReg() );
188 reg_def V1_H ( SOC, SOC, Op_RegF, 1, v1->as_VMReg()->next() );
189 reg_def V1_J ( SOC, SOC, Op_RegF, 1, v1->as_VMReg()->next(2) );
190 reg_def V1_K ( SOC, SOC, Op_RegF, 1, v1->as_VMReg()->next(3) );
191 reg_def V1_L ( SOC, SOC, Op_RegF, 1, v1->as_VMReg()->next(4) );
192 reg_def V1_M ( SOC, SOC, Op_RegF, 1, v1->as_VMReg()->next(5) );
193 reg_def V1_N ( SOC, SOC, Op_RegF, 1, v1->as_VMReg()->next(6) );
194 reg_def V1_O ( SOC, SOC, Op_RegF, 1, v1->as_VMReg()->next(7) );
195
196 reg_def V2 ( SOC, SOC, Op_RegF, 2, v2->as_VMReg() );
197 reg_def V2_H ( SOC, SOC, Op_RegF, 2, v2->as_VMReg()->next() );
198 reg_def V2_J ( SOC, SOC, Op_RegF, 2, v2->as_VMReg()->next(2) );
199 reg_def V2_K ( SOC, SOC, Op_RegF, 2, v2->as_VMReg()->next(3) );
200 reg_def V2_L ( SOC, SOC, Op_RegF, 2, v2->as_VMReg()->next(4) );
201 reg_def V2_M ( SOC, SOC, Op_RegF, 2, v2->as_VMReg()->next(5) );
202 reg_def V2_N ( SOC, SOC, Op_RegF, 2, v2->as_VMReg()->next(6) );
203 reg_def V2_O ( SOC, SOC, Op_RegF, 2, v2->as_VMReg()->next(7) );
204
205 reg_def V3 ( SOC, SOC, Op_RegF, 3, v3->as_VMReg() );
206 reg_def V3_H ( SOC, SOC, Op_RegF, 3, v3->as_VMReg()->next() );
207 reg_def V3_J ( SOC, SOC, Op_RegF, 3, v3->as_VMReg()->next(2) );
208 reg_def V3_K ( SOC, SOC, Op_RegF, 3, v3->as_VMReg()->next(3) );
209 reg_def V3_L ( SOC, SOC, Op_RegF, 3, v3->as_VMReg()->next(4) );
210 reg_def V3_M ( SOC, SOC, Op_RegF, 3, v3->as_VMReg()->next(5) );
211 reg_def V3_N ( SOC, SOC, Op_RegF, 3, v3->as_VMReg()->next(6) );
212 reg_def V3_O ( SOC, SOC, Op_RegF, 3, v3->as_VMReg()->next(7) );
213
214 reg_def V4 ( SOC, SOC, Op_RegF, 4, v4->as_VMReg() );
215 reg_def V4_H ( SOC, SOC, Op_RegF, 4, v4->as_VMReg()->next() );
216 reg_def V4_J ( SOC, SOC, Op_RegF, 4, v4->as_VMReg()->next(2) );
217 reg_def V4_K ( SOC, SOC, Op_RegF, 4, v4->as_VMReg()->next(3) );
218 reg_def V4_L ( SOC, SOC, Op_RegF, 4, v4->as_VMReg()->next(4) );
219 reg_def V4_M ( SOC, SOC, Op_RegF, 4, v4->as_VMReg()->next(5) );
220 reg_def V4_N ( SOC, SOC, Op_RegF, 4, v4->as_VMReg()->next(6) );
221 reg_def V4_O ( SOC, SOC, Op_RegF, 4, v4->as_VMReg()->next(7) );
222
223 reg_def V5 ( SOC, SOC, Op_RegF, 5, v5->as_VMReg() );
224 reg_def V5_H ( SOC, SOC, Op_RegF, 5, v5->as_VMReg()->next() );
225 reg_def V5_J ( SOC, SOC, Op_RegF, 5, v5->as_VMReg()->next(2) );
226 reg_def V5_K ( SOC, SOC, Op_RegF, 5, v5->as_VMReg()->next(3) );
227 reg_def V5_L ( SOC, SOC, Op_RegF, 5, v5->as_VMReg()->next(4) );
228 reg_def V5_M ( SOC, SOC, Op_RegF, 5, v5->as_VMReg()->next(5) );
229 reg_def V5_N ( SOC, SOC, Op_RegF, 5, v5->as_VMReg()->next(6) );
230 reg_def V5_O ( SOC, SOC, Op_RegF, 5, v5->as_VMReg()->next(7) );
231
232 reg_def V6 ( SOC, SOC, Op_RegF, 6, v6->as_VMReg() );
233 reg_def V6_H ( SOC, SOC, Op_RegF, 6, v6->as_VMReg()->next() );
234 reg_def V6_J ( SOC, SOC, Op_RegF, 6, v6->as_VMReg()->next(2) );
235 reg_def V6_K ( SOC, SOC, Op_RegF, 6, v6->as_VMReg()->next(3) );
236 reg_def V6_L ( SOC, SOC, Op_RegF, 6, v6->as_VMReg()->next(4) );
237 reg_def V6_M ( SOC, SOC, Op_RegF, 6, v6->as_VMReg()->next(5) );
238 reg_def V6_N ( SOC, SOC, Op_RegF, 6, v6->as_VMReg()->next(6) );
239 reg_def V6_O ( SOC, SOC, Op_RegF, 6, v6->as_VMReg()->next(7) );
240
241 reg_def V7 ( SOC, SOC, Op_RegF, 7, v7->as_VMReg() );
242 reg_def V7_H ( SOC, SOC, Op_RegF, 7, v7->as_VMReg()->next() );
243 reg_def V7_J ( SOC, SOC, Op_RegF, 7, v7->as_VMReg()->next(2) );
244 reg_def V7_K ( SOC, SOC, Op_RegF, 7, v7->as_VMReg()->next(3) );
245 reg_def V7_L ( SOC, SOC, Op_RegF, 7, v7->as_VMReg()->next(4) );
246 reg_def V7_M ( SOC, SOC, Op_RegF, 7, v7->as_VMReg()->next(5) );
247 reg_def V7_N ( SOC, SOC, Op_RegF, 7, v7->as_VMReg()->next(6) );
248 reg_def V7_O ( SOC, SOC, Op_RegF, 7, v7->as_VMReg()->next(7) );
249
250 reg_def V8 ( SOC, SOE, Op_RegF, 8, v8->as_VMReg() );
251 reg_def V8_H ( SOC, SOE, Op_RegF, 8, v8->as_VMReg()->next() );
252 reg_def V8_J ( SOC, SOC, Op_RegF, 8, v8->as_VMReg()->next(2) );
253 reg_def V8_K ( SOC, SOC, Op_RegF, 8, v8->as_VMReg()->next(3) );
254 reg_def V8_L ( SOC, SOC, Op_RegF, 8, v8->as_VMReg()->next(4) );
255 reg_def V8_M ( SOC, SOC, Op_RegF, 8, v8->as_VMReg()->next(5) );
256 reg_def V8_N ( SOC, SOC, Op_RegF, 8, v8->as_VMReg()->next(6) );
257 reg_def V8_O ( SOC, SOC, Op_RegF, 8, v8->as_VMReg()->next(7) );
258
259 reg_def V9 ( SOC, SOE, Op_RegF, 9, v9->as_VMReg() );
260 reg_def V9_H ( SOC, SOE, Op_RegF, 9, v9->as_VMReg()->next() );
261 reg_def V9_J ( SOC, SOC, Op_RegF, 9, v9->as_VMReg()->next(2) );
262 reg_def V9_K ( SOC, SOC, Op_RegF, 9, v9->as_VMReg()->next(3) );
263 reg_def V9_L ( SOC, SOC, Op_RegF, 9, v9->as_VMReg()->next(4) );
264 reg_def V9_M ( SOC, SOC, Op_RegF, 9, v9->as_VMReg()->next(5) );
265 reg_def V9_N ( SOC, SOC, Op_RegF, 9, v9->as_VMReg()->next(6) );
266 reg_def V9_O ( SOC, SOC, Op_RegF, 9, v9->as_VMReg()->next(7) );
267
268 reg_def V10 ( SOC, SOE, Op_RegF, 10, v10->as_VMReg() );
269 reg_def V10_H ( SOC, SOE, Op_RegF, 10, v10->as_VMReg()->next() );
270 reg_def V10_J ( SOC, SOC, Op_RegF, 10, v10->as_VMReg()->next(2) );
271 reg_def V10_K ( SOC, SOC, Op_RegF, 10, v10->as_VMReg()->next(3) );
272 reg_def V10_L ( SOC, SOC, Op_RegF, 10, v10->as_VMReg()->next(4) );
273 reg_def V10_M ( SOC, SOC, Op_RegF, 10, v10->as_VMReg()->next(5) );
274 reg_def V10_N ( SOC, SOC, Op_RegF, 10, v10->as_VMReg()->next(6) );
275 reg_def V10_O ( SOC, SOC, Op_RegF, 10, v10->as_VMReg()->next(7) );
276
277 reg_def V11 ( SOC, SOE, Op_RegF, 11, v11->as_VMReg() );
278 reg_def V11_H ( SOC, SOE, Op_RegF, 11, v11->as_VMReg()->next() );
279 reg_def V11_J ( SOC, SOC, Op_RegF, 11, v11->as_VMReg()->next(2) );
280 reg_def V11_K ( SOC, SOC, Op_RegF, 11, v11->as_VMReg()->next(3) );
281 reg_def V11_L ( SOC, SOC, Op_RegF, 11, v11->as_VMReg()->next(4) );
282 reg_def V11_M ( SOC, SOC, Op_RegF, 11, v11->as_VMReg()->next(5) );
283 reg_def V11_N ( SOC, SOC, Op_RegF, 11, v11->as_VMReg()->next(6) );
284 reg_def V11_O ( SOC, SOC, Op_RegF, 11, v11->as_VMReg()->next(7) );
285
286 reg_def V12 ( SOC, SOE, Op_RegF, 12, v12->as_VMReg() );
287 reg_def V12_H ( SOC, SOE, Op_RegF, 12, v12->as_VMReg()->next() );
288 reg_def V12_J ( SOC, SOC, Op_RegF, 12, v12->as_VMReg()->next(2) );
289 reg_def V12_K ( SOC, SOC, Op_RegF, 12, v12->as_VMReg()->next(3) );
290 reg_def V12_L ( SOC, SOC, Op_RegF, 12, v12->as_VMReg()->next(4) );
291 reg_def V12_M ( SOC, SOC, Op_RegF, 12, v12->as_VMReg()->next(5) );
292 reg_def V12_N ( SOC, SOC, Op_RegF, 12, v12->as_VMReg()->next(6) );
293 reg_def V12_O ( SOC, SOC, Op_RegF, 12, v12->as_VMReg()->next(7) );
294
295 reg_def V13 ( SOC, SOE, Op_RegF, 13, v13->as_VMReg() );
296 reg_def V13_H ( SOC, SOE, Op_RegF, 13, v13->as_VMReg()->next() );
297 reg_def V13_J ( SOC, SOC, Op_RegF, 13, v13->as_VMReg()->next(2) );
298 reg_def V13_K ( SOC, SOC, Op_RegF, 13, v13->as_VMReg()->next(3) );
299 reg_def V13_L ( SOC, SOC, Op_RegF, 13, v13->as_VMReg()->next(4) );
300 reg_def V13_M ( SOC, SOC, Op_RegF, 13, v13->as_VMReg()->next(5) );
301 reg_def V13_N ( SOC, SOC, Op_RegF, 13, v13->as_VMReg()->next(6) );
302 reg_def V13_O ( SOC, SOC, Op_RegF, 13, v13->as_VMReg()->next(7) );
303
304 reg_def V14 ( SOC, SOE, Op_RegF, 14, v14->as_VMReg() );
305 reg_def V14_H ( SOC, SOE, Op_RegF, 14, v14->as_VMReg()->next() );
306 reg_def V14_J ( SOC, SOC, Op_RegF, 14, v14->as_VMReg()->next(2) );
307 reg_def V14_K ( SOC, SOC, Op_RegF, 14, v14->as_VMReg()->next(3) );
308 reg_def V14_L ( SOC, SOC, Op_RegF, 14, v14->as_VMReg()->next(4) );
309 reg_def V14_M ( SOC, SOC, Op_RegF, 14, v14->as_VMReg()->next(5) );
310 reg_def V14_N ( SOC, SOC, Op_RegF, 14, v14->as_VMReg()->next(6) );
311 reg_def V14_O ( SOC, SOC, Op_RegF, 14, v14->as_VMReg()->next(7) );
312
313 reg_def V15 ( SOC, SOE, Op_RegF, 15, v15->as_VMReg() );
314 reg_def V15_H ( SOC, SOE, Op_RegF, 15, v15->as_VMReg()->next() );
315 reg_def V15_J ( SOC, SOC, Op_RegF, 15, v15->as_VMReg()->next(2) );
316 reg_def V15_K ( SOC, SOC, Op_RegF, 15, v15->as_VMReg()->next(3) );
317 reg_def V15_L ( SOC, SOC, Op_RegF, 15, v15->as_VMReg()->next(4) );
318 reg_def V15_M ( SOC, SOC, Op_RegF, 15, v15->as_VMReg()->next(5) );
319 reg_def V15_N ( SOC, SOC, Op_RegF, 15, v15->as_VMReg()->next(6) );
320 reg_def V15_O ( SOC, SOC, Op_RegF, 15, v15->as_VMReg()->next(7) );
321
322 reg_def V16 ( SOC, SOC, Op_RegF, 16, v16->as_VMReg() );
323 reg_def V16_H ( SOC, SOC, Op_RegF, 16, v16->as_VMReg()->next() );
324 reg_def V16_J ( SOC, SOC, Op_RegF, 16, v16->as_VMReg()->next(2) );
325 reg_def V16_K ( SOC, SOC, Op_RegF, 16, v16->as_VMReg()->next(3) );
326 reg_def V16_L ( SOC, SOC, Op_RegF, 16, v16->as_VMReg()->next(4) );
327 reg_def V16_M ( SOC, SOC, Op_RegF, 16, v16->as_VMReg()->next(5) );
328 reg_def V16_N ( SOC, SOC, Op_RegF, 16, v16->as_VMReg()->next(6) );
329 reg_def V16_O ( SOC, SOC, Op_RegF, 16, v16->as_VMReg()->next(7) );
330
331 reg_def V17 ( SOC, SOC, Op_RegF, 17, v17->as_VMReg() );
332 reg_def V17_H ( SOC, SOC, Op_RegF, 17, v17->as_VMReg()->next() );
333 reg_def V17_J ( SOC, SOC, Op_RegF, 17, v17->as_VMReg()->next(2) );
334 reg_def V17_K ( SOC, SOC, Op_RegF, 17, v17->as_VMReg()->next(3) );
335 reg_def V17_L ( SOC, SOC, Op_RegF, 17, v17->as_VMReg()->next(4) );
336 reg_def V17_M ( SOC, SOC, Op_RegF, 17, v17->as_VMReg()->next(5) );
337 reg_def V17_N ( SOC, SOC, Op_RegF, 17, v17->as_VMReg()->next(6) );
338 reg_def V17_O ( SOC, SOC, Op_RegF, 17, v17->as_VMReg()->next(7) );
339
340 reg_def V18 ( SOC, SOC, Op_RegF, 18, v18->as_VMReg() );
341 reg_def V18_H ( SOC, SOC, Op_RegF, 18, v18->as_VMReg()->next() );
342 reg_def V18_J ( SOC, SOC, Op_RegF, 18, v18->as_VMReg()->next(2) );
343 reg_def V18_K ( SOC, SOC, Op_RegF, 18, v18->as_VMReg()->next(3) );
344 reg_def V18_L ( SOC, SOC, Op_RegF, 18, v18->as_VMReg()->next(4) );
345 reg_def V18_M ( SOC, SOC, Op_RegF, 18, v18->as_VMReg()->next(5) );
346 reg_def V18_N ( SOC, SOC, Op_RegF, 18, v18->as_VMReg()->next(6) );
347 reg_def V18_O ( SOC, SOC, Op_RegF, 18, v18->as_VMReg()->next(7) );
348
349 reg_def V19 ( SOC, SOC, Op_RegF, 19, v19->as_VMReg() );
350 reg_def V19_H ( SOC, SOC, Op_RegF, 19, v19->as_VMReg()->next() );
351 reg_def V19_J ( SOC, SOC, Op_RegF, 19, v19->as_VMReg()->next(2) );
352 reg_def V19_K ( SOC, SOC, Op_RegF, 19, v19->as_VMReg()->next(3) );
353 reg_def V19_L ( SOC, SOC, Op_RegF, 19, v19->as_VMReg()->next(4) );
354 reg_def V19_M ( SOC, SOC, Op_RegF, 19, v19->as_VMReg()->next(5) );
355 reg_def V19_N ( SOC, SOC, Op_RegF, 19, v19->as_VMReg()->next(6) );
356 reg_def V19_O ( SOC, SOC, Op_RegF, 19, v19->as_VMReg()->next(7) );
357
358 reg_def V20 ( SOC, SOC, Op_RegF, 20, v20->as_VMReg() );
359 reg_def V20_H ( SOC, SOC, Op_RegF, 20, v20->as_VMReg()->next() );
360 reg_def V20_J ( SOC, SOC, Op_RegF, 20, v20->as_VMReg()->next(2) );
361 reg_def V20_K ( SOC, SOC, Op_RegF, 20, v20->as_VMReg()->next(3) );
362 reg_def V20_L ( SOC, SOC, Op_RegF, 20, v20->as_VMReg()->next(4) );
363 reg_def V20_M ( SOC, SOC, Op_RegF, 20, v20->as_VMReg()->next(5) );
364 reg_def V20_N ( SOC, SOC, Op_RegF, 20, v20->as_VMReg()->next(6) );
365 reg_def V20_O ( SOC, SOC, Op_RegF, 20, v20->as_VMReg()->next(7) );
366
367 reg_def V21 ( SOC, SOC, Op_RegF, 21, v21->as_VMReg() );
368 reg_def V21_H ( SOC, SOC, Op_RegF, 21, v21->as_VMReg()->next() );
369 reg_def V21_J ( SOC, SOC, Op_RegF, 21, v21->as_VMReg()->next(2) );
370 reg_def V21_K ( SOC, SOC, Op_RegF, 21, v21->as_VMReg()->next(3) );
371 reg_def V21_L ( SOC, SOC, Op_RegF, 21, v21->as_VMReg()->next(4) );
372 reg_def V21_M ( SOC, SOC, Op_RegF, 21, v21->as_VMReg()->next(5) );
373 reg_def V21_N ( SOC, SOC, Op_RegF, 21, v21->as_VMReg()->next(6) );
374 reg_def V21_O ( SOC, SOC, Op_RegF, 21, v21->as_VMReg()->next(7) );
375
376 reg_def V22 ( SOC, SOC, Op_RegF, 22, v22->as_VMReg() );
377 reg_def V22_H ( SOC, SOC, Op_RegF, 22, v22->as_VMReg()->next() );
378 reg_def V22_J ( SOC, SOC, Op_RegF, 22, v22->as_VMReg()->next(2) );
379 reg_def V22_K ( SOC, SOC, Op_RegF, 22, v22->as_VMReg()->next(3) );
380 reg_def V22_L ( SOC, SOC, Op_RegF, 22, v22->as_VMReg()->next(4) );
381 reg_def V22_M ( SOC, SOC, Op_RegF, 22, v22->as_VMReg()->next(5) );
382 reg_def V22_N ( SOC, SOC, Op_RegF, 22, v22->as_VMReg()->next(6) );
383 reg_def V22_O ( SOC, SOC, Op_RegF, 22, v22->as_VMReg()->next(7) );
384
385 reg_def V23 ( SOC, SOC, Op_RegF, 23, v23->as_VMReg() );
386 reg_def V23_H ( SOC, SOC, Op_RegF, 23, v23->as_VMReg()->next() );
387 reg_def V23_J ( SOC, SOC, Op_RegF, 23, v23->as_VMReg()->next(2) );
388 reg_def V23_K ( SOC, SOC, Op_RegF, 23, v23->as_VMReg()->next(3) );
389 reg_def V23_L ( SOC, SOC, Op_RegF, 23, v23->as_VMReg()->next(4) );
390 reg_def V23_M ( SOC, SOC, Op_RegF, 23, v23->as_VMReg()->next(5) );
391 reg_def V23_N ( SOC, SOC, Op_RegF, 23, v23->as_VMReg()->next(6) );
392 reg_def V23_O ( SOC, SOC, Op_RegF, 23, v23->as_VMReg()->next(7) );
393
394 reg_def V24 ( SOC, SOC, Op_RegF, 24, v24->as_VMReg() );
395 reg_def V24_H ( SOC, SOC, Op_RegF, 24, v24->as_VMReg()->next() );
396 reg_def V24_J ( SOC, SOC, Op_RegF, 24, v24->as_VMReg()->next(2) );
397 reg_def V24_K ( SOC, SOC, Op_RegF, 24, v24->as_VMReg()->next(3) );
398 reg_def V24_L ( SOC, SOC, Op_RegF, 24, v24->as_VMReg()->next(4) );
399 reg_def V24_M ( SOC, SOC, Op_RegF, 24, v24->as_VMReg()->next(5) );
400 reg_def V24_N ( SOC, SOC, Op_RegF, 24, v24->as_VMReg()->next(6) );
401 reg_def V24_O ( SOC, SOC, Op_RegF, 24, v24->as_VMReg()->next(7) );
402
403 reg_def V25 ( SOC, SOC, Op_RegF, 25, v25->as_VMReg() );
404 reg_def V25_H ( SOC, SOC, Op_RegF, 25, v25->as_VMReg()->next() );
405 reg_def V25_J ( SOC, SOC, Op_RegF, 25, v25->as_VMReg()->next(2) );
406 reg_def V25_K ( SOC, SOC, Op_RegF, 25, v25->as_VMReg()->next(3) );
407 reg_def V25_L ( SOC, SOC, Op_RegF, 25, v25->as_VMReg()->next(4) );
408 reg_def V25_M ( SOC, SOC, Op_RegF, 25, v25->as_VMReg()->next(5) );
409 reg_def V25_N ( SOC, SOC, Op_RegF, 25, v25->as_VMReg()->next(6) );
410 reg_def V25_O ( SOC, SOC, Op_RegF, 25, v25->as_VMReg()->next(7) );
411
412 reg_def V26 ( SOC, SOC, Op_RegF, 26, v26->as_VMReg() );
413 reg_def V26_H ( SOC, SOC, Op_RegF, 26, v26->as_VMReg()->next() );
414 reg_def V26_J ( SOC, SOC, Op_RegF, 26, v26->as_VMReg()->next(2) );
415 reg_def V26_K ( SOC, SOC, Op_RegF, 26, v26->as_VMReg()->next(3) );
416 reg_def V26_L ( SOC, SOC, Op_RegF, 26, v26->as_VMReg()->next(4) );
417 reg_def V26_M ( SOC, SOC, Op_RegF, 26, v26->as_VMReg()->next(5) );
418 reg_def V26_N ( SOC, SOC, Op_RegF, 26, v26->as_VMReg()->next(6) );
419 reg_def V26_O ( SOC, SOC, Op_RegF, 26, v26->as_VMReg()->next(7) );
420
421 reg_def V27 ( SOC, SOC, Op_RegF, 27, v27->as_VMReg() );
422 reg_def V27_H ( SOC, SOC, Op_RegF, 27, v27->as_VMReg()->next() );
423 reg_def V27_J ( SOC, SOC, Op_RegF, 27, v27->as_VMReg()->next(2) );
424 reg_def V27_K ( SOC, SOC, Op_RegF, 27, v27->as_VMReg()->next(3) );
425 reg_def V27_L ( SOC, SOC, Op_RegF, 27, v27->as_VMReg()->next(4) );
426 reg_def V27_M ( SOC, SOC, Op_RegF, 27, v27->as_VMReg()->next(5) );
427 reg_def V27_N ( SOC, SOC, Op_RegF, 27, v27->as_VMReg()->next(6) );
428 reg_def V27_O ( SOC, SOC, Op_RegF, 27, v27->as_VMReg()->next(7) );
429
430 reg_def V28 ( SOC, SOC, Op_RegF, 28, v28->as_VMReg() );
431 reg_def V28_H ( SOC, SOC, Op_RegF, 28, v28->as_VMReg()->next() );
432 reg_def V28_J ( SOC, SOC, Op_RegF, 28, v28->as_VMReg()->next(2) );
433 reg_def V28_K ( SOC, SOC, Op_RegF, 28, v28->as_VMReg()->next(3) );
434 reg_def V28_L ( SOC, SOC, Op_RegF, 28, v28->as_VMReg()->next(4) );
435 reg_def V28_M ( SOC, SOC, Op_RegF, 28, v28->as_VMReg()->next(5) );
436 reg_def V28_N ( SOC, SOC, Op_RegF, 28, v28->as_VMReg()->next(6) );
437 reg_def V28_O ( SOC, SOC, Op_RegF, 28, v28->as_VMReg()->next(7) );
438
439 reg_def V29 ( SOC, SOC, Op_RegF, 29, v29->as_VMReg() );
440 reg_def V29_H ( SOC, SOC, Op_RegF, 29, v29->as_VMReg()->next() );
441 reg_def V29_J ( SOC, SOC, Op_RegF, 29, v29->as_VMReg()->next(2) );
442 reg_def V29_K ( SOC, SOC, Op_RegF, 29, v29->as_VMReg()->next(3) );
443 reg_def V29_L ( SOC, SOC, Op_RegF, 29, v29->as_VMReg()->next(4) );
444 reg_def V29_M ( SOC, SOC, Op_RegF, 29, v29->as_VMReg()->next(5) );
445 reg_def V29_N ( SOC, SOC, Op_RegF, 29, v29->as_VMReg()->next(6) );
446 reg_def V29_O ( SOC, SOC, Op_RegF, 29, v29->as_VMReg()->next(7) );
447
448 reg_def V30 ( SOC, SOC, Op_RegF, 30, v30->as_VMReg() );
449 reg_def V30_H ( SOC, SOC, Op_RegF, 30, v30->as_VMReg()->next() );
450 reg_def V30_J ( SOC, SOC, Op_RegF, 30, v30->as_VMReg()->next(2) );
451 reg_def V30_K ( SOC, SOC, Op_RegF, 30, v30->as_VMReg()->next(3) );
452 reg_def V30_L ( SOC, SOC, Op_RegF, 30, v30->as_VMReg()->next(4) );
453 reg_def V30_M ( SOC, SOC, Op_RegF, 30, v30->as_VMReg()->next(5) );
454 reg_def V30_N ( SOC, SOC, Op_RegF, 30, v30->as_VMReg()->next(6) );
455 reg_def V30_O ( SOC, SOC, Op_RegF, 30, v30->as_VMReg()->next(7) );
456
457 reg_def V31 ( SOC, SOC, Op_RegF, 31, v31->as_VMReg() );
458 reg_def V31_H ( SOC, SOC, Op_RegF, 31, v31->as_VMReg()->next() );
459 reg_def V31_J ( SOC, SOC, Op_RegF, 31, v31->as_VMReg()->next(2) );
460 reg_def V31_K ( SOC, SOC, Op_RegF, 31, v31->as_VMReg()->next(3) );
461 reg_def V31_L ( SOC, SOC, Op_RegF, 31, v31->as_VMReg()->next(4) );
462 reg_def V31_M ( SOC, SOC, Op_RegF, 31, v31->as_VMReg()->next(5) );
463 reg_def V31_N ( SOC, SOC, Op_RegF, 31, v31->as_VMReg()->next(6) );
464 reg_def V31_O ( SOC, SOC, Op_RegF, 31, v31->as_VMReg()->next(7) );
465
466
467 // ----------------------------
468 // SVE Predicate Registers
469 // ----------------------------
470 reg_def P0 (SOC, SOC, Op_RegVectMask, 0, p0->as_VMReg());
471 reg_def P1 (SOC, SOC, Op_RegVectMask, 1, p1->as_VMReg());
472 reg_def P2 (SOC, SOC, Op_RegVectMask, 2, p2->as_VMReg());
473 reg_def P3 (SOC, SOC, Op_RegVectMask, 3, p3->as_VMReg());
474 reg_def P4 (SOC, SOC, Op_RegVectMask, 4, p4->as_VMReg());
475 reg_def P5 (SOC, SOC, Op_RegVectMask, 5, p5->as_VMReg());
476 reg_def P6 (SOC, SOC, Op_RegVectMask, 6, p6->as_VMReg());
477 reg_def P7 (SOC, SOC, Op_RegVectMask, 7, p7->as_VMReg());
478 reg_def P8 (SOC, SOC, Op_RegVectMask, 8, p8->as_VMReg());
479 reg_def P9 (SOC, SOC, Op_RegVectMask, 9, p9->as_VMReg());
480 reg_def P10 (SOC, SOC, Op_RegVectMask, 10, p10->as_VMReg());
481 reg_def P11 (SOC, SOC, Op_RegVectMask, 11, p11->as_VMReg());
482 reg_def P12 (SOC, SOC, Op_RegVectMask, 12, p12->as_VMReg());
483 reg_def P13 (SOC, SOC, Op_RegVectMask, 13, p13->as_VMReg());
484 reg_def P14 (SOC, SOC, Op_RegVectMask, 14, p14->as_VMReg());
485 reg_def P15 (SOC, SOC, Op_RegVectMask, 15, p15->as_VMReg());
486
487 // ----------------------------
488 // Special Registers
489 // ----------------------------
490
491 // the AArch64 CSPR status flag register is not directly accessible as
492 // instruction operand. the FPSR status flag register is a system
493 // register which can be written/read using MSR/MRS but again does not
494 // appear as an operand (a code identifying the FSPR occurs as an
495 // immediate value in the instruction).
496
497 reg_def RFLAGS(SOC, SOC, 0, 32, VMRegImpl::Bad());
498
499 // Specify priority of register selection within phases of register
500 // allocation. Highest priority is first. A useful heuristic is to
501 // give registers a low priority when they are required by machine
502 // instructions, like EAX and EDX on I486, and choose no-save registers
503 // before save-on-call, & save-on-call before save-on-entry. Registers
504 // which participate in fixed calling sequences should come last.
505 // Registers which are used as pairs must fall on an even boundary.
506
507 alloc_class chunk0(
508 // volatiles
509 R10, R10_H,
510 R11, R11_H,
511 R12, R12_H,
512 R13, R13_H,
513 R14, R14_H,
514 R15, R15_H,
515 R16, R16_H,
516 R17, R17_H,
517 R18, R18_H,
518
519 // arg registers
520 R0, R0_H,
521 R1, R1_H,
522 R2, R2_H,
523 R3, R3_H,
524 R4, R4_H,
525 R5, R5_H,
526 R6, R6_H,
527 R7, R7_H,
528
529 // non-volatiles
530 R19, R19_H,
531 R20, R20_H,
532 R21, R21_H,
533 R22, R22_H,
534 R23, R23_H,
535 R24, R24_H,
536 R25, R25_H,
537 R26, R26_H,
538
539 // non-allocatable registers
540
541 R27, R27_H, // heapbase
542 R28, R28_H, // thread
543 R29, R29_H, // fp
544 R30, R30_H, // lr
545 R31, R31_H, // sp
546 R8, R8_H, // rscratch1
547 R9, R9_H, // rscratch2
548 );
549
550 alloc_class chunk1(
551
552 // no save
553 V16, V16_H, V16_J, V16_K, V16_L, V16_M, V16_N, V16_O,
554 V17, V17_H, V17_J, V17_K, V17_L, V17_M, V17_N, V17_O,
555 V18, V18_H, V18_J, V18_K, V18_L, V18_M, V18_N, V18_O,
556 V19, V19_H, V19_J, V19_K, V19_L, V19_M, V19_N, V19_O,
557 V20, V20_H, V20_J, V20_K, V20_L, V20_M, V20_N, V20_O,
558 V21, V21_H, V21_J, V21_K, V21_L, V21_M, V21_N, V21_O,
559 V22, V22_H, V22_J, V22_K, V22_L, V22_M, V22_N, V22_O,
560 V23, V23_H, V23_J, V23_K, V23_L, V23_M, V23_N, V23_O,
561 V24, V24_H, V24_J, V24_K, V24_L, V24_M, V24_N, V24_O,
562 V25, V25_H, V25_J, V25_K, V25_L, V25_M, V25_N, V25_O,
563 V26, V26_H, V26_J, V26_K, V26_L, V26_M, V26_N, V26_O,
564 V27, V27_H, V27_J, V27_K, V27_L, V27_M, V27_N, V27_O,
565 V28, V28_H, V28_J, V28_K, V28_L, V28_M, V28_N, V28_O,
566 V29, V29_H, V29_J, V29_K, V29_L, V29_M, V29_N, V29_O,
567 V30, V30_H, V30_J, V30_K, V30_L, V30_M, V30_N, V30_O,
568 V31, V31_H, V31_J, V31_K, V31_L, V31_M, V31_N, V31_O,
569
570 // arg registers
571 V0, V0_H, V0_J, V0_K, V0_L, V0_M, V0_N, V0_O,
572 V1, V1_H, V1_J, V1_K, V1_L, V1_M, V1_N, V1_O,
573 V2, V2_H, V2_J, V2_K, V2_L, V2_M, V2_N, V2_O,
574 V3, V3_H, V3_J, V3_K, V3_L, V3_M, V3_N, V3_O,
575 V4, V4_H, V4_J, V4_K, V4_L, V4_M, V4_N, V4_O,
576 V5, V5_H, V5_J, V5_K, V5_L, V5_M, V5_N, V5_O,
577 V6, V6_H, V6_J, V6_K, V6_L, V6_M, V6_N, V6_O,
578 V7, V7_H, V7_J, V7_K, V7_L, V7_M, V7_N, V7_O,
579
580 // non-volatiles
581 V8, V8_H, V8_J, V8_K, V8_L, V8_M, V8_N, V8_O,
582 V9, V9_H, V9_J, V9_K, V9_L, V9_M, V9_N, V9_O,
583 V10, V10_H, V10_J, V10_K, V10_L, V10_M, V10_N, V10_O,
584 V11, V11_H, V11_J, V11_K, V11_L, V11_M, V11_N, V11_O,
585 V12, V12_H, V12_J, V12_K, V12_L, V12_M, V12_N, V12_O,
586 V13, V13_H, V13_J, V13_K, V13_L, V13_M, V13_N, V13_O,
587 V14, V14_H, V14_J, V14_K, V14_L, V14_M, V14_N, V14_O,
588 V15, V15_H, V15_J, V15_K, V15_L, V15_M, V15_N, V15_O,
589 );
590
591 alloc_class chunk2 (
592 P0,
593 P1,
594 P2,
595 P3,
596 P4,
597 P5,
598 P6,
599 P7,
600
601 P8,
602 P9,
603 P10,
604 P11,
605 P12,
606 P13,
607 P14,
608 P15,
609 );
610
611 alloc_class chunk3(RFLAGS);
612
613 //----------Architecture Description Register Classes--------------------------
614 // Several register classes are automatically defined based upon information in
615 // this architecture description.
616 // 1) reg_class inline_cache_reg ( /* as def'd in frame section */ )
617 // 2) reg_class stack_slots( /* one chunk of stack-based "registers" */ )
618 //
619
620 // Class for all 32 bit general purpose registers
621 reg_class all_reg32(
622 R0,
623 R1,
624 R2,
625 R3,
626 R4,
627 R5,
628 R6,
629 R7,
630 R10,
631 R11,
632 R12,
633 R13,
634 R14,
635 R15,
636 R16,
637 R17,
638 R18,
639 R19,
640 R20,
641 R21,
642 R22,
643 R23,
644 R24,
645 R25,
646 R26,
647 R27,
648 R28,
649 R29,
650 R30,
651 R31
652 );
653
654
655 // Class for all 32 bit integer registers (excluding SP which
656 // will never be used as an integer register)
657 reg_class any_reg32 %{
658 return _ANY_REG32_mask;
659 %}
660
661 // Singleton class for R0 int register
662 reg_class int_r0_reg(R0);
663
664 // Singleton class for R2 int register
665 reg_class int_r2_reg(R2);
666
667 // Singleton class for R3 int register
668 reg_class int_r3_reg(R3);
669
670 // Singleton class for R4 int register
671 reg_class int_r4_reg(R4);
672
673 // Singleton class for R31 int register
674 reg_class int_r31_reg(R31);
675
676 // Class for all 64 bit general purpose registers
677 reg_class all_reg(
678 R0, R0_H,
679 R1, R1_H,
680 R2, R2_H,
681 R3, R3_H,
682 R4, R4_H,
683 R5, R5_H,
684 R6, R6_H,
685 R7, R7_H,
686 R10, R10_H,
687 R11, R11_H,
688 R12, R12_H,
689 R13, R13_H,
690 R14, R14_H,
691 R15, R15_H,
692 R16, R16_H,
693 R17, R17_H,
694 R18, R18_H,
695 R19, R19_H,
696 R20, R20_H,
697 R21, R21_H,
698 R22, R22_H,
699 R23, R23_H,
700 R24, R24_H,
701 R25, R25_H,
702 R26, R26_H,
703 R27, R27_H,
704 R28, R28_H,
705 R29, R29_H,
706 R30, R30_H,
707 R31, R31_H
708 );
709
710 // Class for all long integer registers (including SP)
711 reg_class any_reg %{
712 return _ANY_REG_mask;
713 %}
714
715 // Class for non-allocatable 32 bit registers
716 reg_class non_allocatable_reg32(
717 #ifdef R18_RESERVED
718 // See comment in register_aarch64.hpp
719 R18, // tls on Windows
720 #endif
721 R28, // thread
722 R30, // lr
723 R31 // sp
724 );
725
726 // Class for non-allocatable 64 bit registers
727 reg_class non_allocatable_reg(
728 #ifdef R18_RESERVED
729 // See comment in register_aarch64.hpp
730 R18, R18_H, // tls on Windows, platform register on macOS
731 #endif
732 R28, R28_H, // thread
733 R30, R30_H, // lr
734 R31, R31_H // sp
735 );
736
737 // Class for all non-special integer registers
738 reg_class no_special_reg32 %{
739 return _NO_SPECIAL_REG32_mask;
740 %}
741
742 // Class for all non-special long integer registers
743 reg_class no_special_reg %{
744 return _NO_SPECIAL_REG_mask;
745 %}
746
747 // Class for 64 bit register r0
748 reg_class r0_reg(
749 R0, R0_H
750 );
751
752 // Class for 64 bit register r1
753 reg_class r1_reg(
754 R1, R1_H
755 );
756
757 // Class for 64 bit register r2
758 reg_class r2_reg(
759 R2, R2_H
760 );
761
762 // Class for 64 bit register r3
763 reg_class r3_reg(
764 R3, R3_H
765 );
766
767 // Class for 64 bit register r4
768 reg_class r4_reg(
769 R4, R4_H
770 );
771
772 // Class for 64 bit register r5
773 reg_class r5_reg(
774 R5, R5_H
775 );
776
777 // Class for 64 bit register r10
778 reg_class r10_reg(
779 R10, R10_H
780 );
781
782 // Class for 64 bit register r11
783 reg_class r11_reg(
784 R11, R11_H
785 );
786
787 // Class for method register
788 reg_class method_reg(
789 R12, R12_H
790 );
791
792 // Class for heapbase register
793 reg_class heapbase_reg(
794 R27, R27_H
795 );
796
797 // Class for thread register
798 reg_class thread_reg(
799 R28, R28_H
800 );
801
802 // Class for frame pointer register
803 reg_class fp_reg(
804 R29, R29_H
805 );
806
807 // Class for link register
808 reg_class lr_reg(
809 R30, R30_H
810 );
811
812 // Class for long sp register
813 reg_class sp_reg(
814 R31, R31_H
815 );
816
817 // Class for all pointer registers
818 reg_class ptr_reg %{
819 return _PTR_REG_mask;
820 %}
821
822 // Class for all non_special pointer registers
823 reg_class no_special_ptr_reg %{
824 return _NO_SPECIAL_PTR_REG_mask;
825 %}
826
827 // Class for all float registers
828 reg_class float_reg(
829 V0,
830 V1,
831 V2,
832 V3,
833 V4,
834 V5,
835 V6,
836 V7,
837 V8,
838 V9,
839 V10,
840 V11,
841 V12,
842 V13,
843 V14,
844 V15,
845 V16,
846 V17,
847 V18,
848 V19,
849 V20,
850 V21,
851 V22,
852 V23,
853 V24,
854 V25,
855 V26,
856 V27,
857 V28,
858 V29,
859 V30,
860 V31
861 );
862
863 // Double precision float registers have virtual `high halves' that
864 // are needed by the allocator.
865 // Class for all double registers
866 reg_class double_reg(
867 V0, V0_H,
868 V1, V1_H,
869 V2, V2_H,
870 V3, V3_H,
871 V4, V4_H,
872 V5, V5_H,
873 V6, V6_H,
874 V7, V7_H,
875 V8, V8_H,
876 V9, V9_H,
877 V10, V10_H,
878 V11, V11_H,
879 V12, V12_H,
880 V13, V13_H,
881 V14, V14_H,
882 V15, V15_H,
883 V16, V16_H,
884 V17, V17_H,
885 V18, V18_H,
886 V19, V19_H,
887 V20, V20_H,
888 V21, V21_H,
889 V22, V22_H,
890 V23, V23_H,
891 V24, V24_H,
892 V25, V25_H,
893 V26, V26_H,
894 V27, V27_H,
895 V28, V28_H,
896 V29, V29_H,
897 V30, V30_H,
898 V31, V31_H
899 );
900
901 // Class for all SVE vector registers.
902 reg_class vectora_reg (
903 V0, V0_H, V0_J, V0_K, V0_L, V0_M, V0_N, V0_O,
904 V1, V1_H, V1_J, V1_K, V1_L, V1_M, V1_N, V1_O,
905 V2, V2_H, V2_J, V2_K, V2_L, V2_M, V2_N, V2_O,
906 V3, V3_H, V3_J, V3_K, V3_L, V3_M, V3_N, V3_O,
907 V4, V4_H, V4_J, V4_K, V4_L, V4_M, V4_N, V4_O,
908 V5, V5_H, V5_J, V5_K, V5_L, V5_M, V5_N, V5_O,
909 V6, V6_H, V6_J, V6_K, V6_L, V6_M, V6_N, V6_O,
910 V7, V7_H, V7_J, V7_K, V7_L, V7_M, V7_N, V7_O,
911 V8, V8_H, V8_J, V8_K, V8_L, V8_M, V8_N, V8_O,
912 V9, V9_H, V9_J, V9_K, V9_L, V9_M, V9_N, V9_O,
913 V10, V10_H, V10_J, V10_K, V10_L, V10_M, V10_N, V10_O,
914 V11, V11_H, V11_J, V11_K, V11_L, V11_M, V11_N, V11_O,
915 V12, V12_H, V12_J, V12_K, V12_L, V12_M, V12_N, V12_O,
916 V13, V13_H, V13_J, V13_K, V13_L, V13_M, V13_N, V13_O,
917 V14, V14_H, V14_J, V14_K, V14_L, V14_M, V14_N, V14_O,
918 V15, V15_H, V15_J, V15_K, V15_L, V15_M, V15_N, V15_O,
919 V16, V16_H, V16_J, V16_K, V16_L, V16_M, V16_N, V16_O,
920 V17, V17_H, V17_J, V17_K, V17_L, V17_M, V17_N, V17_O,
921 V18, V18_H, V18_J, V18_K, V18_L, V18_M, V18_N, V18_O,
922 V19, V19_H, V19_J, V19_K, V19_L, V19_M, V19_N, V19_O,
923 V20, V20_H, V20_J, V20_K, V20_L, V20_M, V20_N, V20_O,
924 V21, V21_H, V21_J, V21_K, V21_L, V21_M, V21_N, V21_O,
925 V22, V22_H, V22_J, V22_K, V22_L, V22_M, V22_N, V22_O,
926 V23, V23_H, V23_J, V23_K, V23_L, V23_M, V23_N, V23_O,
927 V24, V24_H, V24_J, V24_K, V24_L, V24_M, V24_N, V24_O,
928 V25, V25_H, V25_J, V25_K, V25_L, V25_M, V25_N, V25_O,
929 V26, V26_H, V26_J, V26_K, V26_L, V26_M, V26_N, V26_O,
930 V27, V27_H, V27_J, V27_K, V27_L, V27_M, V27_N, V27_O,
931 V28, V28_H, V28_J, V28_K, V28_L, V28_M, V28_N, V28_O,
932 V29, V29_H, V29_J, V29_K, V29_L, V29_M, V29_N, V29_O,
933 V30, V30_H, V30_J, V30_K, V30_L, V30_M, V30_N, V30_O,
934 V31, V31_H, V31_J, V31_K, V31_L, V31_M, V31_N, V31_O,
935 );
936
937 // Class for all 64bit vector registers
938 reg_class vectord_reg(
939 V0, V0_H,
940 V1, V1_H,
941 V2, V2_H,
942 V3, V3_H,
943 V4, V4_H,
944 V5, V5_H,
945 V6, V6_H,
946 V7, V7_H,
947 V8, V8_H,
948 V9, V9_H,
949 V10, V10_H,
950 V11, V11_H,
951 V12, V12_H,
952 V13, V13_H,
953 V14, V14_H,
954 V15, V15_H,
955 V16, V16_H,
956 V17, V17_H,
957 V18, V18_H,
958 V19, V19_H,
959 V20, V20_H,
960 V21, V21_H,
961 V22, V22_H,
962 V23, V23_H,
963 V24, V24_H,
964 V25, V25_H,
965 V26, V26_H,
966 V27, V27_H,
967 V28, V28_H,
968 V29, V29_H,
969 V30, V30_H,
970 V31, V31_H
971 );
972
973 // Class for all 128bit vector registers
974 reg_class vectorx_reg(
975 V0, V0_H, V0_J, V0_K,
976 V1, V1_H, V1_J, V1_K,
977 V2, V2_H, V2_J, V2_K,
978 V3, V3_H, V3_J, V3_K,
979 V4, V4_H, V4_J, V4_K,
980 V5, V5_H, V5_J, V5_K,
981 V6, V6_H, V6_J, V6_K,
982 V7, V7_H, V7_J, V7_K,
983 V8, V8_H, V8_J, V8_K,
984 V9, V9_H, V9_J, V9_K,
985 V10, V10_H, V10_J, V10_K,
986 V11, V11_H, V11_J, V11_K,
987 V12, V12_H, V12_J, V12_K,
988 V13, V13_H, V13_J, V13_K,
989 V14, V14_H, V14_J, V14_K,
990 V15, V15_H, V15_J, V15_K,
991 V16, V16_H, V16_J, V16_K,
992 V17, V17_H, V17_J, V17_K,
993 V18, V18_H, V18_J, V18_K,
994 V19, V19_H, V19_J, V19_K,
995 V20, V20_H, V20_J, V20_K,
996 V21, V21_H, V21_J, V21_K,
997 V22, V22_H, V22_J, V22_K,
998 V23, V23_H, V23_J, V23_K,
999 V24, V24_H, V24_J, V24_K,
1000 V25, V25_H, V25_J, V25_K,
1001 V26, V26_H, V26_J, V26_K,
1002 V27, V27_H, V27_J, V27_K,
1003 V28, V28_H, V28_J, V28_K,
1004 V29, V29_H, V29_J, V29_K,
1005 V30, V30_H, V30_J, V30_K,
1006 V31, V31_H, V31_J, V31_K
1007 );
1008
1009 // Class for 128 bit register v0
1010 reg_class v0_reg(
1011 V0, V0_H
1012 );
1013
1014 // Class for 128 bit register v1
1015 reg_class v1_reg(
1016 V1, V1_H
1017 );
1018
1019 // Class for 128 bit register v2
1020 reg_class v2_reg(
1021 V2, V2_H
1022 );
1023
1024 // Class for 128 bit register v3
1025 reg_class v3_reg(
1026 V3, V3_H
1027 );
1028
1029 // Class for 128 bit register v4
1030 reg_class v4_reg(
1031 V4, V4_H
1032 );
1033
1034 // Class for 128 bit register v5
1035 reg_class v5_reg(
1036 V5, V5_H
1037 );
1038
1039 // Class for 128 bit register v6
1040 reg_class v6_reg(
1041 V6, V6_H
1042 );
1043
1044 // Class for 128 bit register v7
1045 reg_class v7_reg(
1046 V7, V7_H
1047 );
1048
1049 // Class for 128 bit register v8
1050 reg_class v8_reg(
1051 V8, V8_H
1052 );
1053
1054 // Class for 128 bit register v9
1055 reg_class v9_reg(
1056 V9, V9_H
1057 );
1058
1059 // Class for 128 bit register v10
1060 reg_class v10_reg(
1061 V10, V10_H
1062 );
1063
1064 // Class for 128 bit register v11
1065 reg_class v11_reg(
1066 V11, V11_H
1067 );
1068
1069 // Class for 128 bit register v12
1070 reg_class v12_reg(
1071 V12, V12_H
1072 );
1073
1074 // Class for 128 bit register v13
1075 reg_class v13_reg(
1076 V13, V13_H
1077 );
1078
1079 // Class for 128 bit register v14
1080 reg_class v14_reg(
1081 V14, V14_H
1082 );
1083
1084 // Class for 128 bit register v15
1085 reg_class v15_reg(
1086 V15, V15_H
1087 );
1088
1089 // Class for 128 bit register v16
1090 reg_class v16_reg(
1091 V16, V16_H
1092 );
1093
1094 // Class for 128 bit register v17
1095 reg_class v17_reg(
1096 V17, V17_H
1097 );
1098
1099 // Class for 128 bit register v18
1100 reg_class v18_reg(
1101 V18, V18_H
1102 );
1103
1104 // Class for 128 bit register v19
1105 reg_class v19_reg(
1106 V19, V19_H
1107 );
1108
1109 // Class for 128 bit register v20
1110 reg_class v20_reg(
1111 V20, V20_H
1112 );
1113
1114 // Class for 128 bit register v21
1115 reg_class v21_reg(
1116 V21, V21_H
1117 );
1118
1119 // Class for 128 bit register v22
1120 reg_class v22_reg(
1121 V22, V22_H
1122 );
1123
1124 // Class for 128 bit register v23
1125 reg_class v23_reg(
1126 V23, V23_H
1127 );
1128
1129 // Class for 128 bit register v24
1130 reg_class v24_reg(
1131 V24, V24_H
1132 );
1133
1134 // Class for 128 bit register v25
1135 reg_class v25_reg(
1136 V25, V25_H
1137 );
1138
1139 // Class for 128 bit register v26
1140 reg_class v26_reg(
1141 V26, V26_H
1142 );
1143
1144 // Class for 128 bit register v27
1145 reg_class v27_reg(
1146 V27, V27_H
1147 );
1148
1149 // Class for 128 bit register v28
1150 reg_class v28_reg(
1151 V28, V28_H
1152 );
1153
1154 // Class for 128 bit register v29
1155 reg_class v29_reg(
1156 V29, V29_H
1157 );
1158
1159 // Class for 128 bit register v30
1160 reg_class v30_reg(
1161 V30, V30_H
1162 );
1163
1164 // Class for 128 bit register v31
1165 reg_class v31_reg(
1166 V31, V31_H
1167 );
1168
1169 // Class for all SVE predicate registers.
1170 reg_class pr_reg (
1171 P0,
1172 P1,
1173 P2,
1174 P3,
1175 P4,
1176 P5,
1177 P6,
1178 // P7, non-allocatable, preserved with all elements preset to TRUE.
1179 P8,
1180 P9,
1181 P10,
1182 P11,
1183 P12,
1184 P13,
1185 P14,
1186 P15
1187 );
1188
1189 // Class for SVE governing predicate registers, which are used
1190 // to determine the active elements of a predicated instruction.
1191 reg_class gov_pr (
1192 P0,
1193 P1,
1194 P2,
1195 P3,
1196 P4,
1197 P5,
1198 P6,
1199 // P7, non-allocatable, preserved with all elements preset to TRUE.
1200 );
1201
1202 // Singleton class for condition codes
1203 reg_class int_flags(RFLAGS);
1204
1205 %}
1206
1207 //----------DEFINITION BLOCK---------------------------------------------------
1208 // Define name --> value mappings to inform the ADLC of an integer valued name
1209 // Current support includes integer values in the range [0, 0x7FFFFFFF]
1210 // Format:
1211 // int_def <name> ( <int_value>, <expression>);
1212 // Generated Code in ad_<arch>.hpp
1213 // #define <name> (<expression>)
1214 // // value == <int_value>
1215 // Generated code in ad_<arch>.cpp adlc_verification()
1216 // assert( <name> == <int_value>, "Expect (<expression>) to equal <int_value>");
1217 //
1218
1219 // we follow the ppc-aix port in using a simple cost model which ranks
1220 // register operations as cheap, memory ops as more expensive and
1221 // branches as most expensive. the first two have a low as well as a
1222 // normal cost. huge cost appears to be a way of saying don't do
1223 // something
1224
1225 definitions %{
1226 // The default cost (of a register move instruction).
1227 int_def INSN_COST ( 100, 100);
1228 int_def BRANCH_COST ( 200, 2 * INSN_COST);
1229 int_def CALL_COST ( 200, 2 * INSN_COST);
1230 int_def VOLATILE_REF_COST ( 1000, 10 * INSN_COST);
1231 %}
1232
1233
1234 //----------SOURCE BLOCK-------------------------------------------------------
1235 // This is a block of C++ code which provides values, functions, and
1236 // definitions necessary in the rest of the architecture description
1237
1238 source_hpp %{
1239
1240 #include "asm/macroAssembler.hpp"
1241 #include "gc/shared/barrierSetAssembler.hpp"
1242 #include "gc/shared/cardTable.hpp"
1243 #include "gc/shared/cardTableBarrierSet.hpp"
1244 #include "gc/shared/collectedHeap.hpp"
1245 #include "opto/addnode.hpp"
1246 #include "opto/convertnode.hpp"
1247 #include "runtime/objectMonitor.hpp"
1248
1249 extern RegMask _ANY_REG32_mask;
1250 extern RegMask _ANY_REG_mask;
1251 extern RegMask _PTR_REG_mask;
1252 extern RegMask _NO_SPECIAL_REG32_mask;
1253 extern RegMask _NO_SPECIAL_REG_mask;
1254 extern RegMask _NO_SPECIAL_PTR_REG_mask;
1255
1256 class CallStubImpl {
1257
1258 //--------------------------------------------------------------
1259 //---< Used for optimization in Compile::shorten_branches >---
1260 //--------------------------------------------------------------
1261
1262 public:
1263 // Size of call trampoline stub.
1264 static uint size_call_trampoline() {
1265 return 0; // no call trampolines on this platform
1266 }
1267
1268 // number of relocations needed by a call trampoline stub
1269 static uint reloc_call_trampoline() {
1270 return 0; // no call trampolines on this platform
1271 }
1272 };
1273
1274 class HandlerImpl {
1275
1276 public:
1277
1278 static int emit_exception_handler(CodeBuffer &cbuf);
1279 static int emit_deopt_handler(CodeBuffer& cbuf);
1280
1281 static uint size_exception_handler() {
1282 return MacroAssembler::far_codestub_branch_size();
1283 }
1284
1285 static uint size_deopt_handler() {
1286 // count one adr and one far branch instruction
1287 return NativeInstruction::instruction_size + MacroAssembler::far_codestub_branch_size();
1288 }
1289 };
1290
1291 class Node::PD {
1292 public:
1293 enum NodeFlags {
1294 _last_flag = Node::_last_flag
1295 };
1296 };
1297
1298 bool is_CAS(int opcode, bool maybe_volatile);
1299
1300 // predicates controlling emit of ldr<x>/ldar<x> and associated dmb
1301
1302 bool unnecessary_acquire(const Node *barrier);
1303 bool needs_acquiring_load(const Node *load);
1304
1305 // predicates controlling emit of str<x>/stlr<x> and associated dmbs
1306
1307 bool unnecessary_release(const Node *barrier);
1308 bool unnecessary_volatile(const Node *barrier);
1309 bool needs_releasing_store(const Node *store);
1310
1311 // predicate controlling translation of CompareAndSwapX
1312 bool needs_acquiring_load_exclusive(const Node *load);
1313
1314 // Assert that the given node is not a variable shift.
1315 bool assert_not_var_shift(const Node* n);
1316
1317 // predicate controlling addressing modes
1318 bool size_fits_all_mem_uses(AddPNode* addp, int shift);
1319 %}
1320
1321 source %{
1322
1323 // Derived RegMask with conditionally allocatable registers
1324
1325 void PhaseOutput::pd_perform_mach_node_analysis() {
1326 }
1327
1328 int MachNode::pd_alignment_required() const {
1329 return 1;
1330 }
1331
1332 int MachNode::compute_padding(int current_offset) const {
1333 return 0;
1334 }
1335
1336 RegMask _ANY_REG32_mask;
1337 RegMask _ANY_REG_mask;
1338 RegMask _PTR_REG_mask;
1339 RegMask _NO_SPECIAL_REG32_mask;
1340 RegMask _NO_SPECIAL_REG_mask;
1341 RegMask _NO_SPECIAL_PTR_REG_mask;
1342
1343 void reg_mask_init() {
1344 // We derive below RegMask(s) from the ones which are auto-generated from
1345 // adlc register classes to make AArch64 rheapbase (r27) and rfp (r29)
1346 // registers conditionally reserved.
1347
1348 _ANY_REG32_mask = _ALL_REG32_mask;
1349 _ANY_REG32_mask.Remove(OptoReg::as_OptoReg(r31_sp->as_VMReg()));
1350
1351 _ANY_REG_mask = _ALL_REG_mask;
1352
1353 _PTR_REG_mask = _ALL_REG_mask;
1354
1355 _NO_SPECIAL_REG32_mask = _ALL_REG32_mask;
1356 _NO_SPECIAL_REG32_mask.SUBTRACT(_NON_ALLOCATABLE_REG32_mask);
1357
1358 _NO_SPECIAL_REG_mask = _ALL_REG_mask;
1359 _NO_SPECIAL_REG_mask.SUBTRACT(_NON_ALLOCATABLE_REG_mask);
1360
1361 _NO_SPECIAL_PTR_REG_mask = _ALL_REG_mask;
1362 _NO_SPECIAL_PTR_REG_mask.SUBTRACT(_NON_ALLOCATABLE_REG_mask);
1363
1364 // r27 is not allocatable when compressed oops is on and heapbase is not
1365 // zero, compressed klass pointers doesn't use r27 after JDK-8234794
1366 if (UseCompressedOops && (CompressedOops::ptrs_base() != NULL)) {
1367 _NO_SPECIAL_REG32_mask.Remove(OptoReg::as_OptoReg(r27->as_VMReg()));
1368 _NO_SPECIAL_REG_mask.SUBTRACT(_HEAPBASE_REG_mask);
1369 _NO_SPECIAL_PTR_REG_mask.SUBTRACT(_HEAPBASE_REG_mask);
1370 }
1371
1372 // r29 is not allocatable when PreserveFramePointer is on
1373 if (PreserveFramePointer) {
1374 _NO_SPECIAL_REG32_mask.Remove(OptoReg::as_OptoReg(r29->as_VMReg()));
1375 _NO_SPECIAL_REG_mask.SUBTRACT(_FP_REG_mask);
1376 _NO_SPECIAL_PTR_REG_mask.SUBTRACT(_FP_REG_mask);
1377 }
1378 }
1379
1380 // Optimizaton of volatile gets and puts
1381 // -------------------------------------
1382 //
1383 // AArch64 has ldar<x> and stlr<x> instructions which we can safely
1384 // use to implement volatile reads and writes. For a volatile read
1385 // we simply need
1386 //
1387 // ldar<x>
1388 //
1389 // and for a volatile write we need
1390 //
1391 // stlr<x>
1392 //
1393 // Alternatively, we can implement them by pairing a normal
1394 // load/store with a memory barrier. For a volatile read we need
1395 //
1396 // ldr<x>
1397 // dmb ishld
1398 //
1399 // for a volatile write
1400 //
1401 // dmb ish
1402 // str<x>
1403 // dmb ish
1404 //
1405 // We can also use ldaxr and stlxr to implement compare and swap CAS
1406 // sequences. These are normally translated to an instruction
1407 // sequence like the following
1408 //
1409 // dmb ish
1410 // retry:
1411 // ldxr<x> rval raddr
1412 // cmp rval rold
1413 // b.ne done
1414 // stlxr<x> rval, rnew, rold
1415 // cbnz rval retry
1416 // done:
1417 // cset r0, eq
1418 // dmb ishld
1419 //
1420 // Note that the exclusive store is already using an stlxr
1421 // instruction. That is required to ensure visibility to other
1422 // threads of the exclusive write (assuming it succeeds) before that
1423 // of any subsequent writes.
1424 //
1425 // The following instruction sequence is an improvement on the above
1426 //
1427 // retry:
1428 // ldaxr<x> rval raddr
1429 // cmp rval rold
1430 // b.ne done
1431 // stlxr<x> rval, rnew, rold
1432 // cbnz rval retry
1433 // done:
1434 // cset r0, eq
1435 //
1436 // We don't need the leading dmb ish since the stlxr guarantees
1437 // visibility of prior writes in the case that the swap is
1438 // successful. Crucially we don't have to worry about the case where
1439 // the swap is not successful since no valid program should be
1440 // relying on visibility of prior changes by the attempting thread
1441 // in the case where the CAS fails.
1442 //
1443 // Similarly, we don't need the trailing dmb ishld if we substitute
1444 // an ldaxr instruction since that will provide all the guarantees we
1445 // require regarding observation of changes made by other threads
1446 // before any change to the CAS address observed by the load.
1447 //
1448 // In order to generate the desired instruction sequence we need to
1449 // be able to identify specific 'signature' ideal graph node
1450 // sequences which i) occur as a translation of a volatile reads or
1451 // writes or CAS operations and ii) do not occur through any other
1452 // translation or graph transformation. We can then provide
1453 // alternative aldc matching rules which translate these node
1454 // sequences to the desired machine code sequences. Selection of the
1455 // alternative rules can be implemented by predicates which identify
1456 // the relevant node sequences.
1457 //
1458 // The ideal graph generator translates a volatile read to the node
1459 // sequence
1460 //
1461 // LoadX[mo_acquire]
1462 // MemBarAcquire
1463 //
1464 // As a special case when using the compressed oops optimization we
1465 // may also see this variant
1466 //
1467 // LoadN[mo_acquire]
1468 // DecodeN
1469 // MemBarAcquire
1470 //
1471 // A volatile write is translated to the node sequence
1472 //
1473 // MemBarRelease
1474 // StoreX[mo_release] {CardMark}-optional
1475 // MemBarVolatile
1476 //
1477 // n.b. the above node patterns are generated with a strict
1478 // 'signature' configuration of input and output dependencies (see
1479 // the predicates below for exact details). The card mark may be as
1480 // simple as a few extra nodes or, in a few GC configurations, may
1481 // include more complex control flow between the leading and
1482 // trailing memory barriers. However, whatever the card mark
1483 // configuration these signatures are unique to translated volatile
1484 // reads/stores -- they will not appear as a result of any other
1485 // bytecode translation or inlining nor as a consequence of
1486 // optimizing transforms.
1487 //
1488 // We also want to catch inlined unsafe volatile gets and puts and
1489 // be able to implement them using either ldar<x>/stlr<x> or some
1490 // combination of ldr<x>/stlr<x> and dmb instructions.
1491 //
1492 // Inlined unsafe volatiles puts manifest as a minor variant of the
1493 // normal volatile put node sequence containing an extra cpuorder
1494 // membar
1495 //
1496 // MemBarRelease
1497 // MemBarCPUOrder
1498 // StoreX[mo_release] {CardMark}-optional
1499 // MemBarCPUOrder
1500 // MemBarVolatile
1501 //
1502 // n.b. as an aside, a cpuorder membar is not itself subject to
1503 // matching and translation by adlc rules. However, the rule
1504 // predicates need to detect its presence in order to correctly
1505 // select the desired adlc rules.
1506 //
1507 // Inlined unsafe volatile gets manifest as a slightly different
1508 // node sequence to a normal volatile get because of the
1509 // introduction of some CPUOrder memory barriers to bracket the
1510 // Load. However, but the same basic skeleton of a LoadX feeding a
1511 // MemBarAcquire, possibly through an optional DecodeN, is still
1512 // present
1513 //
1514 // MemBarCPUOrder
1515 // || \\
1516 // MemBarCPUOrder LoadX[mo_acquire]
1517 // || |
1518 // || {DecodeN} optional
1519 // || /
1520 // MemBarAcquire
1521 //
1522 // In this case the acquire membar does not directly depend on the
1523 // load. However, we can be sure that the load is generated from an
1524 // inlined unsafe volatile get if we see it dependent on this unique
1525 // sequence of membar nodes. Similarly, given an acquire membar we
1526 // can know that it was added because of an inlined unsafe volatile
1527 // get if it is fed and feeds a cpuorder membar and if its feed
1528 // membar also feeds an acquiring load.
1529 //
1530 // Finally an inlined (Unsafe) CAS operation is translated to the
1531 // following ideal graph
1532 //
1533 // MemBarRelease
1534 // MemBarCPUOrder
1535 // CompareAndSwapX {CardMark}-optional
1536 // MemBarCPUOrder
1537 // MemBarAcquire
1538 //
1539 // So, where we can identify these volatile read and write
1540 // signatures we can choose to plant either of the above two code
1541 // sequences. For a volatile read we can simply plant a normal
1542 // ldr<x> and translate the MemBarAcquire to a dmb. However, we can
1543 // also choose to inhibit translation of the MemBarAcquire and
1544 // inhibit planting of the ldr<x>, instead planting an ldar<x>.
1545 //
1546 // When we recognise a volatile store signature we can choose to
1547 // plant at a dmb ish as a translation for the MemBarRelease, a
1548 // normal str<x> and then a dmb ish for the MemBarVolatile.
1549 // Alternatively, we can inhibit translation of the MemBarRelease
1550 // and MemBarVolatile and instead plant a simple stlr<x>
1551 // instruction.
1552 //
1553 // when we recognise a CAS signature we can choose to plant a dmb
1554 // ish as a translation for the MemBarRelease, the conventional
1555 // macro-instruction sequence for the CompareAndSwap node (which
1556 // uses ldxr<x>) and then a dmb ishld for the MemBarAcquire.
1557 // Alternatively, we can elide generation of the dmb instructions
1558 // and plant the alternative CompareAndSwap macro-instruction
1559 // sequence (which uses ldaxr<x>).
1560 //
1561 // Of course, the above only applies when we see these signature
1562 // configurations. We still want to plant dmb instructions in any
1563 // other cases where we may see a MemBarAcquire, MemBarRelease or
1564 // MemBarVolatile. For example, at the end of a constructor which
1565 // writes final/volatile fields we will see a MemBarRelease
1566 // instruction and this needs a 'dmb ish' lest we risk the
1567 // constructed object being visible without making the
1568 // final/volatile field writes visible.
1569 //
1570 // n.b. the translation rules below which rely on detection of the
1571 // volatile signatures and insert ldar<x> or stlr<x> are failsafe.
1572 // If we see anything other than the signature configurations we
1573 // always just translate the loads and stores to ldr<x> and str<x>
1574 // and translate acquire, release and volatile membars to the
1575 // relevant dmb instructions.
1576 //
1577
1578 // is_CAS(int opcode, bool maybe_volatile)
1579 //
1580 // return true if opcode is one of the possible CompareAndSwapX
1581 // values otherwise false.
1582
1583 bool is_CAS(int opcode, bool maybe_volatile)
1584 {
1585 switch(opcode) {
1586 // We handle these
1587 case Op_CompareAndSwapI:
1588 case Op_CompareAndSwapL:
1589 case Op_CompareAndSwapP:
1590 case Op_CompareAndSwapN:
1591 case Op_ShenandoahCompareAndSwapP:
1592 case Op_ShenandoahCompareAndSwapN:
1593 case Op_CompareAndSwapB:
1594 case Op_CompareAndSwapS:
1595 case Op_GetAndSetI:
1596 case Op_GetAndSetL:
1597 case Op_GetAndSetP:
1598 case Op_GetAndSetN:
1599 case Op_GetAndAddI:
1600 case Op_GetAndAddL:
1601 return true;
1602 case Op_CompareAndExchangeI:
1603 case Op_CompareAndExchangeN:
1604 case Op_CompareAndExchangeB:
1605 case Op_CompareAndExchangeS:
1606 case Op_CompareAndExchangeL:
1607 case Op_CompareAndExchangeP:
1608 case Op_WeakCompareAndSwapB:
1609 case Op_WeakCompareAndSwapS:
1610 case Op_WeakCompareAndSwapI:
1611 case Op_WeakCompareAndSwapL:
1612 case Op_WeakCompareAndSwapP:
1613 case Op_WeakCompareAndSwapN:
1614 case Op_ShenandoahWeakCompareAndSwapP:
1615 case Op_ShenandoahWeakCompareAndSwapN:
1616 case Op_ShenandoahCompareAndExchangeP:
1617 case Op_ShenandoahCompareAndExchangeN:
1618 return maybe_volatile;
1619 default:
1620 return false;
1621 }
1622 }
1623
1624 // helper to determine the maximum number of Phi nodes we may need to
1625 // traverse when searching from a card mark membar for the merge mem
1626 // feeding a trailing membar or vice versa
1627
1628 // predicates controlling emit of ldr<x>/ldar<x>
1629
1630 bool unnecessary_acquire(const Node *barrier)
1631 {
1632 assert(barrier->is_MemBar(), "expecting a membar");
1633
1634 MemBarNode* mb = barrier->as_MemBar();
1635
1636 if (mb->trailing_load()) {
1637 return true;
1638 }
1639
1640 if (mb->trailing_load_store()) {
1641 Node* load_store = mb->in(MemBarNode::Precedent);
1642 assert(load_store->is_LoadStore(), "unexpected graph shape");
1643 return is_CAS(load_store->Opcode(), true);
1644 }
1645
1646 return false;
1647 }
1648
1649 bool needs_acquiring_load(const Node *n)
1650 {
1651 assert(n->is_Load(), "expecting a load");
1652 LoadNode *ld = n->as_Load();
1653 return ld->is_acquire();
1654 }
1655
1656 bool unnecessary_release(const Node *n)
1657 {
1658 assert((n->is_MemBar() &&
1659 n->Opcode() == Op_MemBarRelease),
1660 "expecting a release membar");
1661
1662 MemBarNode *barrier = n->as_MemBar();
1663 if (!barrier->leading()) {
1664 return false;
1665 } else {
1666 Node* trailing = barrier->trailing_membar();
1667 MemBarNode* trailing_mb = trailing->as_MemBar();
1668 assert(trailing_mb->trailing(), "Not a trailing membar?");
1669 assert(trailing_mb->leading_membar() == n, "inconsistent leading/trailing membars");
1670
1671 Node* mem = trailing_mb->in(MemBarNode::Precedent);
1672 if (mem->is_Store()) {
1673 assert(mem->as_Store()->is_release(), "");
1674 assert(trailing_mb->Opcode() == Op_MemBarVolatile, "");
1675 return true;
1676 } else {
1677 assert(mem->is_LoadStore(), "");
1678 assert(trailing_mb->Opcode() == Op_MemBarAcquire, "");
1679 return is_CAS(mem->Opcode(), true);
1680 }
1681 }
1682 return false;
1683 }
1684
1685 bool unnecessary_volatile(const Node *n)
1686 {
1687 // assert n->is_MemBar();
1688 MemBarNode *mbvol = n->as_MemBar();
1689
1690 bool release = mbvol->trailing_store();
1691 assert(!release || (mbvol->in(MemBarNode::Precedent)->is_Store() && mbvol->in(MemBarNode::Precedent)->as_Store()->is_release()), "");
1692 #ifdef ASSERT
1693 if (release) {
1694 Node* leading = mbvol->leading_membar();
1695 assert(leading->Opcode() == Op_MemBarRelease, "");
1696 assert(leading->as_MemBar()->leading_store(), "");
1697 assert(leading->as_MemBar()->trailing_membar() == mbvol, "");
1698 }
1699 #endif
1700
1701 return release;
1702 }
1703
1704 // predicates controlling emit of str<x>/stlr<x>
1705
1706 bool needs_releasing_store(const Node *n)
1707 {
1708 // assert n->is_Store();
1709 StoreNode *st = n->as_Store();
1710 return st->trailing_membar() != NULL;
1711 }
1712
1713 // predicate controlling translation of CAS
1714 //
1715 // returns true if CAS needs to use an acquiring load otherwise false
1716
1717 bool needs_acquiring_load_exclusive(const Node *n)
1718 {
1719 assert(is_CAS(n->Opcode(), true), "expecting a compare and swap");
1720 LoadStoreNode* ldst = n->as_LoadStore();
1721 if (is_CAS(n->Opcode(), false)) {
1722 assert(ldst->trailing_membar() != NULL, "expected trailing membar");
1723 } else {
1724 return ldst->trailing_membar() != NULL;
1725 }
1726
1727 // so we can just return true here
1728 return true;
1729 }
1730
1731 // Assert that the given node is not a variable shift.
1732 bool assert_not_var_shift(const Node* n) {
1733 assert(!n->as_ShiftV()->is_var_shift(), "illegal variable shift");
1734 return true;
1735 }
1736
1737 #define __ _masm.
1738
1739 // advance declarations for helper functions to convert register
1740 // indices to register objects
1741
1742 // the ad file has to provide implementations of certain methods
1743 // expected by the generic code
1744 //
1745 // REQUIRED FUNCTIONALITY
1746
1747 //=============================================================================
1748
1749 // !!!!! Special hack to get all types of calls to specify the byte offset
1750 // from the start of the call to the point where the return address
1751 // will point.
1752
1753 int MachCallStaticJavaNode::ret_addr_offset()
1754 {
1755 // call should be a simple bl
1756 int off = 4;
1757 return off;
1758 }
1759
1760 int MachCallDynamicJavaNode::ret_addr_offset()
1761 {
1762 return 16; // movz, movk, movk, bl
1763 }
1764
1765 int MachCallRuntimeNode::ret_addr_offset() {
1766 // for generated stubs the call will be
1767 // bl(addr)
1768 // or with far branches
1769 // bl(trampoline_stub)
1770 // for real runtime callouts it will be six instructions
1771 // see aarch64_enc_java_to_runtime
1772 // adr(rscratch2, retaddr)
1773 // lea(rscratch1, RuntimeAddress(addr)
1774 // stp(zr, rscratch2, Address(__ pre(sp, -2 * wordSize)))
1775 // blr(rscratch1)
1776 CodeBlob *cb = CodeCache::find_blob(_entry_point);
1777 if (cb) {
1778 return 1 * NativeInstruction::instruction_size;
1779 } else {
1780 return 6 * NativeInstruction::instruction_size;
1781 }
1782 }
1783
1784 int MachCallNativeNode::ret_addr_offset() {
1785 // This is implemented using aarch64_enc_java_to_runtime as above.
1786 CodeBlob *cb = CodeCache::find_blob(_entry_point);
1787 if (cb) {
1788 return 1 * NativeInstruction::instruction_size;
1789 } else {
1790 return 6 * NativeInstruction::instruction_size;
1791 }
1792 }
1793
1794 //=============================================================================
1795
1796 #ifndef PRODUCT
1797 void MachBreakpointNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
1798 st->print("BREAKPOINT");
1799 }
1800 #endif
1801
1802 void MachBreakpointNode::emit(CodeBuffer &cbuf, PhaseRegAlloc *ra_) const {
1803 C2_MacroAssembler _masm(&cbuf);
1804 __ brk(0);
1805 }
1806
1807 uint MachBreakpointNode::size(PhaseRegAlloc *ra_) const {
1808 return MachNode::size(ra_);
1809 }
1810
1811 //=============================================================================
1812
1813 #ifndef PRODUCT
1814 void MachNopNode::format(PhaseRegAlloc*, outputStream* st) const {
1815 st->print("nop \t# %d bytes pad for loops and calls", _count);
1816 }
1817 #endif
1818
1819 void MachNopNode::emit(CodeBuffer &cbuf, PhaseRegAlloc*) const {
1820 C2_MacroAssembler _masm(&cbuf);
1821 for (int i = 0; i < _count; i++) {
1822 __ nop();
1823 }
1824 }
1825
1826 uint MachNopNode::size(PhaseRegAlloc*) const {
1827 return _count * NativeInstruction::instruction_size;
1828 }
1829
1830 //=============================================================================
1831 const RegMask& MachConstantBaseNode::_out_RegMask = RegMask::Empty;
1832
1833 int ConstantTable::calculate_table_base_offset() const {
1834 return 0; // absolute addressing, no offset
1835 }
1836
1837 bool MachConstantBaseNode::requires_postalloc_expand() const { return false; }
1838 void MachConstantBaseNode::postalloc_expand(GrowableArray <Node *> *nodes, PhaseRegAlloc *ra_) {
1839 ShouldNotReachHere();
1840 }
1841
1842 void MachConstantBaseNode::emit(CodeBuffer& cbuf, PhaseRegAlloc* ra_) const {
1843 // Empty encoding
1844 }
1845
1846 uint MachConstantBaseNode::size(PhaseRegAlloc* ra_) const {
1847 return 0;
1848 }
1849
1850 #ifndef PRODUCT
1851 void MachConstantBaseNode::format(PhaseRegAlloc* ra_, outputStream* st) const {
1852 st->print("-- \t// MachConstantBaseNode (empty encoding)");
1853 }
1854 #endif
1855
1856 #ifndef PRODUCT
1857 void MachPrologNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
1858 Compile* C = ra_->C;
1859
1860 int framesize = C->output()->frame_slots() << LogBytesPerInt;
1861
1862 if (C->output()->need_stack_bang(framesize))
1863 st->print("# stack bang size=%d\n\t", framesize);
1864
1865 if (VM_Version::use_rop_protection()) {
1866 st->print("ldr zr, [lr]\n\t");
1867 st->print("pacia lr, rfp\n\t");
1868 }
1869 if (framesize < ((1 << 9) + 2 * wordSize)) {
1870 st->print("sub sp, sp, #%d\n\t", framesize);
1871 st->print("stp rfp, lr, [sp, #%d]", framesize - 2 * wordSize);
1872 if (PreserveFramePointer) st->print("\n\tadd rfp, sp, #%d", framesize - 2 * wordSize);
1873 } else {
1874 st->print("stp lr, rfp, [sp, #%d]!\n\t", -(2 * wordSize));
1875 if (PreserveFramePointer) st->print("mov rfp, sp\n\t");
1876 st->print("mov rscratch1, #%d\n\t", framesize - 2 * wordSize);
1877 st->print("sub sp, sp, rscratch1");
1878 }
1879 if (C->stub_function() == NULL && BarrierSet::barrier_set()->barrier_set_nmethod() != NULL) {
1880 st->print("\n\t");
1881 st->print("ldr rscratch1, [guard]\n\t");
1882 st->print("dmb ishld\n\t");
1883 st->print("ldr rscratch2, [rthread, #thread_disarmed_offset]\n\t");
1884 st->print("cmp rscratch1, rscratch2\n\t");
1885 st->print("b.eq skip");
1886 st->print("\n\t");
1887 st->print("blr #nmethod_entry_barrier_stub\n\t");
1888 st->print("b skip\n\t");
1889 st->print("guard: int\n\t");
1890 st->print("\n\t");
1891 st->print("skip:\n\t");
1892 }
1893 }
1894 #endif
1895
1896 void MachPrologNode::emit(CodeBuffer &cbuf, PhaseRegAlloc *ra_) const {
1897 Compile* C = ra_->C;
1898 C2_MacroAssembler _masm(&cbuf);
1899
1900 // n.b. frame size includes space for return pc and rfp
1901 const int framesize = C->output()->frame_size_in_bytes();
1902
1903 // insert a nop at the start of the prolog so we can patch in a
1904 // branch if we need to invalidate the method later
1905 __ nop();
1906
1907 if (C->clinit_barrier_on_entry()) {
1908 assert(!C->method()->holder()->is_not_initialized(), "initialization should have been started");
1909
1910 Label L_skip_barrier;
1911
1912 __ mov_metadata(rscratch2, C->method()->holder()->constant_encoding());
1913 __ clinit_barrier(rscratch2, rscratch1, &L_skip_barrier);
1914 __ far_jump(RuntimeAddress(SharedRuntime::get_handle_wrong_method_stub()));
1915 __ bind(L_skip_barrier);
1916 }
1917
1918 if (C->max_vector_size() > 0) {
1919 __ reinitialize_ptrue();
1920 }
1921
1922 int bangsize = C->output()->bang_size_in_bytes();
1923 if (C->output()->need_stack_bang(bangsize))
1924 __ generate_stack_overflow_check(bangsize);
1925
1926 __ build_frame(framesize);
1927
1928 if (C->stub_function() == NULL) {
1929 BarrierSetAssembler* bs = BarrierSet::barrier_set()->barrier_set_assembler();
1930 bs->nmethod_entry_barrier(&_masm);
1931 }
1932
1933 if (VerifyStackAtCalls) {
1934 Unimplemented();
1935 }
1936
1937 C->output()->set_frame_complete(cbuf.insts_size());
1938
1939 if (C->has_mach_constant_base_node()) {
1940 // NOTE: We set the table base offset here because users might be
1941 // emitted before MachConstantBaseNode.
1942 ConstantTable& constant_table = C->output()->constant_table();
1943 constant_table.set_table_base_offset(constant_table.calculate_table_base_offset());
1944 }
1945 }
1946
1947 uint MachPrologNode::size(PhaseRegAlloc* ra_) const
1948 {
1949 return MachNode::size(ra_); // too many variables; just compute it
1950 // the hard way
1951 }
1952
1953 int MachPrologNode::reloc() const
1954 {
1955 return 0;
1956 }
1957
1958 //=============================================================================
1959
1960 #ifndef PRODUCT
1961 void MachEpilogNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
1962 Compile* C = ra_->C;
1963 int framesize = C->output()->frame_slots() << LogBytesPerInt;
1964
1965 st->print("# pop frame %d\n\t",framesize);
1966
1967 if (framesize == 0) {
1968 st->print("ldp lr, rfp, [sp],#%d\n\t", (2 * wordSize));
1969 } else if (framesize < ((1 << 9) + 2 * wordSize)) {
1970 st->print("ldp lr, rfp, [sp,#%d]\n\t", framesize - 2 * wordSize);
1971 st->print("add sp, sp, #%d\n\t", framesize);
1972 } else {
1973 st->print("mov rscratch1, #%d\n\t", framesize - 2 * wordSize);
1974 st->print("add sp, sp, rscratch1\n\t");
1975 st->print("ldp lr, rfp, [sp],#%d\n\t", (2 * wordSize));
1976 }
1977 if (VM_Version::use_rop_protection()) {
1978 st->print("autia lr, rfp\n\t");
1979 st->print("ldr zr, [lr]\n\t");
1980 }
1981
1982 if (do_polling() && C->is_method_compilation()) {
1983 st->print("# test polling word\n\t");
1984 st->print("ldr rscratch1, [rthread],#%d\n\t", in_bytes(JavaThread::polling_word_offset()));
1985 st->print("cmp sp, rscratch1\n\t");
1986 st->print("bhi #slow_path");
1987 }
1988 }
1989 #endif
1990
1991 void MachEpilogNode::emit(CodeBuffer &cbuf, PhaseRegAlloc *ra_) const {
1992 Compile* C = ra_->C;
1993 C2_MacroAssembler _masm(&cbuf);
1994 int framesize = C->output()->frame_slots() << LogBytesPerInt;
1995
1996 __ remove_frame(framesize);
1997
1998 if (StackReservedPages > 0 && C->has_reserved_stack_access()) {
1999 __ reserved_stack_check();
2000 }
2001
2002 if (do_polling() && C->is_method_compilation()) {
2003 Label dummy_label;
2004 Label* code_stub = &dummy_label;
2005 if (!C->output()->in_scratch_emit_size()) {
2006 code_stub = &C->output()->safepoint_poll_table()->add_safepoint(__ offset());
2007 }
2008 __ relocate(relocInfo::poll_return_type);
2009 __ safepoint_poll(*code_stub, true /* at_return */, false /* acquire */, true /* in_nmethod */);
2010 }
2011 }
2012
2013 uint MachEpilogNode::size(PhaseRegAlloc *ra_) const {
2014 // Variable size. Determine dynamically.
2015 return MachNode::size(ra_);
2016 }
2017
2018 int MachEpilogNode::reloc() const {
2019 // Return number of relocatable values contained in this instruction.
2020 return 1; // 1 for polling page.
2021 }
2022
2023 const Pipeline * MachEpilogNode::pipeline() const {
2024 return MachNode::pipeline_class();
2025 }
2026
2027 //=============================================================================
2028
2029 // Figure out which register class each belongs in: rc_int, rc_float or
2030 // rc_stack.
2031 enum RC { rc_bad, rc_int, rc_float, rc_predicate, rc_stack };
2032
2033 static enum RC rc_class(OptoReg::Name reg) {
2034
2035 if (reg == OptoReg::Bad) {
2036 return rc_bad;
2037 }
2038
2039 // we have 32 int registers * 2 halves
2040 int slots_of_int_registers = RegisterImpl::max_slots_per_register * RegisterImpl::number_of_registers;
2041
2042 if (reg < slots_of_int_registers) {
2043 return rc_int;
2044 }
2045
2046 // we have 32 float register * 8 halves
2047 int slots_of_float_registers = FloatRegisterImpl::max_slots_per_register * FloatRegisterImpl::number_of_registers;
2048 if (reg < slots_of_int_registers + slots_of_float_registers) {
2049 return rc_float;
2050 }
2051
2052 int slots_of_predicate_registers = PRegisterImpl::max_slots_per_register * PRegisterImpl::number_of_registers;
2053 if (reg < slots_of_int_registers + slots_of_float_registers + slots_of_predicate_registers) {
2054 return rc_predicate;
2055 }
2056
2057 // Between predicate regs & stack is the flags.
2058 assert(OptoReg::is_stack(reg), "blow up if spilling flags");
2059
2060 return rc_stack;
2061 }
2062
2063 uint MachSpillCopyNode::implementation(CodeBuffer *cbuf, PhaseRegAlloc *ra_, bool do_size, outputStream *st) const {
2064 Compile* C = ra_->C;
2065
2066 // Get registers to move.
2067 OptoReg::Name src_hi = ra_->get_reg_second(in(1));
2068 OptoReg::Name src_lo = ra_->get_reg_first(in(1));
2069 OptoReg::Name dst_hi = ra_->get_reg_second(this);
2070 OptoReg::Name dst_lo = ra_->get_reg_first(this);
2071
2072 enum RC src_hi_rc = rc_class(src_hi);
2073 enum RC src_lo_rc = rc_class(src_lo);
2074 enum RC dst_hi_rc = rc_class(dst_hi);
2075 enum RC dst_lo_rc = rc_class(dst_lo);
2076
2077 assert(src_lo != OptoReg::Bad && dst_lo != OptoReg::Bad, "must move at least 1 register");
2078
2079 if (src_hi != OptoReg::Bad && !bottom_type()->isa_vectmask()) {
2080 assert((src_lo&1)==0 && src_lo+1==src_hi &&
2081 (dst_lo&1)==0 && dst_lo+1==dst_hi,
2082 "expected aligned-adjacent pairs");
2083 }
2084
2085 if (src_lo == dst_lo && src_hi == dst_hi) {
2086 return 0; // Self copy, no move.
2087 }
2088
2089 bool is64 = (src_lo & 1) == 0 && src_lo + 1 == src_hi &&
2090 (dst_lo & 1) == 0 && dst_lo + 1 == dst_hi;
2091 int src_offset = ra_->reg2offset(src_lo);
2092 int dst_offset = ra_->reg2offset(dst_lo);
2093
2094 if (bottom_type()->isa_vect() && !bottom_type()->isa_vectmask()) {
2095 uint ireg = ideal_reg();
2096 if (ireg == Op_VecA && cbuf) {
2097 C2_MacroAssembler _masm(cbuf);
2098 int sve_vector_reg_size_in_bytes = Matcher::scalable_vector_reg_size(T_BYTE);
2099 if (src_lo_rc == rc_stack && dst_lo_rc == rc_stack) {
2100 // stack->stack
2101 __ spill_copy_sve_vector_stack_to_stack(src_offset, dst_offset,
2102 sve_vector_reg_size_in_bytes);
2103 } else if (src_lo_rc == rc_float && dst_lo_rc == rc_stack) {
2104 __ spill_sve_vector(as_FloatRegister(Matcher::_regEncode[src_lo]), ra_->reg2offset(dst_lo),
2105 sve_vector_reg_size_in_bytes);
2106 } else if (src_lo_rc == rc_stack && dst_lo_rc == rc_float) {
2107 __ unspill_sve_vector(as_FloatRegister(Matcher::_regEncode[dst_lo]), ra_->reg2offset(src_lo),
2108 sve_vector_reg_size_in_bytes);
2109 } else if (src_lo_rc == rc_float && dst_lo_rc == rc_float) {
2110 __ sve_orr(as_FloatRegister(Matcher::_regEncode[dst_lo]),
2111 as_FloatRegister(Matcher::_regEncode[src_lo]),
2112 as_FloatRegister(Matcher::_regEncode[src_lo]));
2113 } else {
2114 ShouldNotReachHere();
2115 }
2116 } else if (cbuf) {
2117 assert(ireg == Op_VecD || ireg == Op_VecX, "must be 64 bit or 128 bit vector");
2118 C2_MacroAssembler _masm(cbuf);
2119 assert((src_lo_rc != rc_int && dst_lo_rc != rc_int), "sanity");
2120 if (src_lo_rc == rc_stack && dst_lo_rc == rc_stack) {
2121 // stack->stack
2122 assert((src_offset & 7) == 0 && (dst_offset & 7) == 0, "unaligned stack offset");
2123 if (ireg == Op_VecD) {
2124 __ unspill(rscratch1, true, src_offset);
2125 __ spill(rscratch1, true, dst_offset);
2126 } else {
2127 __ spill_copy128(src_offset, dst_offset);
2128 }
2129 } else if (src_lo_rc == rc_float && dst_lo_rc == rc_float) {
2130 __ mov(as_FloatRegister(Matcher::_regEncode[dst_lo]),
2131 ireg == Op_VecD ? __ T8B : __ T16B,
2132 as_FloatRegister(Matcher::_regEncode[src_lo]));
2133 } else if (src_lo_rc == rc_float && dst_lo_rc == rc_stack) {
2134 __ spill(as_FloatRegister(Matcher::_regEncode[src_lo]),
2135 ireg == Op_VecD ? __ D : __ Q,
2136 ra_->reg2offset(dst_lo));
2137 } else if (src_lo_rc == rc_stack && dst_lo_rc == rc_float) {
2138 __ unspill(as_FloatRegister(Matcher::_regEncode[dst_lo]),
2139 ireg == Op_VecD ? __ D : __ Q,
2140 ra_->reg2offset(src_lo));
2141 } else {
2142 ShouldNotReachHere();
2143 }
2144 }
2145 } else if (cbuf) {
2146 C2_MacroAssembler _masm(cbuf);
2147 switch (src_lo_rc) {
2148 case rc_int:
2149 if (dst_lo_rc == rc_int) { // gpr --> gpr copy
2150 if (is64) {
2151 __ mov(as_Register(Matcher::_regEncode[dst_lo]),
2152 as_Register(Matcher::_regEncode[src_lo]));
2153 } else {
2154 C2_MacroAssembler _masm(cbuf);
2155 __ movw(as_Register(Matcher::_regEncode[dst_lo]),
2156 as_Register(Matcher::_regEncode[src_lo]));
2157 }
2158 } else if (dst_lo_rc == rc_float) { // gpr --> fpr copy
2159 if (is64) {
2160 __ fmovd(as_FloatRegister(Matcher::_regEncode[dst_lo]),
2161 as_Register(Matcher::_regEncode[src_lo]));
2162 } else {
2163 __ fmovs(as_FloatRegister(Matcher::_regEncode[dst_lo]),
2164 as_Register(Matcher::_regEncode[src_lo]));
2165 }
2166 } else { // gpr --> stack spill
2167 assert(dst_lo_rc == rc_stack, "spill to bad register class");
2168 __ spill(as_Register(Matcher::_regEncode[src_lo]), is64, dst_offset);
2169 }
2170 break;
2171 case rc_float:
2172 if (dst_lo_rc == rc_int) { // fpr --> gpr copy
2173 if (is64) {
2174 __ fmovd(as_Register(Matcher::_regEncode[dst_lo]),
2175 as_FloatRegister(Matcher::_regEncode[src_lo]));
2176 } else {
2177 __ fmovs(as_Register(Matcher::_regEncode[dst_lo]),
2178 as_FloatRegister(Matcher::_regEncode[src_lo]));
2179 }
2180 } else if (dst_lo_rc == rc_float) { // fpr --> fpr copy
2181 if (is64) {
2182 __ fmovd(as_FloatRegister(Matcher::_regEncode[dst_lo]),
2183 as_FloatRegister(Matcher::_regEncode[src_lo]));
2184 } else {
2185 __ fmovs(as_FloatRegister(Matcher::_regEncode[dst_lo]),
2186 as_FloatRegister(Matcher::_regEncode[src_lo]));
2187 }
2188 } else { // fpr --> stack spill
2189 assert(dst_lo_rc == rc_stack, "spill to bad register class");
2190 __ spill(as_FloatRegister(Matcher::_regEncode[src_lo]),
2191 is64 ? __ D : __ S, dst_offset);
2192 }
2193 break;
2194 case rc_stack:
2195 if (dst_lo_rc == rc_int) { // stack --> gpr load
2196 __ unspill(as_Register(Matcher::_regEncode[dst_lo]), is64, src_offset);
2197 } else if (dst_lo_rc == rc_float) { // stack --> fpr load
2198 __ unspill(as_FloatRegister(Matcher::_regEncode[dst_lo]),
2199 is64 ? __ D : __ S, src_offset);
2200 } else if (dst_lo_rc == rc_predicate) {
2201 __ unspill_sve_predicate(as_PRegister(Matcher::_regEncode[dst_lo]), ra_->reg2offset(src_lo),
2202 Matcher::scalable_vector_reg_size(T_BYTE) >> 3);
2203 } else { // stack --> stack copy
2204 assert(dst_lo_rc == rc_stack, "spill to bad register class");
2205 if (ideal_reg() == Op_RegVectMask) {
2206 __ spill_copy_sve_predicate_stack_to_stack(src_offset, dst_offset,
2207 Matcher::scalable_vector_reg_size(T_BYTE) >> 3);
2208 } else {
2209 __ unspill(rscratch1, is64, src_offset);
2210 __ spill(rscratch1, is64, dst_offset);
2211 }
2212 }
2213 break;
2214 case rc_predicate:
2215 if (dst_lo_rc == rc_predicate) {
2216 __ sve_mov(as_PRegister(Matcher::_regEncode[dst_lo]), as_PRegister(Matcher::_regEncode[src_lo]));
2217 } else if (dst_lo_rc == rc_stack) {
2218 __ spill_sve_predicate(as_PRegister(Matcher::_regEncode[src_lo]), ra_->reg2offset(dst_lo),
2219 Matcher::scalable_vector_reg_size(T_BYTE) >> 3);
2220 } else {
2221 assert(false, "bad src and dst rc_class combination.");
2222 ShouldNotReachHere();
2223 }
2224 break;
2225 default:
2226 assert(false, "bad rc_class for spill");
2227 ShouldNotReachHere();
2228 }
2229 }
2230
2231 if (st) {
2232 st->print("spill ");
2233 if (src_lo_rc == rc_stack) {
2234 st->print("[sp, #%d] -> ", ra_->reg2offset(src_lo));
2235 } else {
2236 st->print("%s -> ", Matcher::regName[src_lo]);
2237 }
2238 if (dst_lo_rc == rc_stack) {
2239 st->print("[sp, #%d]", ra_->reg2offset(dst_lo));
2240 } else {
2241 st->print("%s", Matcher::regName[dst_lo]);
2242 }
2243 if (bottom_type()->isa_vect() && !bottom_type()->isa_vectmask()) {
2244 int vsize = 0;
2245 switch (ideal_reg()) {
2246 case Op_VecD:
2247 vsize = 64;
2248 break;
2249 case Op_VecX:
2250 vsize = 128;
2251 break;
2252 case Op_VecA:
2253 vsize = Matcher::scalable_vector_reg_size(T_BYTE) * 8;
2254 break;
2255 default:
2256 assert(false, "bad register type for spill");
2257 ShouldNotReachHere();
2258 }
2259 st->print("\t# vector spill size = %d", vsize);
2260 } else if (ideal_reg() == Op_RegVectMask) {
2261 assert(Matcher::supports_scalable_vector(), "bad register type for spill");
2262 int vsize = Matcher::scalable_predicate_reg_slots() * 32;
2263 st->print("\t# predicate spill size = %d", vsize);
2264 } else {
2265 st->print("\t# spill size = %d", is64 ? 64 : 32);
2266 }
2267 }
2268
2269 return 0;
2270
2271 }
2272
2273 #ifndef PRODUCT
2274 void MachSpillCopyNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
2275 if (!ra_)
2276 st->print("N%d = SpillCopy(N%d)", _idx, in(1)->_idx);
2277 else
2278 implementation(NULL, ra_, false, st);
2279 }
2280 #endif
2281
2282 void MachSpillCopyNode::emit(CodeBuffer &cbuf, PhaseRegAlloc *ra_) const {
2283 implementation(&cbuf, ra_, false, NULL);
2284 }
2285
2286 uint MachSpillCopyNode::size(PhaseRegAlloc *ra_) const {
2287 return MachNode::size(ra_);
2288 }
2289
2290 //=============================================================================
2291
2292 #ifndef PRODUCT
2293 void BoxLockNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
2294 int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
2295 int reg = ra_->get_reg_first(this);
2296 st->print("add %s, rsp, #%d]\t# box lock",
2297 Matcher::regName[reg], offset);
2298 }
2299 #endif
2300
2301 void BoxLockNode::emit(CodeBuffer &cbuf, PhaseRegAlloc *ra_) const {
2302 C2_MacroAssembler _masm(&cbuf);
2303
2304 int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
2305 int reg = ra_->get_encode(this);
2306
2307 // This add will handle any 24-bit signed offset. 24 bits allows an
2308 // 8 megabyte stack frame.
2309 __ add(as_Register(reg), sp, offset);
2310 }
2311
2312 uint BoxLockNode::size(PhaseRegAlloc *ra_) const {
2313 // BoxLockNode is not a MachNode, so we can't just call MachNode::size(ra_).
2314 int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
2315
2316 if (Assembler::operand_valid_for_add_sub_immediate(offset)) {
2317 return NativeInstruction::instruction_size;
2318 } else {
2319 return 2 * NativeInstruction::instruction_size;
2320 }
2321 }
2322
2323 //=============================================================================
2324
2325 #ifndef PRODUCT
2326 void MachUEPNode::format(PhaseRegAlloc* ra_, outputStream* st) const
2327 {
2328 st->print_cr("# MachUEPNode");
2329 if (UseCompressedClassPointers) {
2330 st->print_cr("\tldrw rscratch1, j_rarg0 + oopDesc::klass_offset_in_bytes()]\t# compressed klass");
2331 if (CompressedKlassPointers::shift() != 0) {
2332 st->print_cr("\tdecode_klass_not_null rscratch1, rscratch1");
2333 }
2334 } else {
2335 st->print_cr("\tldr rscratch1, j_rarg0 + oopDesc::klass_offset_in_bytes()]\t# compressed klass");
2336 }
2337 st->print_cr("\tcmp r0, rscratch1\t # Inline cache check");
2338 st->print_cr("\tbne, SharedRuntime::_ic_miss_stub");
2339 }
2340 #endif
2341
2342 void MachUEPNode::emit(CodeBuffer& cbuf, PhaseRegAlloc* ra_) const
2343 {
2344 // This is the unverified entry point.
2345 C2_MacroAssembler _masm(&cbuf);
2346
2347 __ cmp_klass(j_rarg0, rscratch2, rscratch1);
2348 Label skip;
2349 // TODO
2350 // can we avoid this skip and still use a reloc?
2351 __ br(Assembler::EQ, skip);
2352 __ far_jump(RuntimeAddress(SharedRuntime::get_ic_miss_stub()));
2353 __ bind(skip);
2354 }
2355
2356 uint MachUEPNode::size(PhaseRegAlloc* ra_) const
2357 {
2358 return MachNode::size(ra_);
2359 }
2360
2361 // REQUIRED EMIT CODE
2362
2363 //=============================================================================
2364
2365 // Emit exception handler code.
2366 int HandlerImpl::emit_exception_handler(CodeBuffer& cbuf)
2367 {
2368 // mov rscratch1 #exception_blob_entry_point
2369 // br rscratch1
2370 // Note that the code buffer's insts_mark is always relative to insts.
2371 // That's why we must use the macroassembler to generate a handler.
2372 C2_MacroAssembler _masm(&cbuf);
2373 address base = __ start_a_stub(size_exception_handler());
2374 if (base == NULL) {
2375 ciEnv::current()->record_failure("CodeCache is full");
2376 return 0; // CodeBuffer::expand failed
2377 }
2378 int offset = __ offset();
2379 __ far_jump(RuntimeAddress(OptoRuntime::exception_blob()->entry_point()));
2380 assert(__ offset() - offset <= (int) size_exception_handler(), "overflow");
2381 __ end_a_stub();
2382 return offset;
2383 }
2384
2385 // Emit deopt handler code.
2386 int HandlerImpl::emit_deopt_handler(CodeBuffer& cbuf)
2387 {
2388 // Note that the code buffer's insts_mark is always relative to insts.
2389 // That's why we must use the macroassembler to generate a handler.
2390 C2_MacroAssembler _masm(&cbuf);
2391 address base = __ start_a_stub(size_deopt_handler());
2392 if (base == NULL) {
2393 ciEnv::current()->record_failure("CodeCache is full");
2394 return 0; // CodeBuffer::expand failed
2395 }
2396 int offset = __ offset();
2397
2398 __ adr(lr, __ pc());
2399 __ far_jump(RuntimeAddress(SharedRuntime::deopt_blob()->unpack()));
2400
2401 assert(__ offset() - offset == (int) size_deopt_handler(), "overflow");
2402 __ end_a_stub();
2403 return offset;
2404 }
2405
2406 // REQUIRED MATCHER CODE
2407
2408 //=============================================================================
2409
2410 const bool Matcher::match_rule_supported(int opcode) {
2411 if (!has_match_rule(opcode))
2412 return false;
2413
2414 bool ret_value = true;
2415 switch (opcode) {
2416 case Op_OnSpinWait:
2417 return VM_Version::supports_on_spin_wait();
2418 case Op_CacheWB:
2419 case Op_CacheWBPreSync:
2420 case Op_CacheWBPostSync:
2421 if (!VM_Version::supports_data_cache_line_flush()) {
2422 ret_value = false;
2423 }
2424 break;
2425 case Op_LoadVectorMasked:
2426 case Op_StoreVectorMasked:
2427 case Op_LoadVectorGatherMasked:
2428 case Op_StoreVectorScatterMasked:
2429 case Op_MaskAll:
2430 case Op_AndVMask:
2431 case Op_OrVMask:
2432 case Op_XorVMask:
2433 if (UseSVE == 0) {
2434 ret_value = false;
2435 }
2436 break;
2437 }
2438
2439 return ret_value; // Per default match rules are supported.
2440 }
2441
2442 // Identify extra cases that we might want to provide match rules for vector nodes and
2443 // other intrinsics guarded with vector length (vlen) and element type (bt).
2444 const bool Matcher::match_rule_supported_vector(int opcode, int vlen, BasicType bt) {
2445 if (!match_rule_supported(opcode)) {
2446 return false;
2447 }
2448 int bit_size = vlen * type2aelembytes(bt) * 8;
2449 if (UseSVE == 0 && bit_size > 128) {
2450 return false;
2451 }
2452 if (UseSVE > 0) {
2453 return op_sve_supported(opcode, vlen, bt);
2454 } else { // NEON
2455 // Special cases
2456 switch (opcode) {
2457 case Op_VectorMaskCmp:
2458 if (vlen < 2 || bit_size < 64) {
2459 return false;
2460 }
2461 break;
2462 case Op_MulAddVS2VI:
2463 if (bit_size < 128) {
2464 return false;
2465 }
2466 break;
2467 case Op_MulVL:
2468 return false;
2469 case Op_VectorLoadShuffle:
2470 case Op_VectorRearrange:
2471 if (vlen < 4) {
2472 return false;
2473 }
2474 break;
2475 case Op_LoadVectorGather:
2476 case Op_StoreVectorScatter:
2477 return false;
2478 default:
2479 break;
2480 }
2481 }
2482 return vector_size_supported(bt, vlen);
2483 }
2484
2485 const bool Matcher::match_rule_supported_vector_masked(int opcode, int vlen, BasicType bt) {
2486 // Only SVE supports masked operations.
2487 if (UseSVE == 0) {
2488 return false;
2489 }
2490 return match_rule_supported(opcode) &&
2491 masked_op_sve_supported(opcode, vlen, bt);
2492 }
2493
2494 const RegMask* Matcher::predicate_reg_mask(void) {
2495 return &_PR_REG_mask;
2496 }
2497
2498 const TypeVectMask* Matcher::predicate_reg_type(const Type* elemTy, int length) {
2499 return new TypeVectMask(elemTy, length);
2500 }
2501
2502 // Vector calling convention not yet implemented.
2503 const bool Matcher::supports_vector_calling_convention(void) {
2504 return false;
2505 }
2506
2507 OptoRegPair Matcher::vector_return_value(uint ideal_reg) {
2508 Unimplemented();
2509 return OptoRegPair(0, 0);
2510 }
2511
2512 // Is this branch offset short enough that a short branch can be used?
2513 //
2514 // NOTE: If the platform does not provide any short branch variants, then
2515 // this method should return false for offset 0.
2516 bool Matcher::is_short_branch_offset(int rule, int br_size, int offset) {
2517 // The passed offset is relative to address of the branch.
2518
2519 return (-32768 <= offset && offset < 32768);
2520 }
2521
2522 // Vector width in bytes.
2523 const int Matcher::vector_width_in_bytes(BasicType bt) {
2524 // The MaxVectorSize should have been set by detecting SVE max vector register size.
2525 int size = MIN2((UseSVE > 0) ? 256 : 16, (int)MaxVectorSize);
2526 // Minimum 2 values in vector
2527 if (size < 2*type2aelembytes(bt)) size = 0;
2528 // But never < 4
2529 if (size < 4) size = 0;
2530 return size;
2531 }
2532
2533 // Limits on vector size (number of elements) loaded into vector.
2534 const int Matcher::max_vector_size(const BasicType bt) {
2535 return vector_width_in_bytes(bt)/type2aelembytes(bt);
2536 }
2537
2538 const int Matcher::min_vector_size(const BasicType bt) {
2539 int max_size = max_vector_size(bt);
2540 // Limit the min vector size to 8 bytes.
2541 int size = 8 / type2aelembytes(bt);
2542 if (bt == T_BYTE) {
2543 // To support vector api shuffle/rearrange.
2544 size = 4;
2545 } else if (bt == T_BOOLEAN) {
2546 // To support vector api load/store mask.
2547 size = 2;
2548 }
2549 if (size < 2) size = 2;
2550 return MIN2(size, max_size);
2551 }
2552
2553 // Actual max scalable vector register length.
2554 const int Matcher::scalable_vector_reg_size(const BasicType bt) {
2555 return Matcher::max_vector_size(bt);
2556 }
2557
2558 // Vector ideal reg.
2559 const uint Matcher::vector_ideal_reg(int len) {
2560 if (UseSVE > 0 && 2 <= len && len <= 256) {
2561 return Op_VecA;
2562 }
2563 switch(len) {
2564 // For 16-bit/32-bit mask vector, reuse VecD.
2565 case 2:
2566 case 4:
2567 case 8: return Op_VecD;
2568 case 16: return Op_VecX;
2569 }
2570 ShouldNotReachHere();
2571 return 0;
2572 }
2573
2574 MachOper* Matcher::pd_specialize_generic_vector_operand(MachOper* original_opnd, uint ideal_reg, bool is_temp) {
2575 ShouldNotReachHere(); // generic vector operands not supported
2576 return NULL;
2577 }
2578
2579 bool Matcher::is_reg2reg_move(MachNode* m) {
2580 ShouldNotReachHere(); // generic vector operands not supported
2581 return false;
2582 }
2583
2584 bool Matcher::is_generic_vector(MachOper* opnd) {
2585 ShouldNotReachHere(); // generic vector operands not supported
2586 return false;
2587 }
2588
2589 // Return whether or not this register is ever used as an argument.
2590 // This function is used on startup to build the trampoline stubs in
2591 // generateOptoStub. Registers not mentioned will be killed by the VM
2592 // call in the trampoline, and arguments in those registers not be
2593 // available to the callee.
2594 bool Matcher::can_be_java_arg(int reg)
2595 {
2596 return
2597 reg == R0_num || reg == R0_H_num ||
2598 reg == R1_num || reg == R1_H_num ||
2599 reg == R2_num || reg == R2_H_num ||
2600 reg == R3_num || reg == R3_H_num ||
2601 reg == R4_num || reg == R4_H_num ||
2602 reg == R5_num || reg == R5_H_num ||
2603 reg == R6_num || reg == R6_H_num ||
2604 reg == R7_num || reg == R7_H_num ||
2605 reg == V0_num || reg == V0_H_num ||
2606 reg == V1_num || reg == V1_H_num ||
2607 reg == V2_num || reg == V2_H_num ||
2608 reg == V3_num || reg == V3_H_num ||
2609 reg == V4_num || reg == V4_H_num ||
2610 reg == V5_num || reg == V5_H_num ||
2611 reg == V6_num || reg == V6_H_num ||
2612 reg == V7_num || reg == V7_H_num;
2613 }
2614
2615 bool Matcher::is_spillable_arg(int reg)
2616 {
2617 return can_be_java_arg(reg);
2618 }
2619
2620 uint Matcher::int_pressure_limit()
2621 {
2622 // JDK-8183543: When taking the number of available registers as int
2623 // register pressure threshold, the jtreg test:
2624 // test/hotspot/jtreg/compiler/regalloc/TestC2IntPressure.java
2625 // failed due to C2 compilation failure with
2626 // "COMPILE SKIPPED: failed spill-split-recycle sanity check".
2627 //
2628 // A derived pointer is live at CallNode and then is flagged by RA
2629 // as a spilled LRG. Spilling heuristics(Spill-USE) explicitly skip
2630 // derived pointers and lastly fail to spill after reaching maximum
2631 // number of iterations. Lowering the default pressure threshold to
2632 // (_NO_SPECIAL_REG32_mask.Size() minus 1) forces CallNode to become
2633 // a high register pressure area of the code so that split_DEF can
2634 // generate DefinitionSpillCopy for the derived pointer.
2635 uint default_int_pressure_threshold = _NO_SPECIAL_REG32_mask.Size() - 1;
2636 if (!PreserveFramePointer) {
2637 // When PreserveFramePointer is off, frame pointer is allocatable,
2638 // but different from other SOC registers, it is excluded from
2639 // fatproj's mask because its save type is No-Save. Decrease 1 to
2640 // ensure high pressure at fatproj when PreserveFramePointer is off.
2641 // See check_pressure_at_fatproj().
2642 default_int_pressure_threshold--;
2643 }
2644 return (INTPRESSURE == -1) ? default_int_pressure_threshold : INTPRESSURE;
2645 }
2646
2647 uint Matcher::float_pressure_limit()
2648 {
2649 // _FLOAT_REG_mask is generated by adlc from the float_reg register class.
2650 return (FLOATPRESSURE == -1) ? _FLOAT_REG_mask.Size() : FLOATPRESSURE;
2651 }
2652
2653 bool Matcher::use_asm_for_ldiv_by_con(jlong divisor) {
2654 return false;
2655 }
2656
2657 RegMask Matcher::divI_proj_mask() {
2658 ShouldNotReachHere();
2659 return RegMask();
2660 }
2661
2662 // Register for MODI projection of divmodI.
2663 RegMask Matcher::modI_proj_mask() {
2664 ShouldNotReachHere();
2665 return RegMask();
2666 }
2667
2668 // Register for DIVL projection of divmodL.
2669 RegMask Matcher::divL_proj_mask() {
2670 ShouldNotReachHere();
2671 return RegMask();
2672 }
2673
2674 // Register for MODL projection of divmodL.
2675 RegMask Matcher::modL_proj_mask() {
2676 ShouldNotReachHere();
2677 return RegMask();
2678 }
2679
2680 const RegMask Matcher::method_handle_invoke_SP_save_mask() {
2681 return FP_REG_mask();
2682 }
2683
2684 bool size_fits_all_mem_uses(AddPNode* addp, int shift) {
2685 for (DUIterator_Fast imax, i = addp->fast_outs(imax); i < imax; i++) {
2686 Node* u = addp->fast_out(i);
2687 if (u->is_LoadStore()) {
2688 // On AArch64, LoadStoreNodes (i.e. compare and swap
2689 // instructions) only take register indirect as an operand, so
2690 // any attempt to use an AddPNode as an input to a LoadStoreNode
2691 // must fail.
2692 return false;
2693 }
2694 if (u->is_Mem()) {
2695 int opsize = u->as_Mem()->memory_size();
2696 assert(opsize > 0, "unexpected memory operand size");
2697 if (u->as_Mem()->memory_size() != (1<<shift)) {
2698 return false;
2699 }
2700 }
2701 }
2702 return true;
2703 }
2704
2705 bool can_combine_with_imm(Node* binary_node, Node* replicate_node) {
2706 if (UseSVE == 0 || !VectorNode::is_invariant_vector(replicate_node)){
2707 return false;
2708 }
2709 Node* imm_node = replicate_node->in(1);
2710 if (!imm_node->is_Con()) {
2711 return false;
2712 }
2713
2714 const Type* t = imm_node->bottom_type();
2715 if (!(t->isa_int() || t->isa_long())) {
2716 return false;
2717 }
2718
2719 switch (binary_node->Opcode()) {
2720 case Op_AndV:
2721 case Op_OrV:
2722 case Op_XorV: {
2723 Assembler::SIMD_RegVariant T = Assembler::elemType_to_regVariant(Matcher::vector_element_basic_type(binary_node));
2724 uint64_t value = t->isa_long() ? (uint64_t)imm_node->get_long() : (uint64_t)imm_node->get_int();
2725 return Assembler::operand_valid_for_sve_logical_immediate(Assembler::regVariant_to_elemBits(T), value);
2726 }
2727 case Op_AddVB:
2728 return (imm_node->get_int() <= 255 && imm_node->get_int() >= -255);
2729 case Op_AddVS:
2730 case Op_AddVI:
2731 return Assembler::operand_valid_for_sve_add_sub_immediate((int64_t)imm_node->get_int());
2732 case Op_AddVL:
2733 return Assembler::operand_valid_for_sve_add_sub_immediate(imm_node->get_long());
2734 default:
2735 return false;
2736 }
2737 }
2738
2739 bool is_vector_arith_imm_pattern(Node* n, Node* m) {
2740 if (n != NULL && m != NULL) {
2741 return can_combine_with_imm(n, m);
2742 }
2743 return false;
2744 }
2745
2746 // Should the matcher clone input 'm' of node 'n'?
2747 bool Matcher::pd_clone_node(Node* n, Node* m, Matcher::MStack& mstack) {
2748 // ShiftV src (ShiftCntV con)
2749 // Binary src (Replicate con)
2750 if (is_vshift_con_pattern(n, m) || is_vector_arith_imm_pattern(n, m)) {
2751 mstack.push(m, Visit);
2752 return true;
2753 }
2754
2755 return false;
2756 }
2757
2758 // Should the Matcher clone shifts on addressing modes, expecting them
2759 // to be subsumed into complex addressing expressions or compute them
2760 // into registers?
2761 bool Matcher::pd_clone_address_expressions(AddPNode* m, Matcher::MStack& mstack, VectorSet& address_visited) {
2762 if (clone_base_plus_offset_address(m, mstack, address_visited)) {
2763 return true;
2764 }
2765
2766 Node *off = m->in(AddPNode::Offset);
2767 if (off->Opcode() == Op_LShiftL && off->in(2)->is_Con() &&
2768 size_fits_all_mem_uses(m, off->in(2)->get_int()) &&
2769 // Are there other uses besides address expressions?
2770 !is_visited(off)) {
2771 address_visited.set(off->_idx); // Flag as address_visited
2772 mstack.push(off->in(2), Visit);
2773 Node *conv = off->in(1);
2774 if (conv->Opcode() == Op_ConvI2L &&
2775 // Are there other uses besides address expressions?
2776 !is_visited(conv)) {
2777 address_visited.set(conv->_idx); // Flag as address_visited
2778 mstack.push(conv->in(1), Pre_Visit);
2779 } else {
2780 mstack.push(conv, Pre_Visit);
2781 }
2782 address_visited.test_set(m->_idx); // Flag as address_visited
2783 mstack.push(m->in(AddPNode::Address), Pre_Visit);
2784 mstack.push(m->in(AddPNode::Base), Pre_Visit);
2785 return true;
2786 } else if (off->Opcode() == Op_ConvI2L &&
2787 // Are there other uses besides address expressions?
2788 !is_visited(off)) {
2789 address_visited.test_set(m->_idx); // Flag as address_visited
2790 address_visited.set(off->_idx); // Flag as address_visited
2791 mstack.push(off->in(1), Pre_Visit);
2792 mstack.push(m->in(AddPNode::Address), Pre_Visit);
2793 mstack.push(m->in(AddPNode::Base), Pre_Visit);
2794 return true;
2795 }
2796 return false;
2797 }
2798
2799 bool Parse::do_one_bytecode_targeted() {
2800 return false;
2801 }
2802
2803 #define MOV_VOLATILE(REG, BASE, INDEX, SCALE, DISP, SCRATCH, INSN) \
2804 C2_MacroAssembler _masm(&cbuf); \
2805 { \
2806 guarantee(INDEX == -1, "mode not permitted for volatile"); \
2807 guarantee(DISP == 0, "mode not permitted for volatile"); \
2808 guarantee(SCALE == 0, "mode not permitted for volatile"); \
2809 __ INSN(REG, as_Register(BASE)); \
2810 }
2811
2812
2813 static Address mem2address(int opcode, Register base, int index, int size, int disp)
2814 {
2815 Address::extend scale;
2816
2817 // Hooboy, this is fugly. We need a way to communicate to the
2818 // encoder that the index needs to be sign extended, so we have to
2819 // enumerate all the cases.
2820 switch (opcode) {
2821 case INDINDEXSCALEDI2L:
2822 case INDINDEXSCALEDI2LN:
2823 case INDINDEXI2L:
2824 case INDINDEXI2LN:
2825 scale = Address::sxtw(size);
2826 break;
2827 default:
2828 scale = Address::lsl(size);
2829 }
2830
2831 if (index == -1) {
2832 return Address(base, disp);
2833 } else {
2834 assert(disp == 0, "unsupported address mode: disp = %d", disp);
2835 return Address(base, as_Register(index), scale);
2836 }
2837 }
2838
2839
2840 typedef void (MacroAssembler::* mem_insn)(Register Rt, const Address &adr);
2841 typedef void (MacroAssembler::* mem_insn2)(Register Rt, Register adr);
2842 typedef void (MacroAssembler::* mem_float_insn)(FloatRegister Rt, const Address &adr);
2843 typedef void (MacroAssembler::* mem_vector_insn)(FloatRegister Rt,
2844 MacroAssembler::SIMD_RegVariant T, const Address &adr);
2845
2846 // Used for all non-volatile memory accesses. The use of
2847 // $mem->opcode() to discover whether this pattern uses sign-extended
2848 // offsets is something of a kludge.
2849 static void loadStore(C2_MacroAssembler masm, mem_insn insn,
2850 Register reg, int opcode,
2851 Register base, int index, int scale, int disp,
2852 int size_in_memory)
2853 {
2854 Address addr = mem2address(opcode, base, index, scale, disp);
2855 if (addr.getMode() == Address::base_plus_offset) {
2856 /* If we get an out-of-range offset it is a bug in the compiler,
2857 so we assert here. */
2858 assert(Address::offset_ok_for_immed(addr.offset(), exact_log2(size_in_memory)),
2859 "c2 compiler bug");
2860 /* Fix up any out-of-range offsets. */
2861 assert_different_registers(rscratch1, base);
2862 assert_different_registers(rscratch1, reg);
2863 addr = masm.legitimize_address(addr, size_in_memory, rscratch1);
2864 }
2865 (masm.*insn)(reg, addr);
2866 }
2867
2868 static void loadStore(C2_MacroAssembler masm, mem_float_insn insn,
2869 FloatRegister reg, int opcode,
2870 Register base, int index, int size, int disp,
2871 int size_in_memory)
2872 {
2873 Address::extend scale;
2874
2875 switch (opcode) {
2876 case INDINDEXSCALEDI2L:
2877 case INDINDEXSCALEDI2LN:
2878 scale = Address::sxtw(size);
2879 break;
2880 default:
2881 scale = Address::lsl(size);
2882 }
2883
2884 if (index == -1) {
2885 /* If we get an out-of-range offset it is a bug in the compiler,
2886 so we assert here. */
2887 assert(Address::offset_ok_for_immed(disp, exact_log2(size_in_memory)), "c2 compiler bug");
2888 /* Fix up any out-of-range offsets. */
2889 assert_different_registers(rscratch1, base);
2890 Address addr = Address(base, disp);
2891 addr = masm.legitimize_address(addr, size_in_memory, rscratch1);
2892 (masm.*insn)(reg, addr);
2893 } else {
2894 assert(disp == 0, "unsupported address mode: disp = %d", disp);
2895 (masm.*insn)(reg, Address(base, as_Register(index), scale));
2896 }
2897 }
2898
2899 static void loadStore(C2_MacroAssembler masm, mem_vector_insn insn,
2900 FloatRegister reg, MacroAssembler::SIMD_RegVariant T,
2901 int opcode, Register base, int index, int size, int disp)
2902 {
2903 if (index == -1) {
2904 (masm.*insn)(reg, T, Address(base, disp));
2905 } else {
2906 assert(disp == 0, "unsupported address mode");
2907 (masm.*insn)(reg, T, Address(base, as_Register(index), Address::lsl(size)));
2908 }
2909 }
2910
2911 %}
2912
2913
2914
2915 //----------ENCODING BLOCK-----------------------------------------------------
2916 // This block specifies the encoding classes used by the compiler to
2917 // output byte streams. Encoding classes are parameterized macros
2918 // used by Machine Instruction Nodes in order to generate the bit
2919 // encoding of the instruction. Operands specify their base encoding
2920 // interface with the interface keyword. There are currently
2921 // supported four interfaces, REG_INTER, CONST_INTER, MEMORY_INTER, &
2922 // COND_INTER. REG_INTER causes an operand to generate a function
2923 // which returns its register number when queried. CONST_INTER causes
2924 // an operand to generate a function which returns the value of the
2925 // constant when queried. MEMORY_INTER causes an operand to generate
2926 // four functions which return the Base Register, the Index Register,
2927 // the Scale Value, and the Offset Value of the operand when queried.
2928 // COND_INTER causes an operand to generate six functions which return
2929 // the encoding code (ie - encoding bits for the instruction)
2930 // associated with each basic boolean condition for a conditional
2931 // instruction.
2932 //
2933 // Instructions specify two basic values for encoding. Again, a
2934 // function is available to check if the constant displacement is an
2935 // oop. They use the ins_encode keyword to specify their encoding
2936 // classes (which must be a sequence of enc_class names, and their
2937 // parameters, specified in the encoding block), and they use the
2938 // opcode keyword to specify, in order, their primary, secondary, and
2939 // tertiary opcode. Only the opcode sections which a particular
2940 // instruction needs for encoding need to be specified.
2941 encode %{
2942 // Build emit functions for each basic byte or larger field in the
2943 // intel encoding scheme (opcode, rm, sib, immediate), and call them
2944 // from C++ code in the enc_class source block. Emit functions will
2945 // live in the main source block for now. In future, we can
2946 // generalize this by adding a syntax that specifies the sizes of
2947 // fields in an order, so that the adlc can build the emit functions
2948 // automagically
2949
2950 // catch all for unimplemented encodings
2951 enc_class enc_unimplemented %{
2952 C2_MacroAssembler _masm(&cbuf);
2953 __ unimplemented("C2 catch all");
2954 %}
2955
2956 // BEGIN Non-volatile memory access
2957
2958 // This encoding class is generated automatically from ad_encode.m4.
2959 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2960 enc_class aarch64_enc_ldrsbw(iRegI dst, memory1 mem) %{
2961 Register dst_reg = as_Register($dst$$reg);
2962 loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrsbw, dst_reg, $mem->opcode(),
2963 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
2964 %}
2965
2966 // This encoding class is generated automatically from ad_encode.m4.
2967 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2968 enc_class aarch64_enc_ldrsb(iRegI dst, memory1 mem) %{
2969 Register dst_reg = as_Register($dst$$reg);
2970 loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrsb, dst_reg, $mem->opcode(),
2971 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
2972 %}
2973
2974 // This encoding class is generated automatically from ad_encode.m4.
2975 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2976 enc_class aarch64_enc_ldrb(iRegI dst, memory1 mem) %{
2977 Register dst_reg = as_Register($dst$$reg);
2978 loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrb, dst_reg, $mem->opcode(),
2979 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
2980 %}
2981
2982 // This encoding class is generated automatically from ad_encode.m4.
2983 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2984 enc_class aarch64_enc_ldrb(iRegL dst, memory1 mem) %{
2985 Register dst_reg = as_Register($dst$$reg);
2986 loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrb, dst_reg, $mem->opcode(),
2987 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
2988 %}
2989
2990 // This encoding class is generated automatically from ad_encode.m4.
2991 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2992 enc_class aarch64_enc_ldrshw(iRegI dst, memory2 mem) %{
2993 Register dst_reg = as_Register($dst$$reg);
2994 loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrshw, dst_reg, $mem->opcode(),
2995 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
2996 %}
2997
2998 // This encoding class is generated automatically from ad_encode.m4.
2999 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
3000 enc_class aarch64_enc_ldrsh(iRegI dst, memory2 mem) %{
3001 Register dst_reg = as_Register($dst$$reg);
3002 loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrsh, dst_reg, $mem->opcode(),
3003 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
3004 %}
3005
3006 // This encoding class is generated automatically from ad_encode.m4.
3007 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
3008 enc_class aarch64_enc_ldrh(iRegI dst, memory2 mem) %{
3009 Register dst_reg = as_Register($dst$$reg);
3010 loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrh, dst_reg, $mem->opcode(),
3011 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
3012 %}
3013
3014 // This encoding class is generated automatically from ad_encode.m4.
3015 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
3016 enc_class aarch64_enc_ldrh(iRegL dst, memory2 mem) %{
3017 Register dst_reg = as_Register($dst$$reg);
3018 loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrh, dst_reg, $mem->opcode(),
3019 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
3020 %}
3021
3022 // This encoding class is generated automatically from ad_encode.m4.
3023 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
3024 enc_class aarch64_enc_ldrw(iRegI dst, memory4 mem) %{
3025 Register dst_reg = as_Register($dst$$reg);
3026 loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrw, dst_reg, $mem->opcode(),
3027 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
3028 %}
3029
3030 // This encoding class is generated automatically from ad_encode.m4.
3031 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
3032 enc_class aarch64_enc_ldrw(iRegL dst, memory4 mem) %{
3033 Register dst_reg = as_Register($dst$$reg);
3034 loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrw, dst_reg, $mem->opcode(),
3035 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
3036 %}
3037
3038 // This encoding class is generated automatically from ad_encode.m4.
3039 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
3040 enc_class aarch64_enc_ldrsw(iRegL dst, memory4 mem) %{
3041 Register dst_reg = as_Register($dst$$reg);
3042 loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrsw, dst_reg, $mem->opcode(),
3043 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
3044 %}
3045
3046 // This encoding class is generated automatically from ad_encode.m4.
3047 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
3048 enc_class aarch64_enc_ldr(iRegL dst, memory8 mem) %{
3049 Register dst_reg = as_Register($dst$$reg);
3050 loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldr, dst_reg, $mem->opcode(),
3051 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
3052 %}
3053
3054 // This encoding class is generated automatically from ad_encode.m4.
3055 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
3056 enc_class aarch64_enc_ldrs(vRegF dst, memory4 mem) %{
3057 FloatRegister dst_reg = as_FloatRegister($dst$$reg);
3058 loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrs, dst_reg, $mem->opcode(),
3059 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
3060 %}
3061
3062 // This encoding class is generated automatically from ad_encode.m4.
3063 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
3064 enc_class aarch64_enc_ldrd(vRegD dst, memory8 mem) %{
3065 FloatRegister dst_reg = as_FloatRegister($dst$$reg);
3066 loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrd, dst_reg, $mem->opcode(),
3067 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
3068 %}
3069
3070 // This encoding class is generated automatically from ad_encode.m4.
3071 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
3072 enc_class aarch64_enc_strb(iRegI src, memory1 mem) %{
3073 Register src_reg = as_Register($src$$reg);
3074 loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::strb, src_reg, $mem->opcode(),
3075 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
3076 %}
3077
3078 // This encoding class is generated automatically from ad_encode.m4.
3079 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
3080 enc_class aarch64_enc_strb0(memory1 mem) %{
3081 C2_MacroAssembler _masm(&cbuf);
3082 loadStore(_masm, &MacroAssembler::strb, zr, $mem->opcode(),
3083 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
3084 %}
3085
3086 // This encoding class is generated automatically from ad_encode.m4.
3087 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
3088 enc_class aarch64_enc_strh(iRegI src, memory2 mem) %{
3089 Register src_reg = as_Register($src$$reg);
3090 loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::strh, src_reg, $mem->opcode(),
3091 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
3092 %}
3093
3094 // This encoding class is generated automatically from ad_encode.m4.
3095 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
3096 enc_class aarch64_enc_strh0(memory2 mem) %{
3097 C2_MacroAssembler _masm(&cbuf);
3098 loadStore(_masm, &MacroAssembler::strh, zr, $mem->opcode(),
3099 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
3100 %}
3101
3102 // This encoding class is generated automatically from ad_encode.m4.
3103 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
3104 enc_class aarch64_enc_strw(iRegI src, memory4 mem) %{
3105 Register src_reg = as_Register($src$$reg);
3106 loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::strw, src_reg, $mem->opcode(),
3107 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
3108 %}
3109
3110 // This encoding class is generated automatically from ad_encode.m4.
3111 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
3112 enc_class aarch64_enc_strw0(memory4 mem) %{
3113 C2_MacroAssembler _masm(&cbuf);
3114 loadStore(_masm, &MacroAssembler::strw, zr, $mem->opcode(),
3115 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
3116 %}
3117
3118 // This encoding class is generated automatically from ad_encode.m4.
3119 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
3120 enc_class aarch64_enc_str(iRegL src, memory8 mem) %{
3121 Register src_reg = as_Register($src$$reg);
3122 // we sometimes get asked to store the stack pointer into the
3123 // current thread -- we cannot do that directly on AArch64
3124 if (src_reg == r31_sp) {
3125 C2_MacroAssembler _masm(&cbuf);
3126 assert(as_Register($mem$$base) == rthread, "unexpected store for sp");
3127 __ mov(rscratch2, sp);
3128 src_reg = rscratch2;
3129 }
3130 loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::str, src_reg, $mem->opcode(),
3131 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
3132 %}
3133
3134 // This encoding class is generated automatically from ad_encode.m4.
3135 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
3136 enc_class aarch64_enc_str0(memory8 mem) %{
3137 C2_MacroAssembler _masm(&cbuf);
3138 loadStore(_masm, &MacroAssembler::str, zr, $mem->opcode(),
3139 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
3140 %}
3141
3142 // This encoding class is generated automatically from ad_encode.m4.
3143 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
3144 enc_class aarch64_enc_strs(vRegF src, memory4 mem) %{
3145 FloatRegister src_reg = as_FloatRegister($src$$reg);
3146 loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::strs, src_reg, $mem->opcode(),
3147 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
3148 %}
3149
3150 // This encoding class is generated automatically from ad_encode.m4.
3151 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
3152 enc_class aarch64_enc_strd(vRegD src, memory8 mem) %{
3153 FloatRegister src_reg = as_FloatRegister($src$$reg);
3154 loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::strd, src_reg, $mem->opcode(),
3155 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
3156 %}
3157
3158 // This encoding class is generated automatically from ad_encode.m4.
3159 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
3160 enc_class aarch64_enc_strb0_ordered(memory4 mem) %{
3161 C2_MacroAssembler _masm(&cbuf);
3162 __ membar(Assembler::StoreStore);
3163 loadStore(_masm, &MacroAssembler::strb, zr, $mem->opcode(),
3164 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
3165 %}
3166
3167 // END Non-volatile memory access
3168
3169 // Vector loads and stores
3170 enc_class aarch64_enc_ldrvH(vecD dst, memory mem) %{
3171 FloatRegister dst_reg = as_FloatRegister($dst$$reg);
3172 loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldr, dst_reg, MacroAssembler::H,
3173 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3174 %}
3175
3176 enc_class aarch64_enc_ldrvS(vecD dst, memory mem) %{
3177 FloatRegister dst_reg = as_FloatRegister($dst$$reg);
3178 loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldr, dst_reg, MacroAssembler::S,
3179 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3180 %}
3181
3182 enc_class aarch64_enc_ldrvD(vecD dst, memory mem) %{
3183 FloatRegister dst_reg = as_FloatRegister($dst$$reg);
3184 loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldr, dst_reg, MacroAssembler::D,
3185 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3186 %}
3187
3188 enc_class aarch64_enc_ldrvQ(vecX dst, memory mem) %{
3189 FloatRegister dst_reg = as_FloatRegister($dst$$reg);
3190 loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldr, dst_reg, MacroAssembler::Q,
3191 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3192 %}
3193
3194 enc_class aarch64_enc_strvH(vecD src, memory mem) %{
3195 FloatRegister src_reg = as_FloatRegister($src$$reg);
3196 loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::str, src_reg, MacroAssembler::H,
3197 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3198 %}
3199
3200 enc_class aarch64_enc_strvS(vecD src, memory mem) %{
3201 FloatRegister src_reg = as_FloatRegister($src$$reg);
3202 loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::str, src_reg, MacroAssembler::S,
3203 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3204 %}
3205
3206 enc_class aarch64_enc_strvD(vecD src, memory mem) %{
3207 FloatRegister src_reg = as_FloatRegister($src$$reg);
3208 loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::str, src_reg, MacroAssembler::D,
3209 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3210 %}
3211
3212 enc_class aarch64_enc_strvQ(vecX src, memory mem) %{
3213 FloatRegister src_reg = as_FloatRegister($src$$reg);
3214 loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::str, src_reg, MacroAssembler::Q,
3215 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3216 %}
3217
3218 // volatile loads and stores
3219
3220 enc_class aarch64_enc_stlrb(iRegI src, memory mem) %{
3221 MOV_VOLATILE(as_Register($src$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3222 rscratch1, stlrb);
3223 %}
3224
3225 enc_class aarch64_enc_stlrb0(memory mem) %{
3226 MOV_VOLATILE(zr, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3227 rscratch1, stlrb);
3228 %}
3229
3230 enc_class aarch64_enc_stlrh(iRegI src, memory mem) %{
3231 MOV_VOLATILE(as_Register($src$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3232 rscratch1, stlrh);
3233 %}
3234
3235 enc_class aarch64_enc_stlrh0(memory mem) %{
3236 MOV_VOLATILE(zr, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3237 rscratch1, stlrh);
3238 %}
3239
3240 enc_class aarch64_enc_stlrw(iRegI src, memory mem) %{
3241 MOV_VOLATILE(as_Register($src$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3242 rscratch1, stlrw);
3243 %}
3244
3245 enc_class aarch64_enc_stlrw0(memory mem) %{
3246 MOV_VOLATILE(zr, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3247 rscratch1, stlrw);
3248 %}
3249
3250 enc_class aarch64_enc_ldarsbw(iRegI dst, memory mem) %{
3251 Register dst_reg = as_Register($dst$$reg);
3252 MOV_VOLATILE(dst_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3253 rscratch1, ldarb);
3254 __ sxtbw(dst_reg, dst_reg);
3255 %}
3256
3257 enc_class aarch64_enc_ldarsb(iRegL dst, memory mem) %{
3258 Register dst_reg = as_Register($dst$$reg);
3259 MOV_VOLATILE(dst_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3260 rscratch1, ldarb);
3261 __ sxtb(dst_reg, dst_reg);
3262 %}
3263
3264 enc_class aarch64_enc_ldarbw(iRegI dst, memory mem) %{
3265 MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3266 rscratch1, ldarb);
3267 %}
3268
3269 enc_class aarch64_enc_ldarb(iRegL dst, memory mem) %{
3270 MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3271 rscratch1, ldarb);
3272 %}
3273
3274 enc_class aarch64_enc_ldarshw(iRegI dst, memory mem) %{
3275 Register dst_reg = as_Register($dst$$reg);
3276 MOV_VOLATILE(dst_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3277 rscratch1, ldarh);
3278 __ sxthw(dst_reg, dst_reg);
3279 %}
3280
3281 enc_class aarch64_enc_ldarsh(iRegL dst, memory mem) %{
3282 Register dst_reg = as_Register($dst$$reg);
3283 MOV_VOLATILE(dst_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3284 rscratch1, ldarh);
3285 __ sxth(dst_reg, dst_reg);
3286 %}
3287
3288 enc_class aarch64_enc_ldarhw(iRegI dst, memory mem) %{
3289 MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3290 rscratch1, ldarh);
3291 %}
3292
3293 enc_class aarch64_enc_ldarh(iRegL dst, memory mem) %{
3294 MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3295 rscratch1, ldarh);
3296 %}
3297
3298 enc_class aarch64_enc_ldarw(iRegI dst, memory mem) %{
3299 MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3300 rscratch1, ldarw);
3301 %}
3302
3303 enc_class aarch64_enc_ldarw(iRegL dst, memory mem) %{
3304 MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3305 rscratch1, ldarw);
3306 %}
3307
3308 enc_class aarch64_enc_ldar(iRegL dst, memory mem) %{
3309 MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3310 rscratch1, ldar);
3311 %}
3312
3313 enc_class aarch64_enc_fldars(vRegF dst, memory mem) %{
3314 MOV_VOLATILE(rscratch1, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3315 rscratch1, ldarw);
3316 __ fmovs(as_FloatRegister($dst$$reg), rscratch1);
3317 %}
3318
3319 enc_class aarch64_enc_fldard(vRegD dst, memory mem) %{
3320 MOV_VOLATILE(rscratch1, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3321 rscratch1, ldar);
3322 __ fmovd(as_FloatRegister($dst$$reg), rscratch1);
3323 %}
3324
3325 enc_class aarch64_enc_stlr(iRegL src, memory mem) %{
3326 Register src_reg = as_Register($src$$reg);
3327 // we sometimes get asked to store the stack pointer into the
3328 // current thread -- we cannot do that directly on AArch64
3329 if (src_reg == r31_sp) {
3330 C2_MacroAssembler _masm(&cbuf);
3331 assert(as_Register($mem$$base) == rthread, "unexpected store for sp");
3332 __ mov(rscratch2, sp);
3333 src_reg = rscratch2;
3334 }
3335 MOV_VOLATILE(src_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3336 rscratch1, stlr);
3337 %}
3338
3339 enc_class aarch64_enc_stlr0(memory mem) %{
3340 MOV_VOLATILE(zr, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3341 rscratch1, stlr);
3342 %}
3343
3344 enc_class aarch64_enc_fstlrs(vRegF src, memory mem) %{
3345 {
3346 C2_MacroAssembler _masm(&cbuf);
3347 FloatRegister src_reg = as_FloatRegister($src$$reg);
3348 __ fmovs(rscratch2, src_reg);
3349 }
3350 MOV_VOLATILE(rscratch2, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3351 rscratch1, stlrw);
3352 %}
3353
3354 enc_class aarch64_enc_fstlrd(vRegD src, memory mem) %{
3355 {
3356 C2_MacroAssembler _masm(&cbuf);
3357 FloatRegister src_reg = as_FloatRegister($src$$reg);
3358 __ fmovd(rscratch2, src_reg);
3359 }
3360 MOV_VOLATILE(rscratch2, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3361 rscratch1, stlr);
3362 %}
3363
3364 // synchronized read/update encodings
3365
3366 enc_class aarch64_enc_ldaxr(iRegL dst, memory8 mem) %{
3367 C2_MacroAssembler _masm(&cbuf);
3368 Register dst_reg = as_Register($dst$$reg);
3369 Register base = as_Register($mem$$base);
3370 int index = $mem$$index;
3371 int scale = $mem$$scale;
3372 int disp = $mem$$disp;
3373 if (index == -1) {
3374 if (disp != 0) {
3375 __ lea(rscratch1, Address(base, disp));
3376 __ ldaxr(dst_reg, rscratch1);
3377 } else {
3378 // TODO
3379 // should we ever get anything other than this case?
3380 __ ldaxr(dst_reg, base);
3381 }
3382 } else {
3383 Register index_reg = as_Register(index);
3384 if (disp == 0) {
3385 __ lea(rscratch1, Address(base, index_reg, Address::lsl(scale)));
3386 __ ldaxr(dst_reg, rscratch1);
3387 } else {
3388 __ lea(rscratch1, Address(base, disp));
3389 __ lea(rscratch1, Address(rscratch1, index_reg, Address::lsl(scale)));
3390 __ ldaxr(dst_reg, rscratch1);
3391 }
3392 }
3393 %}
3394
3395 enc_class aarch64_enc_stlxr(iRegLNoSp src, memory8 mem) %{
3396 C2_MacroAssembler _masm(&cbuf);
3397 Register src_reg = as_Register($src$$reg);
3398 Register base = as_Register($mem$$base);
3399 int index = $mem$$index;
3400 int scale = $mem$$scale;
3401 int disp = $mem$$disp;
3402 if (index == -1) {
3403 if (disp != 0) {
3404 __ lea(rscratch2, Address(base, disp));
3405 __ stlxr(rscratch1, src_reg, rscratch2);
3406 } else {
3407 // TODO
3408 // should we ever get anything other than this case?
3409 __ stlxr(rscratch1, src_reg, base);
3410 }
3411 } else {
3412 Register index_reg = as_Register(index);
3413 if (disp == 0) {
3414 __ lea(rscratch2, Address(base, index_reg, Address::lsl(scale)));
3415 __ stlxr(rscratch1, src_reg, rscratch2);
3416 } else {
3417 __ lea(rscratch2, Address(base, disp));
3418 __ lea(rscratch2, Address(rscratch2, index_reg, Address::lsl(scale)));
3419 __ stlxr(rscratch1, src_reg, rscratch2);
3420 }
3421 }
3422 __ cmpw(rscratch1, zr);
3423 %}
3424
3425 enc_class aarch64_enc_cmpxchg(memory mem, iRegLNoSp oldval, iRegLNoSp newval) %{
3426 C2_MacroAssembler _masm(&cbuf);
3427 guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
3428 __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
3429 Assembler::xword, /*acquire*/ false, /*release*/ true,
3430 /*weak*/ false, noreg);
3431 %}
3432
3433 enc_class aarch64_enc_cmpxchgw(memory mem, iRegINoSp oldval, iRegINoSp newval) %{
3434 C2_MacroAssembler _masm(&cbuf);
3435 guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
3436 __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
3437 Assembler::word, /*acquire*/ false, /*release*/ true,
3438 /*weak*/ false, noreg);
3439 %}
3440
3441 enc_class aarch64_enc_cmpxchgs(memory mem, iRegINoSp oldval, iRegINoSp newval) %{
3442 C2_MacroAssembler _masm(&cbuf);
3443 guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
3444 __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
3445 Assembler::halfword, /*acquire*/ false, /*release*/ true,
3446 /*weak*/ false, noreg);
3447 %}
3448
3449 enc_class aarch64_enc_cmpxchgb(memory mem, iRegINoSp oldval, iRegINoSp newval) %{
3450 C2_MacroAssembler _masm(&cbuf);
3451 guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
3452 __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
3453 Assembler::byte, /*acquire*/ false, /*release*/ true,
3454 /*weak*/ false, noreg);
3455 %}
3456
3457
3458 // The only difference between aarch64_enc_cmpxchg and
3459 // aarch64_enc_cmpxchg_acq is that we use load-acquire in the
3460 // CompareAndSwap sequence to serve as a barrier on acquiring a
3461 // lock.
3462 enc_class aarch64_enc_cmpxchg_acq(memory mem, iRegLNoSp oldval, iRegLNoSp newval) %{
3463 C2_MacroAssembler _masm(&cbuf);
3464 guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
3465 __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
3466 Assembler::xword, /*acquire*/ true, /*release*/ true,
3467 /*weak*/ false, noreg);
3468 %}
3469
3470 enc_class aarch64_enc_cmpxchgw_acq(memory mem, iRegINoSp oldval, iRegINoSp newval) %{
3471 C2_MacroAssembler _masm(&cbuf);
3472 guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
3473 __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
3474 Assembler::word, /*acquire*/ true, /*release*/ true,
3475 /*weak*/ false, noreg);
3476 %}
3477
3478 enc_class aarch64_enc_cmpxchgs_acq(memory mem, iRegINoSp oldval, iRegINoSp newval) %{
3479 C2_MacroAssembler _masm(&cbuf);
3480 guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
3481 __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
3482 Assembler::halfword, /*acquire*/ true, /*release*/ true,
3483 /*weak*/ false, noreg);
3484 %}
3485
3486 enc_class aarch64_enc_cmpxchgb_acq(memory mem, iRegINoSp oldval, iRegINoSp newval) %{
3487 C2_MacroAssembler _masm(&cbuf);
3488 guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
3489 __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
3490 Assembler::byte, /*acquire*/ true, /*release*/ true,
3491 /*weak*/ false, noreg);
3492 %}
3493
3494 // auxiliary used for CompareAndSwapX to set result register
3495 enc_class aarch64_enc_cset_eq(iRegINoSp res) %{
3496 C2_MacroAssembler _masm(&cbuf);
3497 Register res_reg = as_Register($res$$reg);
3498 __ cset(res_reg, Assembler::EQ);
3499 %}
3500
3501 // prefetch encodings
3502
3503 enc_class aarch64_enc_prefetchw(memory mem) %{
3504 C2_MacroAssembler _masm(&cbuf);
3505 Register base = as_Register($mem$$base);
3506 int index = $mem$$index;
3507 int scale = $mem$$scale;
3508 int disp = $mem$$disp;
3509 if (index == -1) {
3510 __ prfm(Address(base, disp), PSTL1KEEP);
3511 } else {
3512 Register index_reg = as_Register(index);
3513 if (disp == 0) {
3514 __ prfm(Address(base, index_reg, Address::lsl(scale)), PSTL1KEEP);
3515 } else {
3516 __ lea(rscratch1, Address(base, disp));
3517 __ prfm(Address(rscratch1, index_reg, Address::lsl(scale)), PSTL1KEEP);
3518 }
3519 }
3520 %}
3521
3522 /// mov envcodings
3523
3524 enc_class aarch64_enc_movw_imm(iRegI dst, immI src) %{
3525 C2_MacroAssembler _masm(&cbuf);
3526 uint32_t con = (uint32_t)$src$$constant;
3527 Register dst_reg = as_Register($dst$$reg);
3528 if (con == 0) {
3529 __ movw(dst_reg, zr);
3530 } else {
3531 __ movw(dst_reg, con);
3532 }
3533 %}
3534
3535 enc_class aarch64_enc_mov_imm(iRegL dst, immL src) %{
3536 C2_MacroAssembler _masm(&cbuf);
3537 Register dst_reg = as_Register($dst$$reg);
3538 uint64_t con = (uint64_t)$src$$constant;
3539 if (con == 0) {
3540 __ mov(dst_reg, zr);
3541 } else {
3542 __ mov(dst_reg, con);
3543 }
3544 %}
3545
3546 enc_class aarch64_enc_mov_p(iRegP dst, immP src) %{
3547 C2_MacroAssembler _masm(&cbuf);
3548 Register dst_reg = as_Register($dst$$reg);
3549 address con = (address)$src$$constant;
3550 if (con == NULL || con == (address)1) {
3551 ShouldNotReachHere();
3552 } else {
3553 relocInfo::relocType rtype = $src->constant_reloc();
3554 if (rtype == relocInfo::oop_type) {
3555 __ movoop(dst_reg, (jobject)con, /*immediate*/true);
3556 } else if (rtype == relocInfo::metadata_type) {
3557 __ mov_metadata(dst_reg, (Metadata*)con);
3558 } else {
3559 assert(rtype == relocInfo::none, "unexpected reloc type");
3560 if (con < (address)(uintptr_t)os::vm_page_size()) {
3561 __ mov(dst_reg, con);
3562 } else {
3563 uint64_t offset;
3564 __ adrp(dst_reg, con, offset);
3565 __ add(dst_reg, dst_reg, offset);
3566 }
3567 }
3568 }
3569 %}
3570
3571 enc_class aarch64_enc_mov_p0(iRegP dst, immP0 src) %{
3572 C2_MacroAssembler _masm(&cbuf);
3573 Register dst_reg = as_Register($dst$$reg);
3574 __ mov(dst_reg, zr);
3575 %}
3576
3577 enc_class aarch64_enc_mov_p1(iRegP dst, immP_1 src) %{
3578 C2_MacroAssembler _masm(&cbuf);
3579 Register dst_reg = as_Register($dst$$reg);
3580 __ mov(dst_reg, (uint64_t)1);
3581 %}
3582
3583 enc_class aarch64_enc_mov_byte_map_base(iRegP dst, immByteMapBase src) %{
3584 C2_MacroAssembler _masm(&cbuf);
3585 __ load_byte_map_base($dst$$Register);
3586 %}
3587
3588 enc_class aarch64_enc_mov_n(iRegN dst, immN src) %{
3589 C2_MacroAssembler _masm(&cbuf);
3590 Register dst_reg = as_Register($dst$$reg);
3591 address con = (address)$src$$constant;
3592 if (con == NULL) {
3593 ShouldNotReachHere();
3594 } else {
3595 relocInfo::relocType rtype = $src->constant_reloc();
3596 assert(rtype == relocInfo::oop_type, "unexpected reloc type");
3597 __ set_narrow_oop(dst_reg, (jobject)con);
3598 }
3599 %}
3600
3601 enc_class aarch64_enc_mov_n0(iRegN dst, immN0 src) %{
3602 C2_MacroAssembler _masm(&cbuf);
3603 Register dst_reg = as_Register($dst$$reg);
3604 __ mov(dst_reg, zr);
3605 %}
3606
3607 enc_class aarch64_enc_mov_nk(iRegN dst, immNKlass src) %{
3608 C2_MacroAssembler _masm(&cbuf);
3609 Register dst_reg = as_Register($dst$$reg);
3610 address con = (address)$src$$constant;
3611 if (con == NULL) {
3612 ShouldNotReachHere();
3613 } else {
3614 relocInfo::relocType rtype = $src->constant_reloc();
3615 assert(rtype == relocInfo::metadata_type, "unexpected reloc type");
3616 __ set_narrow_klass(dst_reg, (Klass *)con);
3617 }
3618 %}
3619
3620 // arithmetic encodings
3621
3622 enc_class aarch64_enc_addsubw_imm(iRegI dst, iRegI src1, immIAddSub src2) %{
3623 C2_MacroAssembler _masm(&cbuf);
3624 Register dst_reg = as_Register($dst$$reg);
3625 Register src_reg = as_Register($src1$$reg);
3626 int32_t con = (int32_t)$src2$$constant;
3627 // add has primary == 0, subtract has primary == 1
3628 if ($primary) { con = -con; }
3629 if (con < 0) {
3630 __ subw(dst_reg, src_reg, -con);
3631 } else {
3632 __ addw(dst_reg, src_reg, con);
3633 }
3634 %}
3635
3636 enc_class aarch64_enc_addsub_imm(iRegL dst, iRegL src1, immLAddSub src2) %{
3637 C2_MacroAssembler _masm(&cbuf);
3638 Register dst_reg = as_Register($dst$$reg);
3639 Register src_reg = as_Register($src1$$reg);
3640 int32_t con = (int32_t)$src2$$constant;
3641 // add has primary == 0, subtract has primary == 1
3642 if ($primary) { con = -con; }
3643 if (con < 0) {
3644 __ sub(dst_reg, src_reg, -con);
3645 } else {
3646 __ add(dst_reg, src_reg, con);
3647 }
3648 %}
3649
3650 enc_class aarch64_enc_divw(iRegI dst, iRegI src1, iRegI src2) %{
3651 C2_MacroAssembler _masm(&cbuf);
3652 Register dst_reg = as_Register($dst$$reg);
3653 Register src1_reg = as_Register($src1$$reg);
3654 Register src2_reg = as_Register($src2$$reg);
3655 __ corrected_idivl(dst_reg, src1_reg, src2_reg, false, rscratch1);
3656 %}
3657
3658 enc_class aarch64_enc_div(iRegI dst, iRegI src1, iRegI src2) %{
3659 C2_MacroAssembler _masm(&cbuf);
3660 Register dst_reg = as_Register($dst$$reg);
3661 Register src1_reg = as_Register($src1$$reg);
3662 Register src2_reg = as_Register($src2$$reg);
3663 __ corrected_idivq(dst_reg, src1_reg, src2_reg, false, rscratch1);
3664 %}
3665
3666 enc_class aarch64_enc_modw(iRegI dst, iRegI src1, iRegI src2) %{
3667 C2_MacroAssembler _masm(&cbuf);
3668 Register dst_reg = as_Register($dst$$reg);
3669 Register src1_reg = as_Register($src1$$reg);
3670 Register src2_reg = as_Register($src2$$reg);
3671 __ corrected_idivl(dst_reg, src1_reg, src2_reg, true, rscratch1);
3672 %}
3673
3674 enc_class aarch64_enc_mod(iRegI dst, iRegI src1, iRegI src2) %{
3675 C2_MacroAssembler _masm(&cbuf);
3676 Register dst_reg = as_Register($dst$$reg);
3677 Register src1_reg = as_Register($src1$$reg);
3678 Register src2_reg = as_Register($src2$$reg);
3679 __ corrected_idivq(dst_reg, src1_reg, src2_reg, true, rscratch1);
3680 %}
3681
3682 // compare instruction encodings
3683
3684 enc_class aarch64_enc_cmpw(iRegI src1, iRegI src2) %{
3685 C2_MacroAssembler _masm(&cbuf);
3686 Register reg1 = as_Register($src1$$reg);
3687 Register reg2 = as_Register($src2$$reg);
3688 __ cmpw(reg1, reg2);
3689 %}
3690
3691 enc_class aarch64_enc_cmpw_imm_addsub(iRegI src1, immIAddSub src2) %{
3692 C2_MacroAssembler _masm(&cbuf);
3693 Register reg = as_Register($src1$$reg);
3694 int32_t val = $src2$$constant;
3695 if (val >= 0) {
3696 __ subsw(zr, reg, val);
3697 } else {
3698 __ addsw(zr, reg, -val);
3699 }
3700 %}
3701
3702 enc_class aarch64_enc_cmpw_imm(iRegI src1, immI src2) %{
3703 C2_MacroAssembler _masm(&cbuf);
3704 Register reg1 = as_Register($src1$$reg);
3705 uint32_t val = (uint32_t)$src2$$constant;
3706 __ movw(rscratch1, val);
3707 __ cmpw(reg1, rscratch1);
3708 %}
3709
3710 enc_class aarch64_enc_cmp(iRegL src1, iRegL src2) %{
3711 C2_MacroAssembler _masm(&cbuf);
3712 Register reg1 = as_Register($src1$$reg);
3713 Register reg2 = as_Register($src2$$reg);
3714 __ cmp(reg1, reg2);
3715 %}
3716
3717 enc_class aarch64_enc_cmp_imm_addsub(iRegL src1, immL12 src2) %{
3718 C2_MacroAssembler _masm(&cbuf);
3719 Register reg = as_Register($src1$$reg);
3720 int64_t val = $src2$$constant;
3721 if (val >= 0) {
3722 __ subs(zr, reg, val);
3723 } else if (val != -val) {
3724 __ adds(zr, reg, -val);
3725 } else {
3726 // aargh, Long.MIN_VALUE is a special case
3727 __ orr(rscratch1, zr, (uint64_t)val);
3728 __ subs(zr, reg, rscratch1);
3729 }
3730 %}
3731
3732 enc_class aarch64_enc_cmp_imm(iRegL src1, immL src2) %{
3733 C2_MacroAssembler _masm(&cbuf);
3734 Register reg1 = as_Register($src1$$reg);
3735 uint64_t val = (uint64_t)$src2$$constant;
3736 __ mov(rscratch1, val);
3737 __ cmp(reg1, rscratch1);
3738 %}
3739
3740 enc_class aarch64_enc_cmpp(iRegP src1, iRegP src2) %{
3741 C2_MacroAssembler _masm(&cbuf);
3742 Register reg1 = as_Register($src1$$reg);
3743 Register reg2 = as_Register($src2$$reg);
3744 __ cmp(reg1, reg2);
3745 %}
3746
3747 enc_class aarch64_enc_cmpn(iRegN src1, iRegN src2) %{
3748 C2_MacroAssembler _masm(&cbuf);
3749 Register reg1 = as_Register($src1$$reg);
3750 Register reg2 = as_Register($src2$$reg);
3751 __ cmpw(reg1, reg2);
3752 %}
3753
3754 enc_class aarch64_enc_testp(iRegP src) %{
3755 C2_MacroAssembler _masm(&cbuf);
3756 Register reg = as_Register($src$$reg);
3757 __ cmp(reg, zr);
3758 %}
3759
3760 enc_class aarch64_enc_testn(iRegN src) %{
3761 C2_MacroAssembler _masm(&cbuf);
3762 Register reg = as_Register($src$$reg);
3763 __ cmpw(reg, zr);
3764 %}
3765
3766 enc_class aarch64_enc_b(label lbl) %{
3767 C2_MacroAssembler _masm(&cbuf);
3768 Label *L = $lbl$$label;
3769 __ b(*L);
3770 %}
3771
3772 enc_class aarch64_enc_br_con(cmpOp cmp, label lbl) %{
3773 C2_MacroAssembler _masm(&cbuf);
3774 Label *L = $lbl$$label;
3775 __ br ((Assembler::Condition)$cmp$$cmpcode, *L);
3776 %}
3777
3778 enc_class aarch64_enc_br_conU(cmpOpU cmp, label lbl) %{
3779 C2_MacroAssembler _masm(&cbuf);
3780 Label *L = $lbl$$label;
3781 __ br ((Assembler::Condition)$cmp$$cmpcode, *L);
3782 %}
3783
3784 enc_class aarch64_enc_partial_subtype_check(iRegP sub, iRegP super, iRegP temp, iRegP result)
3785 %{
3786 Register sub_reg = as_Register($sub$$reg);
3787 Register super_reg = as_Register($super$$reg);
3788 Register temp_reg = as_Register($temp$$reg);
3789 Register result_reg = as_Register($result$$reg);
3790
3791 Label miss;
3792 C2_MacroAssembler _masm(&cbuf);
3793 __ check_klass_subtype_slow_path(sub_reg, super_reg, temp_reg, result_reg,
3794 NULL, &miss,
3795 /*set_cond_codes:*/ true);
3796 if ($primary) {
3797 __ mov(result_reg, zr);
3798 }
3799 __ bind(miss);
3800 %}
3801
3802 enc_class aarch64_enc_java_static_call(method meth) %{
3803 C2_MacroAssembler _masm(&cbuf);
3804
3805 address addr = (address)$meth$$method;
3806 address call;
3807 if (!_method) {
3808 // A call to a runtime wrapper, e.g. new, new_typeArray_Java, uncommon_trap.
3809 call = __ trampoline_call(Address(addr, relocInfo::runtime_call_type), &cbuf);
3810 if (call == NULL) {
3811 ciEnv::current()->record_failure("CodeCache is full");
3812 return;
3813 }
3814 } else {
3815 int method_index = resolved_method_index(cbuf);
3816 RelocationHolder rspec = _optimized_virtual ? opt_virtual_call_Relocation::spec(method_index)
3817 : static_call_Relocation::spec(method_index);
3818 call = __ trampoline_call(Address(addr, rspec), &cbuf);
3819 if (call == NULL) {
3820 ciEnv::current()->record_failure("CodeCache is full");
3821 return;
3822 }
3823 // Emit stub for static call
3824 address stub = CompiledStaticCall::emit_to_interp_stub(cbuf);
3825 if (stub == NULL) {
3826 ciEnv::current()->record_failure("CodeCache is full");
3827 return;
3828 }
3829 }
3830
3831 // Only non uncommon_trap calls need to reinitialize ptrue.
3832 if (Compile::current()->max_vector_size() > 0 && uncommon_trap_request() == 0) {
3833 __ reinitialize_ptrue();
3834 }
3835 %}
3836
3837 enc_class aarch64_enc_java_dynamic_call(method meth) %{
3838 C2_MacroAssembler _masm(&cbuf);
3839 int method_index = resolved_method_index(cbuf);
3840 address call = __ ic_call((address)$meth$$method, method_index);
3841 if (call == NULL) {
3842 ciEnv::current()->record_failure("CodeCache is full");
3843 return;
3844 } else if (Compile::current()->max_vector_size() > 0) {
3845 __ reinitialize_ptrue();
3846 }
3847 %}
3848
3849 enc_class aarch64_enc_call_epilog() %{
3850 C2_MacroAssembler _masm(&cbuf);
3851 if (VerifyStackAtCalls) {
3852 // Check that stack depth is unchanged: find majik cookie on stack
3853 __ call_Unimplemented();
3854 }
3855 %}
3856
3857 enc_class aarch64_enc_java_to_runtime(method meth) %{
3858 C2_MacroAssembler _masm(&cbuf);
3859
3860 // some calls to generated routines (arraycopy code) are scheduled
3861 // by C2 as runtime calls. if so we can call them using a br (they
3862 // will be in a reachable segment) otherwise we have to use a blr
3863 // which loads the absolute address into a register.
3864 address entry = (address)$meth$$method;
3865 CodeBlob *cb = CodeCache::find_blob(entry);
3866 if (cb) {
3867 address call = __ trampoline_call(Address(entry, relocInfo::runtime_call_type));
3868 if (call == NULL) {
3869 ciEnv::current()->record_failure("CodeCache is full");
3870 return;
3871 }
3872 } else {
3873 Label retaddr;
3874 __ adr(rscratch2, retaddr);
3875 __ lea(rscratch1, RuntimeAddress(entry));
3876 // Leave a breadcrumb for JavaFrameAnchor::capture_last_Java_pc()
3877 __ stp(zr, rscratch2, Address(__ pre(sp, -2 * wordSize)));
3878 __ blr(rscratch1);
3879 __ bind(retaddr);
3880 __ add(sp, sp, 2 * wordSize);
3881 }
3882 if (Compile::current()->max_vector_size() > 0) {
3883 __ reinitialize_ptrue();
3884 }
3885 %}
3886
3887 enc_class aarch64_enc_rethrow() %{
3888 C2_MacroAssembler _masm(&cbuf);
3889 __ far_jump(RuntimeAddress(OptoRuntime::rethrow_stub()));
3890 %}
3891
3892 enc_class aarch64_enc_ret() %{
3893 C2_MacroAssembler _masm(&cbuf);
3894 #ifdef ASSERT
3895 if (Compile::current()->max_vector_size() > 0) {
3896 __ verify_ptrue();
3897 }
3898 #endif
3899 __ ret(lr);
3900 %}
3901
3902 enc_class aarch64_enc_tail_call(iRegP jump_target) %{
3903 C2_MacroAssembler _masm(&cbuf);
3904 Register target_reg = as_Register($jump_target$$reg);
3905 __ br(target_reg);
3906 %}
3907
3908 enc_class aarch64_enc_tail_jmp(iRegP jump_target) %{
3909 C2_MacroAssembler _masm(&cbuf);
3910 Register target_reg = as_Register($jump_target$$reg);
3911 // exception oop should be in r0
3912 // ret addr has been popped into lr
3913 // callee expects it in r3
3914 __ mov(r3, lr);
3915 __ br(target_reg);
3916 %}
3917
3918 enc_class aarch64_enc_fast_lock(iRegP object, iRegP box, iRegP tmp, iRegP tmp2) %{
3919 C2_MacroAssembler _masm(&cbuf);
3920 Register oop = as_Register($object$$reg);
3921 Register box = as_Register($box$$reg);
3922 Register disp_hdr = as_Register($tmp$$reg);
3923 Register tmp = as_Register($tmp2$$reg);
3924 Label cont;
3925 Label object_has_monitor;
3926 Label cas_failed;
3927
3928 assert_different_registers(oop, box, tmp, disp_hdr);
3929
3930 // Load markWord from object into displaced_header.
3931 __ ldr(disp_hdr, Address(oop, oopDesc::mark_offset_in_bytes()));
3932
3933 if (DiagnoseSyncOnValueBasedClasses != 0) {
3934 __ load_klass(tmp, oop);
3935 __ ldrw(tmp, Address(tmp, Klass::access_flags_offset()));
3936 __ tstw(tmp, JVM_ACC_IS_VALUE_BASED_CLASS);
3937 __ br(Assembler::NE, cont);
3938 }
3939
3940 // Check for existing monitor
3941 __ tbnz(disp_hdr, exact_log2(markWord::monitor_value), object_has_monitor);
3942
3943 if (!UseHeavyMonitors) {
3944 // Set tmp to be (markWord of object | UNLOCK_VALUE).
3945 __ orr(tmp, disp_hdr, markWord::unlocked_value);
3946
3947 // Initialize the box. (Must happen before we update the object mark!)
3948 __ str(tmp, Address(box, BasicLock::displaced_header_offset_in_bytes()));
3949
3950 // Compare object markWord with an unlocked value (tmp) and if
3951 // equal exchange the stack address of our box with object markWord.
3952 // On failure disp_hdr contains the possibly locked markWord.
3953 __ cmpxchg(oop, tmp, box, Assembler::xword, /*acquire*/ true,
3954 /*release*/ true, /*weak*/ false, disp_hdr);
3955 __ br(Assembler::EQ, cont);
3956
3957 assert(oopDesc::mark_offset_in_bytes() == 0, "offset of _mark is not 0");
3958
3959 // If the compare-and-exchange succeeded, then we found an unlocked
3960 // object, will have now locked it will continue at label cont
3961
3962 __ bind(cas_failed);
3963 // We did not see an unlocked object so try the fast recursive case.
3964
3965 // Check if the owner is self by comparing the value in the
3966 // markWord of object (disp_hdr) with the stack pointer.
3967 __ mov(rscratch1, sp);
3968 __ sub(disp_hdr, disp_hdr, rscratch1);
3969 __ mov(tmp, (address) (~(os::vm_page_size()-1) | markWord::lock_mask_in_place));
3970 // If condition is true we are cont and hence we can store 0 as the
3971 // displaced header in the box, which indicates that it is a recursive lock.
3972 __ ands(tmp/*==0?*/, disp_hdr, tmp); // Sets flags for result
3973 __ str(tmp/*==0, perhaps*/, Address(box, BasicLock::displaced_header_offset_in_bytes()));
3974 } else {
3975 __ tst(oop, oop); // Set NE to indicate 'failure' -> take slow-path. We know that oop != 0.
3976 }
3977 __ b(cont);
3978
3979 // Handle existing monitor.
3980 __ bind(object_has_monitor);
3981
3982 // The object's monitor m is unlocked iff m->owner == NULL,
3983 // otherwise m->owner may contain a thread or a stack address.
3984 //
3985 // Try to CAS m->owner from NULL to current thread.
3986 __ add(tmp, disp_hdr, (ObjectMonitor::owner_offset_in_bytes()-markWord::monitor_value));
3987 __ cmpxchg(tmp, zr, rthread, Assembler::xword, /*acquire*/ true,
3988 /*release*/ true, /*weak*/ false, rscratch1); // Sets flags for result
3989
3990 // Store a non-null value into the box to avoid looking like a re-entrant
3991 // lock. The fast-path monitor unlock code checks for
3992 // markWord::monitor_value so use markWord::unused_mark which has the
3993 // relevant bit set, and also matches ObjectSynchronizer::enter.
3994 __ mov(tmp, (address)markWord::unused_mark().value());
3995 __ str(tmp, Address(box, BasicLock::displaced_header_offset_in_bytes()));
3996
3997 __ br(Assembler::EQ, cont); // CAS success means locking succeeded
3998
3999 __ cmp(rscratch1, rthread);
4000 __ br(Assembler::NE, cont); // Check for recursive locking
4001
4002 // Recursive lock case
4003 __ increment(Address(disp_hdr, ObjectMonitor::recursions_offset_in_bytes() - markWord::monitor_value), 1);
4004 // flag == EQ still from the cmp above, checking if this is a reentrant lock
4005
4006 __ bind(cont);
4007 // flag == EQ indicates success
4008 // flag == NE indicates failure
4009 %}
4010
4011 enc_class aarch64_enc_fast_unlock(iRegP object, iRegP box, iRegP tmp, iRegP tmp2) %{
4012 C2_MacroAssembler _masm(&cbuf);
4013 Register oop = as_Register($object$$reg);
4014 Register box = as_Register($box$$reg);
4015 Register disp_hdr = as_Register($tmp$$reg);
4016 Register tmp = as_Register($tmp2$$reg);
4017 Label cont;
4018 Label object_has_monitor;
4019
4020 assert_different_registers(oop, box, tmp, disp_hdr);
4021
4022 if (!UseHeavyMonitors) {
4023 // Find the lock address and load the displaced header from the stack.
4024 __ ldr(disp_hdr, Address(box, BasicLock::displaced_header_offset_in_bytes()));
4025
4026 // If the displaced header is 0, we have a recursive unlock.
4027 __ cmp(disp_hdr, zr);
4028 __ br(Assembler::EQ, cont);
4029 }
4030
4031 // Handle existing monitor.
4032 __ ldr(tmp, Address(oop, oopDesc::mark_offset_in_bytes()));
4033 __ tbnz(disp_hdr, exact_log2(markWord::monitor_value), object_has_monitor);
4034
4035 if (!UseHeavyMonitors) {
4036 // Check if it is still a light weight lock, this is is true if we
4037 // see the stack address of the basicLock in the markWord of the
4038 // object.
4039
4040 __ cmpxchg(oop, box, disp_hdr, Assembler::xword, /*acquire*/ false,
4041 /*release*/ true, /*weak*/ false, tmp);
4042 } else {
4043 __ tst(oop, oop); // Set NE to indicate 'failure' -> take slow-path. We know that oop != 0.
4044 }
4045 __ b(cont);
4046
4047 assert(oopDesc::mark_offset_in_bytes() == 0, "offset of _mark is not 0");
4048
4049 // Handle existing monitor.
4050 __ bind(object_has_monitor);
4051 STATIC_ASSERT(markWord::monitor_value <= INT_MAX);
4052 __ add(tmp, tmp, -(int)markWord::monitor_value); // monitor
4053 __ ldr(disp_hdr, Address(tmp, ObjectMonitor::recursions_offset_in_bytes()));
4054
4055 Label notRecursive;
4056 __ cbz(disp_hdr, notRecursive);
4057
4058 // Recursive lock
4059 __ sub(disp_hdr, disp_hdr, 1u);
4060 __ str(disp_hdr, Address(tmp, ObjectMonitor::recursions_offset_in_bytes()));
4061 __ cmp(disp_hdr, disp_hdr); // Sets flags for result
4062 __ b(cont);
4063
4064 __ bind(notRecursive);
4065 __ ldr(rscratch1, Address(tmp, ObjectMonitor::EntryList_offset_in_bytes()));
4066 __ ldr(disp_hdr, Address(tmp, ObjectMonitor::cxq_offset_in_bytes()));
4067 __ orr(rscratch1, rscratch1, disp_hdr); // Will be 0 if both are 0.
4068 __ cmp(rscratch1, zr); // Sets flags for result
4069 __ cbnz(rscratch1, cont);
4070 // need a release store here
4071 __ lea(tmp, Address(tmp, ObjectMonitor::owner_offset_in_bytes()));
4072 __ stlr(zr, tmp); // set unowned
4073
4074 __ bind(cont);
4075 // flag == EQ indicates success
4076 // flag == NE indicates failure
4077 %}
4078
4079 %}
4080
4081 //----------FRAME--------------------------------------------------------------
4082 // Definition of frame structure and management information.
4083 //
4084 // S T A C K L A Y O U T Allocators stack-slot number
4085 // | (to get allocators register number
4086 // G Owned by | | v add OptoReg::stack0())
4087 // r CALLER | |
4088 // o | +--------+ pad to even-align allocators stack-slot
4089 // w V | pad0 | numbers; owned by CALLER
4090 // t -----------+--------+----> Matcher::_in_arg_limit, unaligned
4091 // h ^ | in | 5
4092 // | | args | 4 Holes in incoming args owned by SELF
4093 // | | | | 3
4094 // | | +--------+
4095 // V | | old out| Empty on Intel, window on Sparc
4096 // | old |preserve| Must be even aligned.
4097 // | SP-+--------+----> Matcher::_old_SP, even aligned
4098 // | | in | 3 area for Intel ret address
4099 // Owned by |preserve| Empty on Sparc.
4100 // SELF +--------+
4101 // | | pad2 | 2 pad to align old SP
4102 // | +--------+ 1
4103 // | | locks | 0
4104 // | +--------+----> OptoReg::stack0(), even aligned
4105 // | | pad1 | 11 pad to align new SP
4106 // | +--------+
4107 // | | | 10
4108 // | | spills | 9 spills
4109 // V | | 8 (pad0 slot for callee)
4110 // -----------+--------+----> Matcher::_out_arg_limit, unaligned
4111 // ^ | out | 7
4112 // | | args | 6 Holes in outgoing args owned by CALLEE
4113 // Owned by +--------+
4114 // CALLEE | new out| 6 Empty on Intel, window on Sparc
4115 // | new |preserve| Must be even-aligned.
4116 // | SP-+--------+----> Matcher::_new_SP, even aligned
4117 // | | |
4118 //
4119 // Note 1: Only region 8-11 is determined by the allocator. Region 0-5 is
4120 // known from SELF's arguments and the Java calling convention.
4121 // Region 6-7 is determined per call site.
4122 // Note 2: If the calling convention leaves holes in the incoming argument
4123 // area, those holes are owned by SELF. Holes in the outgoing area
4124 // are owned by the CALLEE. Holes should not be necessary in the
4125 // incoming area, as the Java calling convention is completely under
4126 // the control of the AD file. Doubles can be sorted and packed to
4127 // avoid holes. Holes in the outgoing arguments may be necessary for
4128 // varargs C calling conventions.
4129 // Note 3: Region 0-3 is even aligned, with pad2 as needed. Region 3-5 is
4130 // even aligned with pad0 as needed.
4131 // Region 6 is even aligned. Region 6-7 is NOT even aligned;
4132 // (the latter is true on Intel but is it false on AArch64?)
4133 // region 6-11 is even aligned; it may be padded out more so that
4134 // the region from SP to FP meets the minimum stack alignment.
4135 // Note 4: For I2C adapters, the incoming FP may not meet the minimum stack
4136 // alignment. Region 11, pad1, may be dynamically extended so that
4137 // SP meets the minimum alignment.
4138
4139 frame %{
4140 // These three registers define part of the calling convention
4141 // between compiled code and the interpreter.
4142
4143 // Inline Cache Register or Method for I2C.
4144 inline_cache_reg(R12);
4145
4146 // Number of stack slots consumed by locking an object
4147 sync_stack_slots(2);
4148
4149 // Compiled code's Frame Pointer
4150 frame_pointer(R31);
4151
4152 // Interpreter stores its frame pointer in a register which is
4153 // stored to the stack by I2CAdaptors.
4154 // I2CAdaptors convert from interpreted java to compiled java.
4155 interpreter_frame_pointer(R29);
4156
4157 // Stack alignment requirement
4158 stack_alignment(StackAlignmentInBytes); // Alignment size in bytes (128-bit -> 16 bytes)
4159
4160 // Number of outgoing stack slots killed above the out_preserve_stack_slots
4161 // for calls to C. Supports the var-args backing area for register parms.
4162 varargs_C_out_slots_killed(frame::arg_reg_save_area_bytes/BytesPerInt);
4163
4164 // The after-PROLOG location of the return address. Location of
4165 // return address specifies a type (REG or STACK) and a number
4166 // representing the register number (i.e. - use a register name) or
4167 // stack slot.
4168 // Ret Addr is on stack in slot 0 if no locks or verification or alignment.
4169 // Otherwise, it is above the locks and verification slot and alignment word
4170 // TODO this may well be correct but need to check why that - 2 is there
4171 // ppc port uses 0 but we definitely need to allow for fixed_slots
4172 // which folds in the space used for monitors
4173 return_addr(STACK - 2 +
4174 align_up((Compile::current()->in_preserve_stack_slots() +
4175 Compile::current()->fixed_slots()),
4176 stack_alignment_in_slots()));
4177
4178 // Location of compiled Java return values. Same as C for now.
4179 return_value
4180 %{
4181 // TODO do we allow ideal_reg == Op_RegN???
4182 assert(ideal_reg >= Op_RegI && ideal_reg <= Op_RegL,
4183 "only return normal values");
4184
4185 static const int lo[Op_RegL + 1] = { // enum name
4186 0, // Op_Node
4187 0, // Op_Set
4188 R0_num, // Op_RegN
4189 R0_num, // Op_RegI
4190 R0_num, // Op_RegP
4191 V0_num, // Op_RegF
4192 V0_num, // Op_RegD
4193 R0_num // Op_RegL
4194 };
4195
4196 static const int hi[Op_RegL + 1] = { // enum name
4197 0, // Op_Node
4198 0, // Op_Set
4199 OptoReg::Bad, // Op_RegN
4200 OptoReg::Bad, // Op_RegI
4201 R0_H_num, // Op_RegP
4202 OptoReg::Bad, // Op_RegF
4203 V0_H_num, // Op_RegD
4204 R0_H_num // Op_RegL
4205 };
4206
4207 return OptoRegPair(hi[ideal_reg], lo[ideal_reg]);
4208 %}
4209 %}
4210
4211 //----------ATTRIBUTES---------------------------------------------------------
4212 //----------Operand Attributes-------------------------------------------------
4213 op_attrib op_cost(1); // Required cost attribute
4214
4215 //----------Instruction Attributes---------------------------------------------
4216 ins_attrib ins_cost(INSN_COST); // Required cost attribute
4217 ins_attrib ins_size(32); // Required size attribute (in bits)
4218 ins_attrib ins_short_branch(0); // Required flag: is this instruction
4219 // a non-matching short branch variant
4220 // of some long branch?
4221 ins_attrib ins_alignment(4); // Required alignment attribute (must
4222 // be a power of 2) specifies the
4223 // alignment that some part of the
4224 // instruction (not necessarily the
4225 // start) requires. If > 1, a
4226 // compute_padding() function must be
4227 // provided for the instruction
4228
4229 //----------OPERANDS-----------------------------------------------------------
4230 // Operand definitions must precede instruction definitions for correct parsing
4231 // in the ADLC because operands constitute user defined types which are used in
4232 // instruction definitions.
4233
4234 //----------Simple Operands----------------------------------------------------
4235
4236 // Integer operands 32 bit
4237 // 32 bit immediate
4238 operand immI()
4239 %{
4240 match(ConI);
4241
4242 op_cost(0);
4243 format %{ %}
4244 interface(CONST_INTER);
4245 %}
4246
4247 // 32 bit zero
4248 operand immI0()
4249 %{
4250 predicate(n->get_int() == 0);
4251 match(ConI);
4252
4253 op_cost(0);
4254 format %{ %}
4255 interface(CONST_INTER);
4256 %}
4257
4258 // 32 bit unit increment
4259 operand immI_1()
4260 %{
4261 predicate(n->get_int() == 1);
4262 match(ConI);
4263
4264 op_cost(0);
4265 format %{ %}
4266 interface(CONST_INTER);
4267 %}
4268
4269 // 32 bit unit decrement
4270 operand immI_M1()
4271 %{
4272 predicate(n->get_int() == -1);
4273 match(ConI);
4274
4275 op_cost(0);
4276 format %{ %}
4277 interface(CONST_INTER);
4278 %}
4279
4280 // Shift values for add/sub extension shift
4281 operand immIExt()
4282 %{
4283 predicate(0 <= n->get_int() && (n->get_int() <= 4));
4284 match(ConI);
4285
4286 op_cost(0);
4287 format %{ %}
4288 interface(CONST_INTER);
4289 %}
4290
4291 operand immI_gt_1()
4292 %{
4293 predicate(n->get_int() > 1);
4294 match(ConI);
4295
4296 op_cost(0);
4297 format %{ %}
4298 interface(CONST_INTER);
4299 %}
4300
4301 operand immI_le_4()
4302 %{
4303 predicate(n->get_int() <= 4);
4304 match(ConI);
4305
4306 op_cost(0);
4307 format %{ %}
4308 interface(CONST_INTER);
4309 %}
4310
4311 operand immI_31()
4312 %{
4313 predicate(n->get_int() == 31);
4314 match(ConI);
4315
4316 op_cost(0);
4317 format %{ %}
4318 interface(CONST_INTER);
4319 %}
4320
4321 operand immI_2()
4322 %{
4323 predicate(n->get_int() == 2);
4324 match(ConI);
4325
4326 op_cost(0);
4327 format %{ %}
4328 interface(CONST_INTER);
4329 %}
4330
4331 operand immI_4()
4332 %{
4333 predicate(n->get_int() == 4);
4334 match(ConI);
4335
4336 op_cost(0);
4337 format %{ %}
4338 interface(CONST_INTER);
4339 %}
4340
4341 operand immI_8()
4342 %{
4343 predicate(n->get_int() == 8);
4344 match(ConI);
4345
4346 op_cost(0);
4347 format %{ %}
4348 interface(CONST_INTER);
4349 %}
4350
4351 operand immI_16()
4352 %{
4353 predicate(n->get_int() == 16);
4354 match(ConI);
4355
4356 op_cost(0);
4357 format %{ %}
4358 interface(CONST_INTER);
4359 %}
4360
4361 operand immI_24()
4362 %{
4363 predicate(n->get_int() == 24);
4364 match(ConI);
4365
4366 op_cost(0);
4367 format %{ %}
4368 interface(CONST_INTER);
4369 %}
4370
4371 operand immI_32()
4372 %{
4373 predicate(n->get_int() == 32);
4374 match(ConI);
4375
4376 op_cost(0);
4377 format %{ %}
4378 interface(CONST_INTER);
4379 %}
4380
4381 operand immI_48()
4382 %{
4383 predicate(n->get_int() == 48);
4384 match(ConI);
4385
4386 op_cost(0);
4387 format %{ %}
4388 interface(CONST_INTER);
4389 %}
4390
4391 operand immI_56()
4392 %{
4393 predicate(n->get_int() == 56);
4394 match(ConI);
4395
4396 op_cost(0);
4397 format %{ %}
4398 interface(CONST_INTER);
4399 %}
4400
4401 operand immI_63()
4402 %{
4403 predicate(n->get_int() == 63);
4404 match(ConI);
4405
4406 op_cost(0);
4407 format %{ %}
4408 interface(CONST_INTER);
4409 %}
4410
4411 operand immI_64()
4412 %{
4413 predicate(n->get_int() == 64);
4414 match(ConI);
4415
4416 op_cost(0);
4417 format %{ %}
4418 interface(CONST_INTER);
4419 %}
4420
4421 operand immI_255()
4422 %{
4423 predicate(n->get_int() == 255);
4424 match(ConI);
4425
4426 op_cost(0);
4427 format %{ %}
4428 interface(CONST_INTER);
4429 %}
4430
4431 operand immI_65535()
4432 %{
4433 predicate(n->get_int() == 65535);
4434 match(ConI);
4435
4436 op_cost(0);
4437 format %{ %}
4438 interface(CONST_INTER);
4439 %}
4440
4441 operand immL_255()
4442 %{
4443 predicate(n->get_long() == 255L);
4444 match(ConL);
4445
4446 op_cost(0);
4447 format %{ %}
4448 interface(CONST_INTER);
4449 %}
4450
4451 operand immL_65535()
4452 %{
4453 predicate(n->get_long() == 65535L);
4454 match(ConL);
4455
4456 op_cost(0);
4457 format %{ %}
4458 interface(CONST_INTER);
4459 %}
4460
4461 operand immL_4294967295()
4462 %{
4463 predicate(n->get_long() == 4294967295L);
4464 match(ConL);
4465
4466 op_cost(0);
4467 format %{ %}
4468 interface(CONST_INTER);
4469 %}
4470
4471 operand immL_bitmask()
4472 %{
4473 predicate((n->get_long() != 0)
4474 && ((n->get_long() & 0xc000000000000000l) == 0)
4475 && is_power_of_2(n->get_long() + 1));
4476 match(ConL);
4477
4478 op_cost(0);
4479 format %{ %}
4480 interface(CONST_INTER);
4481 %}
4482
4483 operand immI_bitmask()
4484 %{
4485 predicate((n->get_int() != 0)
4486 && ((n->get_int() & 0xc0000000) == 0)
4487 && is_power_of_2(n->get_int() + 1));
4488 match(ConI);
4489
4490 op_cost(0);
4491 format %{ %}
4492 interface(CONST_INTER);
4493 %}
4494
4495 operand immL_positive_bitmaskI()
4496 %{
4497 predicate((n->get_long() != 0)
4498 && ((julong)n->get_long() < 0x80000000ULL)
4499 && is_power_of_2(n->get_long() + 1));
4500 match(ConL);
4501
4502 op_cost(0);
4503 format %{ %}
4504 interface(CONST_INTER);
4505 %}
4506
4507 // Scale values for scaled offset addressing modes (up to long but not quad)
4508 operand immIScale()
4509 %{
4510 predicate(0 <= n->get_int() && (n->get_int() <= 3));
4511 match(ConI);
4512
4513 op_cost(0);
4514 format %{ %}
4515 interface(CONST_INTER);
4516 %}
4517
4518 // 26 bit signed offset -- for pc-relative branches
4519 operand immI26()
4520 %{
4521 predicate(((-(1 << 25)) <= n->get_int()) && (n->get_int() < (1 << 25)));
4522 match(ConI);
4523
4524 op_cost(0);
4525 format %{ %}
4526 interface(CONST_INTER);
4527 %}
4528
4529 // 19 bit signed offset -- for pc-relative loads
4530 operand immI19()
4531 %{
4532 predicate(((-(1 << 18)) <= n->get_int()) && (n->get_int() < (1 << 18)));
4533 match(ConI);
4534
4535 op_cost(0);
4536 format %{ %}
4537 interface(CONST_INTER);
4538 %}
4539
4540 // 12 bit unsigned offset -- for base plus immediate loads
4541 operand immIU12()
4542 %{
4543 predicate((0 <= n->get_int()) && (n->get_int() < (1 << 12)));
4544 match(ConI);
4545
4546 op_cost(0);
4547 format %{ %}
4548 interface(CONST_INTER);
4549 %}
4550
4551 operand immLU12()
4552 %{
4553 predicate((0 <= n->get_long()) && (n->get_long() < (1 << 12)));
4554 match(ConL);
4555
4556 op_cost(0);
4557 format %{ %}
4558 interface(CONST_INTER);
4559 %}
4560
4561 // Offset for scaled or unscaled immediate loads and stores
4562 operand immIOffset()
4563 %{
4564 predicate(Address::offset_ok_for_immed(n->get_int(), 0));
4565 match(ConI);
4566
4567 op_cost(0);
4568 format %{ %}
4569 interface(CONST_INTER);
4570 %}
4571
4572 operand immIOffset1()
4573 %{
4574 predicate(Address::offset_ok_for_immed(n->get_int(), 0));
4575 match(ConI);
4576
4577 op_cost(0);
4578 format %{ %}
4579 interface(CONST_INTER);
4580 %}
4581
4582 operand immIOffset2()
4583 %{
4584 predicate(Address::offset_ok_for_immed(n->get_int(), 1));
4585 match(ConI);
4586
4587 op_cost(0);
4588 format %{ %}
4589 interface(CONST_INTER);
4590 %}
4591
4592 operand immIOffset4()
4593 %{
4594 predicate(Address::offset_ok_for_immed(n->get_int(), 2));
4595 match(ConI);
4596
4597 op_cost(0);
4598 format %{ %}
4599 interface(CONST_INTER);
4600 %}
4601
4602 operand immIOffset8()
4603 %{
4604 predicate(Address::offset_ok_for_immed(n->get_int(), 3));
4605 match(ConI);
4606
4607 op_cost(0);
4608 format %{ %}
4609 interface(CONST_INTER);
4610 %}
4611
4612 operand immIOffset16()
4613 %{
4614 predicate(Address::offset_ok_for_immed(n->get_int(), 4));
4615 match(ConI);
4616
4617 op_cost(0);
4618 format %{ %}
4619 interface(CONST_INTER);
4620 %}
4621
4622 operand immLoffset()
4623 %{
4624 predicate(Address::offset_ok_for_immed(n->get_long(), 0));
4625 match(ConL);
4626
4627 op_cost(0);
4628 format %{ %}
4629 interface(CONST_INTER);
4630 %}
4631
4632 operand immLoffset1()
4633 %{
4634 predicate(Address::offset_ok_for_immed(n->get_long(), 0));
4635 match(ConL);
4636
4637 op_cost(0);
4638 format %{ %}
4639 interface(CONST_INTER);
4640 %}
4641
4642 operand immLoffset2()
4643 %{
4644 predicate(Address::offset_ok_for_immed(n->get_long(), 1));
4645 match(ConL);
4646
4647 op_cost(0);
4648 format %{ %}
4649 interface(CONST_INTER);
4650 %}
4651
4652 operand immLoffset4()
4653 %{
4654 predicate(Address::offset_ok_for_immed(n->get_long(), 2));
4655 match(ConL);
4656
4657 op_cost(0);
4658 format %{ %}
4659 interface(CONST_INTER);
4660 %}
4661
4662 operand immLoffset8()
4663 %{
4664 predicate(Address::offset_ok_for_immed(n->get_long(), 3));
4665 match(ConL);
4666
4667 op_cost(0);
4668 format %{ %}
4669 interface(CONST_INTER);
4670 %}
4671
4672 operand immLoffset16()
4673 %{
4674 predicate(Address::offset_ok_for_immed(n->get_long(), 4));
4675 match(ConL);
4676
4677 op_cost(0);
4678 format %{ %}
4679 interface(CONST_INTER);
4680 %}
4681
4682 // 8 bit signed value.
4683 operand immI8()
4684 %{
4685 predicate(n->get_int() <= 127 && n->get_int() >= -128);
4686 match(ConI);
4687
4688 op_cost(0);
4689 format %{ %}
4690 interface(CONST_INTER);
4691 %}
4692
4693 // 8 bit signed value (simm8), or #simm8 LSL 8.
4694 operand immI8_shift8()
4695 %{
4696 predicate((n->get_int() <= 127 && n->get_int() >= -128) ||
4697 (n->get_int() <= 32512 && n->get_int() >= -32768 && (n->get_int() & 0xff) == 0));
4698 match(ConI);
4699
4700 op_cost(0);
4701 format %{ %}
4702 interface(CONST_INTER);
4703 %}
4704
4705 // 8 bit signed value (simm8), or #simm8 LSL 8.
4706 operand immL8_shift8()
4707 %{
4708 predicate((n->get_long() <= 127 && n->get_long() >= -128) ||
4709 (n->get_long() <= 32512 && n->get_long() >= -32768 && (n->get_long() & 0xff) == 0));
4710 match(ConL);
4711
4712 op_cost(0);
4713 format %{ %}
4714 interface(CONST_INTER);
4715 %}
4716
4717 // 8 bit integer valid for vector add sub immediate
4718 operand immBAddSubV()
4719 %{
4720 predicate(n->get_int() <= 255 && n->get_int() >= -255);
4721 match(ConI);
4722
4723 op_cost(0);
4724 format %{ %}
4725 interface(CONST_INTER);
4726 %}
4727
4728 // 32 bit integer valid for add sub immediate
4729 operand immIAddSub()
4730 %{
4731 predicate(Assembler::operand_valid_for_add_sub_immediate((int64_t)n->get_int()));
4732 match(ConI);
4733 op_cost(0);
4734 format %{ %}
4735 interface(CONST_INTER);
4736 %}
4737
4738 // 32 bit integer valid for vector add sub immediate
4739 operand immIAddSubV()
4740 %{
4741 predicate(Assembler::operand_valid_for_sve_add_sub_immediate((int64_t)n->get_int()));
4742 match(ConI);
4743
4744 op_cost(0);
4745 format %{ %}
4746 interface(CONST_INTER);
4747 %}
4748
4749 // 32 bit unsigned integer valid for logical immediate
4750
4751 operand immBLog()
4752 %{
4753 predicate(Assembler::operand_valid_for_sve_logical_immediate(BitsPerByte, (uint64_t)n->get_int()));
4754 match(ConI);
4755
4756 op_cost(0);
4757 format %{ %}
4758 interface(CONST_INTER);
4759 %}
4760
4761 operand immSLog()
4762 %{
4763 predicate(Assembler::operand_valid_for_sve_logical_immediate(BitsPerShort, (uint64_t)n->get_int()));
4764 match(ConI);
4765
4766 op_cost(0);
4767 format %{ %}
4768 interface(CONST_INTER);
4769 %}
4770
4771 operand immILog()
4772 %{
4773 predicate(Assembler::operand_valid_for_logical_immediate(/*is32*/true, (uint64_t)n->get_int()));
4774 match(ConI);
4775
4776 op_cost(0);
4777 format %{ %}
4778 interface(CONST_INTER);
4779 %}
4780
4781 // Integer operands 64 bit
4782 // 64 bit immediate
4783 operand immL()
4784 %{
4785 match(ConL);
4786
4787 op_cost(0);
4788 format %{ %}
4789 interface(CONST_INTER);
4790 %}
4791
4792 // 64 bit zero
4793 operand immL0()
4794 %{
4795 predicate(n->get_long() == 0);
4796 match(ConL);
4797
4798 op_cost(0);
4799 format %{ %}
4800 interface(CONST_INTER);
4801 %}
4802
4803 // 64 bit unit increment
4804 operand immL_1()
4805 %{
4806 predicate(n->get_long() == 1);
4807 match(ConL);
4808
4809 op_cost(0);
4810 format %{ %}
4811 interface(CONST_INTER);
4812 %}
4813
4814 // 64 bit unit decrement
4815 operand immL_M1()
4816 %{
4817 predicate(n->get_long() == -1);
4818 match(ConL);
4819
4820 op_cost(0);
4821 format %{ %}
4822 interface(CONST_INTER);
4823 %}
4824
4825 // 32 bit offset of pc in thread anchor
4826
4827 operand immL_pc_off()
4828 %{
4829 predicate(n->get_long() == in_bytes(JavaThread::frame_anchor_offset()) +
4830 in_bytes(JavaFrameAnchor::last_Java_pc_offset()));
4831 match(ConL);
4832
4833 op_cost(0);
4834 format %{ %}
4835 interface(CONST_INTER);
4836 %}
4837
4838 // 64 bit integer valid for add sub immediate
4839 operand immLAddSub()
4840 %{
4841 predicate(Assembler::operand_valid_for_add_sub_immediate(n->get_long()));
4842 match(ConL);
4843 op_cost(0);
4844 format %{ %}
4845 interface(CONST_INTER);
4846 %}
4847
4848 // 64 bit integer valid for addv subv immediate
4849 operand immLAddSubV()
4850 %{
4851 predicate(Assembler::operand_valid_for_sve_add_sub_immediate(n->get_long()));
4852 match(ConL);
4853
4854 op_cost(0);
4855 format %{ %}
4856 interface(CONST_INTER);
4857 %}
4858
4859 // 64 bit integer valid for logical immediate
4860 operand immLLog()
4861 %{
4862 predicate(Assembler::operand_valid_for_logical_immediate(/*is32*/false, (uint64_t)n->get_long()));
4863 match(ConL);
4864 op_cost(0);
4865 format %{ %}
4866 interface(CONST_INTER);
4867 %}
4868
4869 // Long Immediate: low 32-bit mask
4870 operand immL_32bits()
4871 %{
4872 predicate(n->get_long() == 0xFFFFFFFFL);
4873 match(ConL);
4874 op_cost(0);
4875 format %{ %}
4876 interface(CONST_INTER);
4877 %}
4878
4879 // Pointer operands
4880 // Pointer Immediate
4881 operand immP()
4882 %{
4883 match(ConP);
4884
4885 op_cost(0);
4886 format %{ %}
4887 interface(CONST_INTER);
4888 %}
4889
4890 // NULL Pointer Immediate
4891 operand immP0()
4892 %{
4893 predicate(n->get_ptr() == 0);
4894 match(ConP);
4895
4896 op_cost(0);
4897 format %{ %}
4898 interface(CONST_INTER);
4899 %}
4900
4901 // Pointer Immediate One
4902 // this is used in object initialization (initial object header)
4903 operand immP_1()
4904 %{
4905 predicate(n->get_ptr() == 1);
4906 match(ConP);
4907
4908 op_cost(0);
4909 format %{ %}
4910 interface(CONST_INTER);
4911 %}
4912
4913 // Card Table Byte Map Base
4914 operand immByteMapBase()
4915 %{
4916 // Get base of card map
4917 predicate(BarrierSet::barrier_set()->is_a(BarrierSet::CardTableBarrierSet) &&
4918 (CardTable::CardValue*)n->get_ptr() == ((CardTableBarrierSet*)(BarrierSet::barrier_set()))->card_table()->byte_map_base());
4919 match(ConP);
4920
4921 op_cost(0);
4922 format %{ %}
4923 interface(CONST_INTER);
4924 %}
4925
4926 // Pointer Immediate Minus One
4927 // this is used when we want to write the current PC to the thread anchor
4928 operand immP_M1()
4929 %{
4930 predicate(n->get_ptr() == -1);
4931 match(ConP);
4932
4933 op_cost(0);
4934 format %{ %}
4935 interface(CONST_INTER);
4936 %}
4937
4938 // Pointer Immediate Minus Two
4939 // this is used when we want to write the current PC to the thread anchor
4940 operand immP_M2()
4941 %{
4942 predicate(n->get_ptr() == -2);
4943 match(ConP);
4944
4945 op_cost(0);
4946 format %{ %}
4947 interface(CONST_INTER);
4948 %}
4949
4950 // Float and Double operands
4951 // Double Immediate
4952 operand immD()
4953 %{
4954 match(ConD);
4955 op_cost(0);
4956 format %{ %}
4957 interface(CONST_INTER);
4958 %}
4959
4960 // Double Immediate: +0.0d
4961 operand immD0()
4962 %{
4963 predicate(jlong_cast(n->getd()) == 0);
4964 match(ConD);
4965
4966 op_cost(0);
4967 format %{ %}
4968 interface(CONST_INTER);
4969 %}
4970
4971 // constant 'double +0.0'.
4972 operand immDPacked()
4973 %{
4974 predicate(Assembler::operand_valid_for_float_immediate(n->getd()));
4975 match(ConD);
4976 op_cost(0);
4977 format %{ %}
4978 interface(CONST_INTER);
4979 %}
4980
4981 // Float Immediate
4982 operand immF()
4983 %{
4984 match(ConF);
4985 op_cost(0);
4986 format %{ %}
4987 interface(CONST_INTER);
4988 %}
4989
4990 // Float Immediate: +0.0f.
4991 operand immF0()
4992 %{
4993 predicate(jint_cast(n->getf()) == 0);
4994 match(ConF);
4995
4996 op_cost(0);
4997 format %{ %}
4998 interface(CONST_INTER);
4999 %}
5000
5001 //
5002 operand immFPacked()
5003 %{
5004 predicate(Assembler::operand_valid_for_float_immediate((double)n->getf()));
5005 match(ConF);
5006 op_cost(0);
5007 format %{ %}
5008 interface(CONST_INTER);
5009 %}
5010
5011 // Narrow pointer operands
5012 // Narrow Pointer Immediate
5013 operand immN()
5014 %{
5015 match(ConN);
5016
5017 op_cost(0);
5018 format %{ %}
5019 interface(CONST_INTER);
5020 %}
5021
5022 // Narrow NULL Pointer Immediate
5023 operand immN0()
5024 %{
5025 predicate(n->get_narrowcon() == 0);
5026 match(ConN);
5027
5028 op_cost(0);
5029 format %{ %}
5030 interface(CONST_INTER);
5031 %}
5032
5033 operand immNKlass()
5034 %{
5035 match(ConNKlass);
5036
5037 op_cost(0);
5038 format %{ %}
5039 interface(CONST_INTER);
5040 %}
5041
5042 // Integer 32 bit Register Operands
5043 // Integer 32 bitRegister (excludes SP)
5044 operand iRegI()
5045 %{
5046 constraint(ALLOC_IN_RC(any_reg32));
5047 match(RegI);
5048 match(iRegINoSp);
5049 op_cost(0);
5050 format %{ %}
5051 interface(REG_INTER);
5052 %}
5053
5054 // Integer 32 bit Register not Special
5055 operand iRegINoSp()
5056 %{
5057 constraint(ALLOC_IN_RC(no_special_reg32));
5058 match(RegI);
5059 op_cost(0);
5060 format %{ %}
5061 interface(REG_INTER);
5062 %}
5063
5064 // Integer 64 bit Register Operands
5065 // Integer 64 bit Register (includes SP)
5066 operand iRegL()
5067 %{
5068 constraint(ALLOC_IN_RC(any_reg));
5069 match(RegL);
5070 match(iRegLNoSp);
5071 op_cost(0);
5072 format %{ %}
5073 interface(REG_INTER);
5074 %}
5075
5076 // Integer 64 bit Register not Special
5077 operand iRegLNoSp()
5078 %{
5079 constraint(ALLOC_IN_RC(no_special_reg));
5080 match(RegL);
5081 match(iRegL_R0);
5082 format %{ %}
5083 interface(REG_INTER);
5084 %}
5085
5086 // Pointer Register Operands
5087 // Pointer Register
5088 operand iRegP()
5089 %{
5090 constraint(ALLOC_IN_RC(ptr_reg));
5091 match(RegP);
5092 match(iRegPNoSp);
5093 match(iRegP_R0);
5094 //match(iRegP_R2);
5095 //match(iRegP_R4);
5096 //match(iRegP_R5);
5097 match(thread_RegP);
5098 op_cost(0);
5099 format %{ %}
5100 interface(REG_INTER);
5101 %}
5102
5103 // Pointer 64 bit Register not Special
5104 operand iRegPNoSp()
5105 %{
5106 constraint(ALLOC_IN_RC(no_special_ptr_reg));
5107 match(RegP);
5108 // match(iRegP);
5109 // match(iRegP_R0);
5110 // match(iRegP_R2);
5111 // match(iRegP_R4);
5112 // match(iRegP_R5);
5113 // match(thread_RegP);
5114 op_cost(0);
5115 format %{ %}
5116 interface(REG_INTER);
5117 %}
5118
5119 // Pointer 64 bit Register R0 only
5120 operand iRegP_R0()
5121 %{
5122 constraint(ALLOC_IN_RC(r0_reg));
5123 match(RegP);
5124 // match(iRegP);
5125 match(iRegPNoSp);
5126 op_cost(0);
5127 format %{ %}
5128 interface(REG_INTER);
5129 %}
5130
5131 // Pointer 64 bit Register R1 only
5132 operand iRegP_R1()
5133 %{
5134 constraint(ALLOC_IN_RC(r1_reg));
5135 match(RegP);
5136 // match(iRegP);
5137 match(iRegPNoSp);
5138 op_cost(0);
5139 format %{ %}
5140 interface(REG_INTER);
5141 %}
5142
5143 // Pointer 64 bit Register R2 only
5144 operand iRegP_R2()
5145 %{
5146 constraint(ALLOC_IN_RC(r2_reg));
5147 match(RegP);
5148 // match(iRegP);
5149 match(iRegPNoSp);
5150 op_cost(0);
5151 format %{ %}
5152 interface(REG_INTER);
5153 %}
5154
5155 // Pointer 64 bit Register R3 only
5156 operand iRegP_R3()
5157 %{
5158 constraint(ALLOC_IN_RC(r3_reg));
5159 match(RegP);
5160 // match(iRegP);
5161 match(iRegPNoSp);
5162 op_cost(0);
5163 format %{ %}
5164 interface(REG_INTER);
5165 %}
5166
5167 // Pointer 64 bit Register R4 only
5168 operand iRegP_R4()
5169 %{
5170 constraint(ALLOC_IN_RC(r4_reg));
5171 match(RegP);
5172 // match(iRegP);
5173 match(iRegPNoSp);
5174 op_cost(0);
5175 format %{ %}
5176 interface(REG_INTER);
5177 %}
5178
5179 // Pointer 64 bit Register R5 only
5180 operand iRegP_R5()
5181 %{
5182 constraint(ALLOC_IN_RC(r5_reg));
5183 match(RegP);
5184 // match(iRegP);
5185 match(iRegPNoSp);
5186 op_cost(0);
5187 format %{ %}
5188 interface(REG_INTER);
5189 %}
5190
5191 // Pointer 64 bit Register R10 only
5192 operand iRegP_R10()
5193 %{
5194 constraint(ALLOC_IN_RC(r10_reg));
5195 match(RegP);
5196 // match(iRegP);
5197 match(iRegPNoSp);
5198 op_cost(0);
5199 format %{ %}
5200 interface(REG_INTER);
5201 %}
5202
5203 // Long 64 bit Register R0 only
5204 operand iRegL_R0()
5205 %{
5206 constraint(ALLOC_IN_RC(r0_reg));
5207 match(RegL);
5208 match(iRegLNoSp);
5209 op_cost(0);
5210 format %{ %}
5211 interface(REG_INTER);
5212 %}
5213
5214 // Long 64 bit Register R2 only
5215 operand iRegL_R2()
5216 %{
5217 constraint(ALLOC_IN_RC(r2_reg));
5218 match(RegL);
5219 match(iRegLNoSp);
5220 op_cost(0);
5221 format %{ %}
5222 interface(REG_INTER);
5223 %}
5224
5225 // Long 64 bit Register R3 only
5226 operand iRegL_R3()
5227 %{
5228 constraint(ALLOC_IN_RC(r3_reg));
5229 match(RegL);
5230 match(iRegLNoSp);
5231 op_cost(0);
5232 format %{ %}
5233 interface(REG_INTER);
5234 %}
5235
5236 // Long 64 bit Register R11 only
5237 operand iRegL_R11()
5238 %{
5239 constraint(ALLOC_IN_RC(r11_reg));
5240 match(RegL);
5241 match(iRegLNoSp);
5242 op_cost(0);
5243 format %{ %}
5244 interface(REG_INTER);
5245 %}
5246
5247 // Pointer 64 bit Register FP only
5248 operand iRegP_FP()
5249 %{
5250 constraint(ALLOC_IN_RC(fp_reg));
5251 match(RegP);
5252 // match(iRegP);
5253 op_cost(0);
5254 format %{ %}
5255 interface(REG_INTER);
5256 %}
5257
5258 // Register R0 only
5259 operand iRegI_R0()
5260 %{
5261 constraint(ALLOC_IN_RC(int_r0_reg));
5262 match(RegI);
5263 match(iRegINoSp);
5264 op_cost(0);
5265 format %{ %}
5266 interface(REG_INTER);
5267 %}
5268
5269 // Register R2 only
5270 operand iRegI_R2()
5271 %{
5272 constraint(ALLOC_IN_RC(int_r2_reg));
5273 match(RegI);
5274 match(iRegINoSp);
5275 op_cost(0);
5276 format %{ %}
5277 interface(REG_INTER);
5278 %}
5279
5280 // Register R3 only
5281 operand iRegI_R3()
5282 %{
5283 constraint(ALLOC_IN_RC(int_r3_reg));
5284 match(RegI);
5285 match(iRegINoSp);
5286 op_cost(0);
5287 format %{ %}
5288 interface(REG_INTER);
5289 %}
5290
5291
5292 // Register R4 only
5293 operand iRegI_R4()
5294 %{
5295 constraint(ALLOC_IN_RC(int_r4_reg));
5296 match(RegI);
5297 match(iRegINoSp);
5298 op_cost(0);
5299 format %{ %}
5300 interface(REG_INTER);
5301 %}
5302
5303
5304 // Pointer Register Operands
5305 // Narrow Pointer Register
5306 operand iRegN()
5307 %{
5308 constraint(ALLOC_IN_RC(any_reg32));
5309 match(RegN);
5310 match(iRegNNoSp);
5311 op_cost(0);
5312 format %{ %}
5313 interface(REG_INTER);
5314 %}
5315
5316 operand iRegN_R0()
5317 %{
5318 constraint(ALLOC_IN_RC(r0_reg));
5319 match(iRegN);
5320 op_cost(0);
5321 format %{ %}
5322 interface(REG_INTER);
5323 %}
5324
5325 operand iRegN_R2()
5326 %{
5327 constraint(ALLOC_IN_RC(r2_reg));
5328 match(iRegN);
5329 op_cost(0);
5330 format %{ %}
5331 interface(REG_INTER);
5332 %}
5333
5334 operand iRegN_R3()
5335 %{
5336 constraint(ALLOC_IN_RC(r3_reg));
5337 match(iRegN);
5338 op_cost(0);
5339 format %{ %}
5340 interface(REG_INTER);
5341 %}
5342
5343 // Integer 64 bit Register not Special
5344 operand iRegNNoSp()
5345 %{
5346 constraint(ALLOC_IN_RC(no_special_reg32));
5347 match(RegN);
5348 op_cost(0);
5349 format %{ %}
5350 interface(REG_INTER);
5351 %}
5352
5353 // heap base register -- used for encoding immN0
5354
5355 operand iRegIHeapbase()
5356 %{
5357 constraint(ALLOC_IN_RC(heapbase_reg));
5358 match(RegI);
5359 op_cost(0);
5360 format %{ %}
5361 interface(REG_INTER);
5362 %}
5363
5364 // Float Register
5365 // Float register operands
5366 operand vRegF()
5367 %{
5368 constraint(ALLOC_IN_RC(float_reg));
5369 match(RegF);
5370
5371 op_cost(0);
5372 format %{ %}
5373 interface(REG_INTER);
5374 %}
5375
5376 // Double Register
5377 // Double register operands
5378 operand vRegD()
5379 %{
5380 constraint(ALLOC_IN_RC(double_reg));
5381 match(RegD);
5382
5383 op_cost(0);
5384 format %{ %}
5385 interface(REG_INTER);
5386 %}
5387
5388 // Generic vector class. This will be used for
5389 // all vector operands, including NEON and SVE,
5390 // but currently only used for SVE VecA.
5391 operand vReg()
5392 %{
5393 constraint(ALLOC_IN_RC(vectora_reg));
5394 match(VecA);
5395 op_cost(0);
5396 format %{ %}
5397 interface(REG_INTER);
5398 %}
5399
5400 operand vecD()
5401 %{
5402 constraint(ALLOC_IN_RC(vectord_reg));
5403 match(VecD);
5404
5405 op_cost(0);
5406 format %{ %}
5407 interface(REG_INTER);
5408 %}
5409
5410 operand vecX()
5411 %{
5412 constraint(ALLOC_IN_RC(vectorx_reg));
5413 match(VecX);
5414
5415 op_cost(0);
5416 format %{ %}
5417 interface(REG_INTER);
5418 %}
5419
5420 operand vRegD_V0()
5421 %{
5422 constraint(ALLOC_IN_RC(v0_reg));
5423 match(RegD);
5424 op_cost(0);
5425 format %{ %}
5426 interface(REG_INTER);
5427 %}
5428
5429 operand vRegD_V1()
5430 %{
5431 constraint(ALLOC_IN_RC(v1_reg));
5432 match(RegD);
5433 op_cost(0);
5434 format %{ %}
5435 interface(REG_INTER);
5436 %}
5437
5438 operand vRegD_V2()
5439 %{
5440 constraint(ALLOC_IN_RC(v2_reg));
5441 match(RegD);
5442 op_cost(0);
5443 format %{ %}
5444 interface(REG_INTER);
5445 %}
5446
5447 operand vRegD_V3()
5448 %{
5449 constraint(ALLOC_IN_RC(v3_reg));
5450 match(RegD);
5451 op_cost(0);
5452 format %{ %}
5453 interface(REG_INTER);
5454 %}
5455
5456 operand vRegD_V4()
5457 %{
5458 constraint(ALLOC_IN_RC(v4_reg));
5459 match(RegD);
5460 op_cost(0);
5461 format %{ %}
5462 interface(REG_INTER);
5463 %}
5464
5465 operand vRegD_V5()
5466 %{
5467 constraint(ALLOC_IN_RC(v5_reg));
5468 match(RegD);
5469 op_cost(0);
5470 format %{ %}
5471 interface(REG_INTER);
5472 %}
5473
5474 operand vRegD_V6()
5475 %{
5476 constraint(ALLOC_IN_RC(v6_reg));
5477 match(RegD);
5478 op_cost(0);
5479 format %{ %}
5480 interface(REG_INTER);
5481 %}
5482
5483 operand vRegD_V7()
5484 %{
5485 constraint(ALLOC_IN_RC(v7_reg));
5486 match(RegD);
5487 op_cost(0);
5488 format %{ %}
5489 interface(REG_INTER);
5490 %}
5491
5492 operand vRegD_V8()
5493 %{
5494 constraint(ALLOC_IN_RC(v8_reg));
5495 match(RegD);
5496 op_cost(0);
5497 format %{ %}
5498 interface(REG_INTER);
5499 %}
5500
5501 operand vRegD_V9()
5502 %{
5503 constraint(ALLOC_IN_RC(v9_reg));
5504 match(RegD);
5505 op_cost(0);
5506 format %{ %}
5507 interface(REG_INTER);
5508 %}
5509
5510 operand vRegD_V10()
5511 %{
5512 constraint(ALLOC_IN_RC(v10_reg));
5513 match(RegD);
5514 op_cost(0);
5515 format %{ %}
5516 interface(REG_INTER);
5517 %}
5518
5519 operand vRegD_V11()
5520 %{
5521 constraint(ALLOC_IN_RC(v11_reg));
5522 match(RegD);
5523 op_cost(0);
5524 format %{ %}
5525 interface(REG_INTER);
5526 %}
5527
5528 operand vRegD_V12()
5529 %{
5530 constraint(ALLOC_IN_RC(v12_reg));
5531 match(RegD);
5532 op_cost(0);
5533 format %{ %}
5534 interface(REG_INTER);
5535 %}
5536
5537 operand vRegD_V13()
5538 %{
5539 constraint(ALLOC_IN_RC(v13_reg));
5540 match(RegD);
5541 op_cost(0);
5542 format %{ %}
5543 interface(REG_INTER);
5544 %}
5545
5546 operand vRegD_V14()
5547 %{
5548 constraint(ALLOC_IN_RC(v14_reg));
5549 match(RegD);
5550 op_cost(0);
5551 format %{ %}
5552 interface(REG_INTER);
5553 %}
5554
5555 operand vRegD_V15()
5556 %{
5557 constraint(ALLOC_IN_RC(v15_reg));
5558 match(RegD);
5559 op_cost(0);
5560 format %{ %}
5561 interface(REG_INTER);
5562 %}
5563
5564 operand vRegD_V16()
5565 %{
5566 constraint(ALLOC_IN_RC(v16_reg));
5567 match(RegD);
5568 op_cost(0);
5569 format %{ %}
5570 interface(REG_INTER);
5571 %}
5572
5573 operand vRegD_V17()
5574 %{
5575 constraint(ALLOC_IN_RC(v17_reg));
5576 match(RegD);
5577 op_cost(0);
5578 format %{ %}
5579 interface(REG_INTER);
5580 %}
5581
5582 operand vRegD_V18()
5583 %{
5584 constraint(ALLOC_IN_RC(v18_reg));
5585 match(RegD);
5586 op_cost(0);
5587 format %{ %}
5588 interface(REG_INTER);
5589 %}
5590
5591 operand vRegD_V19()
5592 %{
5593 constraint(ALLOC_IN_RC(v19_reg));
5594 match(RegD);
5595 op_cost(0);
5596 format %{ %}
5597 interface(REG_INTER);
5598 %}
5599
5600 operand vRegD_V20()
5601 %{
5602 constraint(ALLOC_IN_RC(v20_reg));
5603 match(RegD);
5604 op_cost(0);
5605 format %{ %}
5606 interface(REG_INTER);
5607 %}
5608
5609 operand vRegD_V21()
5610 %{
5611 constraint(ALLOC_IN_RC(v21_reg));
5612 match(RegD);
5613 op_cost(0);
5614 format %{ %}
5615 interface(REG_INTER);
5616 %}
5617
5618 operand vRegD_V22()
5619 %{
5620 constraint(ALLOC_IN_RC(v22_reg));
5621 match(RegD);
5622 op_cost(0);
5623 format %{ %}
5624 interface(REG_INTER);
5625 %}
5626
5627 operand vRegD_V23()
5628 %{
5629 constraint(ALLOC_IN_RC(v23_reg));
5630 match(RegD);
5631 op_cost(0);
5632 format %{ %}
5633 interface(REG_INTER);
5634 %}
5635
5636 operand vRegD_V24()
5637 %{
5638 constraint(ALLOC_IN_RC(v24_reg));
5639 match(RegD);
5640 op_cost(0);
5641 format %{ %}
5642 interface(REG_INTER);
5643 %}
5644
5645 operand vRegD_V25()
5646 %{
5647 constraint(ALLOC_IN_RC(v25_reg));
5648 match(RegD);
5649 op_cost(0);
5650 format %{ %}
5651 interface(REG_INTER);
5652 %}
5653
5654 operand vRegD_V26()
5655 %{
5656 constraint(ALLOC_IN_RC(v26_reg));
5657 match(RegD);
5658 op_cost(0);
5659 format %{ %}
5660 interface(REG_INTER);
5661 %}
5662
5663 operand vRegD_V27()
5664 %{
5665 constraint(ALLOC_IN_RC(v27_reg));
5666 match(RegD);
5667 op_cost(0);
5668 format %{ %}
5669 interface(REG_INTER);
5670 %}
5671
5672 operand vRegD_V28()
5673 %{
5674 constraint(ALLOC_IN_RC(v28_reg));
5675 match(RegD);
5676 op_cost(0);
5677 format %{ %}
5678 interface(REG_INTER);
5679 %}
5680
5681 operand vRegD_V29()
5682 %{
5683 constraint(ALLOC_IN_RC(v29_reg));
5684 match(RegD);
5685 op_cost(0);
5686 format %{ %}
5687 interface(REG_INTER);
5688 %}
5689
5690 operand vRegD_V30()
5691 %{
5692 constraint(ALLOC_IN_RC(v30_reg));
5693 match(RegD);
5694 op_cost(0);
5695 format %{ %}
5696 interface(REG_INTER);
5697 %}
5698
5699 operand vRegD_V31()
5700 %{
5701 constraint(ALLOC_IN_RC(v31_reg));
5702 match(RegD);
5703 op_cost(0);
5704 format %{ %}
5705 interface(REG_INTER);
5706 %}
5707
5708 operand pReg()
5709 %{
5710 constraint(ALLOC_IN_RC(pr_reg));
5711 match(RegVectMask);
5712 match(pRegGov);
5713 op_cost(0);
5714 format %{ %}
5715 interface(REG_INTER);
5716 %}
5717
5718 operand pRegGov()
5719 %{
5720 constraint(ALLOC_IN_RC(gov_pr));
5721 match(RegVectMask);
5722 op_cost(0);
5723 format %{ %}
5724 interface(REG_INTER);
5725 %}
5726
5727 // Flags register, used as output of signed compare instructions
5728
5729 // note that on AArch64 we also use this register as the output for
5730 // for floating point compare instructions (CmpF CmpD). this ensures
5731 // that ordered inequality tests use GT, GE, LT or LE none of which
5732 // pass through cases where the result is unordered i.e. one or both
5733 // inputs to the compare is a NaN. this means that the ideal code can
5734 // replace e.g. a GT with an LE and not end up capturing the NaN case
5735 // (where the comparison should always fail). EQ and NE tests are
5736 // always generated in ideal code so that unordered folds into the NE
5737 // case, matching the behaviour of AArch64 NE.
5738 //
5739 // This differs from x86 where the outputs of FP compares use a
5740 // special FP flags registers and where compares based on this
5741 // register are distinguished into ordered inequalities (cmpOpUCF) and
5742 // EQ/NEQ tests (cmpOpUCF2). x86 has to special case the latter tests
5743 // to explicitly handle the unordered case in branches. x86 also has
5744 // to include extra CMoveX rules to accept a cmpOpUCF input.
5745
5746 operand rFlagsReg()
5747 %{
5748 constraint(ALLOC_IN_RC(int_flags));
5749 match(RegFlags);
5750
5751 op_cost(0);
5752 format %{ "RFLAGS" %}
5753 interface(REG_INTER);
5754 %}
5755
5756 // Flags register, used as output of unsigned compare instructions
5757 operand rFlagsRegU()
5758 %{
5759 constraint(ALLOC_IN_RC(int_flags));
5760 match(RegFlags);
5761
5762 op_cost(0);
5763 format %{ "RFLAGSU" %}
5764 interface(REG_INTER);
5765 %}
5766
5767 // Special Registers
5768
5769 // Method Register
5770 operand inline_cache_RegP(iRegP reg)
5771 %{
5772 constraint(ALLOC_IN_RC(method_reg)); // inline_cache_reg
5773 match(reg);
5774 match(iRegPNoSp);
5775 op_cost(0);
5776 format %{ %}
5777 interface(REG_INTER);
5778 %}
5779
5780 // Thread Register
5781 operand thread_RegP(iRegP reg)
5782 %{
5783 constraint(ALLOC_IN_RC(thread_reg)); // link_reg
5784 match(reg);
5785 op_cost(0);
5786 format %{ %}
5787 interface(REG_INTER);
5788 %}
5789
5790 operand lr_RegP(iRegP reg)
5791 %{
5792 constraint(ALLOC_IN_RC(lr_reg)); // link_reg
5793 match(reg);
5794 op_cost(0);
5795 format %{ %}
5796 interface(REG_INTER);
5797 %}
5798
5799 //----------Memory Operands----------------------------------------------------
5800
5801 operand indirect(iRegP reg)
5802 %{
5803 constraint(ALLOC_IN_RC(ptr_reg));
5804 match(reg);
5805 op_cost(0);
5806 format %{ "[$reg]" %}
5807 interface(MEMORY_INTER) %{
5808 base($reg);
5809 index(0xffffffff);
5810 scale(0x0);
5811 disp(0x0);
5812 %}
5813 %}
5814
5815 operand indIndexScaledI2L(iRegP reg, iRegI ireg, immIScale scale)
5816 %{
5817 constraint(ALLOC_IN_RC(ptr_reg));
5818 predicate(size_fits_all_mem_uses(n->as_AddP(), n->in(AddPNode::Offset)->in(2)->get_int()));
5819 match(AddP reg (LShiftL (ConvI2L ireg) scale));
5820 op_cost(0);
5821 format %{ "$reg, $ireg sxtw($scale), 0, I2L" %}
5822 interface(MEMORY_INTER) %{
5823 base($reg);
5824 index($ireg);
5825 scale($scale);
5826 disp(0x0);
5827 %}
5828 %}
5829
5830 operand indIndexScaled(iRegP reg, iRegL lreg, immIScale scale)
5831 %{
5832 constraint(ALLOC_IN_RC(ptr_reg));
5833 predicate(size_fits_all_mem_uses(n->as_AddP(), n->in(AddPNode::Offset)->in(2)->get_int()));
5834 match(AddP reg (LShiftL lreg scale));
5835 op_cost(0);
5836 format %{ "$reg, $lreg lsl($scale)" %}
5837 interface(MEMORY_INTER) %{
5838 base($reg);
5839 index($lreg);
5840 scale($scale);
5841 disp(0x0);
5842 %}
5843 %}
5844
5845 operand indIndexI2L(iRegP reg, iRegI ireg)
5846 %{
5847 constraint(ALLOC_IN_RC(ptr_reg));
5848 match(AddP reg (ConvI2L ireg));
5849 op_cost(0);
5850 format %{ "$reg, $ireg, 0, I2L" %}
5851 interface(MEMORY_INTER) %{
5852 base($reg);
5853 index($ireg);
5854 scale(0x0);
5855 disp(0x0);
5856 %}
5857 %}
5858
5859 operand indIndex(iRegP reg, iRegL lreg)
5860 %{
5861 constraint(ALLOC_IN_RC(ptr_reg));
5862 match(AddP reg lreg);
5863 op_cost(0);
5864 format %{ "$reg, $lreg" %}
5865 interface(MEMORY_INTER) %{
5866 base($reg);
5867 index($lreg);
5868 scale(0x0);
5869 disp(0x0);
5870 %}
5871 %}
5872
5873 operand indOffI(iRegP reg, immIOffset off)
5874 %{
5875 constraint(ALLOC_IN_RC(ptr_reg));
5876 match(AddP reg off);
5877 op_cost(0);
5878 format %{ "[$reg, $off]" %}
5879 interface(MEMORY_INTER) %{
5880 base($reg);
5881 index(0xffffffff);
5882 scale(0x0);
5883 disp($off);
5884 %}
5885 %}
5886
5887 operand indOffI1(iRegP reg, immIOffset1 off)
5888 %{
5889 constraint(ALLOC_IN_RC(ptr_reg));
5890 match(AddP reg off);
5891 op_cost(0);
5892 format %{ "[$reg, $off]" %}
5893 interface(MEMORY_INTER) %{
5894 base($reg);
5895 index(0xffffffff);
5896 scale(0x0);
5897 disp($off);
5898 %}
5899 %}
5900
5901 operand indOffI2(iRegP reg, immIOffset2 off)
5902 %{
5903 constraint(ALLOC_IN_RC(ptr_reg));
5904 match(AddP reg off);
5905 op_cost(0);
5906 format %{ "[$reg, $off]" %}
5907 interface(MEMORY_INTER) %{
5908 base($reg);
5909 index(0xffffffff);
5910 scale(0x0);
5911 disp($off);
5912 %}
5913 %}
5914
5915 operand indOffI4(iRegP reg, immIOffset4 off)
5916 %{
5917 constraint(ALLOC_IN_RC(ptr_reg));
5918 match(AddP reg off);
5919 op_cost(0);
5920 format %{ "[$reg, $off]" %}
5921 interface(MEMORY_INTER) %{
5922 base($reg);
5923 index(0xffffffff);
5924 scale(0x0);
5925 disp($off);
5926 %}
5927 %}
5928
5929 operand indOffI8(iRegP reg, immIOffset8 off)
5930 %{
5931 constraint(ALLOC_IN_RC(ptr_reg));
5932 match(AddP reg off);
5933 op_cost(0);
5934 format %{ "[$reg, $off]" %}
5935 interface(MEMORY_INTER) %{
5936 base($reg);
5937 index(0xffffffff);
5938 scale(0x0);
5939 disp($off);
5940 %}
5941 %}
5942
5943 operand indOffI16(iRegP reg, immIOffset16 off)
5944 %{
5945 constraint(ALLOC_IN_RC(ptr_reg));
5946 match(AddP reg off);
5947 op_cost(0);
5948 format %{ "[$reg, $off]" %}
5949 interface(MEMORY_INTER) %{
5950 base($reg);
5951 index(0xffffffff);
5952 scale(0x0);
5953 disp($off);
5954 %}
5955 %}
5956
5957 operand indOffL(iRegP reg, immLoffset off)
5958 %{
5959 constraint(ALLOC_IN_RC(ptr_reg));
5960 match(AddP reg off);
5961 op_cost(0);
5962 format %{ "[$reg, $off]" %}
5963 interface(MEMORY_INTER) %{
5964 base($reg);
5965 index(0xffffffff);
5966 scale(0x0);
5967 disp($off);
5968 %}
5969 %}
5970
5971 operand indOffL1(iRegP reg, immLoffset1 off)
5972 %{
5973 constraint(ALLOC_IN_RC(ptr_reg));
5974 match(AddP reg off);
5975 op_cost(0);
5976 format %{ "[$reg, $off]" %}
5977 interface(MEMORY_INTER) %{
5978 base($reg);
5979 index(0xffffffff);
5980 scale(0x0);
5981 disp($off);
5982 %}
5983 %}
5984
5985 operand indOffL2(iRegP reg, immLoffset2 off)
5986 %{
5987 constraint(ALLOC_IN_RC(ptr_reg));
5988 match(AddP reg off);
5989 op_cost(0);
5990 format %{ "[$reg, $off]" %}
5991 interface(MEMORY_INTER) %{
5992 base($reg);
5993 index(0xffffffff);
5994 scale(0x0);
5995 disp($off);
5996 %}
5997 %}
5998
5999 operand indOffL4(iRegP reg, immLoffset4 off)
6000 %{
6001 constraint(ALLOC_IN_RC(ptr_reg));
6002 match(AddP reg off);
6003 op_cost(0);
6004 format %{ "[$reg, $off]" %}
6005 interface(MEMORY_INTER) %{
6006 base($reg);
6007 index(0xffffffff);
6008 scale(0x0);
6009 disp($off);
6010 %}
6011 %}
6012
6013 operand indOffL8(iRegP reg, immLoffset8 off)
6014 %{
6015 constraint(ALLOC_IN_RC(ptr_reg));
6016 match(AddP reg off);
6017 op_cost(0);
6018 format %{ "[$reg, $off]" %}
6019 interface(MEMORY_INTER) %{
6020 base($reg);
6021 index(0xffffffff);
6022 scale(0x0);
6023 disp($off);
6024 %}
6025 %}
6026
6027 operand indOffL16(iRegP reg, immLoffset16 off)
6028 %{
6029 constraint(ALLOC_IN_RC(ptr_reg));
6030 match(AddP reg off);
6031 op_cost(0);
6032 format %{ "[$reg, $off]" %}
6033 interface(MEMORY_INTER) %{
6034 base($reg);
6035 index(0xffffffff);
6036 scale(0x0);
6037 disp($off);
6038 %}
6039 %}
6040
6041 operand indirectN(iRegN reg)
6042 %{
6043 predicate(CompressedOops::shift() == 0);
6044 constraint(ALLOC_IN_RC(ptr_reg));
6045 match(DecodeN reg);
6046 op_cost(0);
6047 format %{ "[$reg]\t# narrow" %}
6048 interface(MEMORY_INTER) %{
6049 base($reg);
6050 index(0xffffffff);
6051 scale(0x0);
6052 disp(0x0);
6053 %}
6054 %}
6055
6056 operand indIndexScaledI2LN(iRegN reg, iRegI ireg, immIScale scale)
6057 %{
6058 predicate(CompressedOops::shift() == 0 && size_fits_all_mem_uses(n->as_AddP(), n->in(AddPNode::Offset)->in(2)->get_int()));
6059 constraint(ALLOC_IN_RC(ptr_reg));
6060 match(AddP (DecodeN reg) (LShiftL (ConvI2L ireg) scale));
6061 op_cost(0);
6062 format %{ "$reg, $ireg sxtw($scale), 0, I2L\t# narrow" %}
6063 interface(MEMORY_INTER) %{
6064 base($reg);
6065 index($ireg);
6066 scale($scale);
6067 disp(0x0);
6068 %}
6069 %}
6070
6071 operand indIndexScaledN(iRegN reg, iRegL lreg, immIScale scale)
6072 %{
6073 predicate(CompressedOops::shift() == 0 && size_fits_all_mem_uses(n->as_AddP(), n->in(AddPNode::Offset)->in(2)->get_int()));
6074 constraint(ALLOC_IN_RC(ptr_reg));
6075 match(AddP (DecodeN reg) (LShiftL lreg scale));
6076 op_cost(0);
6077 format %{ "$reg, $lreg lsl($scale)\t# narrow" %}
6078 interface(MEMORY_INTER) %{
6079 base($reg);
6080 index($lreg);
6081 scale($scale);
6082 disp(0x0);
6083 %}
6084 %}
6085
6086 operand indIndexI2LN(iRegN reg, iRegI ireg)
6087 %{
6088 predicate(CompressedOops::shift() == 0);
6089 constraint(ALLOC_IN_RC(ptr_reg));
6090 match(AddP (DecodeN reg) (ConvI2L ireg));
6091 op_cost(0);
6092 format %{ "$reg, $ireg, 0, I2L\t# narrow" %}
6093 interface(MEMORY_INTER) %{
6094 base($reg);
6095 index($ireg);
6096 scale(0x0);
6097 disp(0x0);
6098 %}
6099 %}
6100
6101 operand indIndexN(iRegN reg, iRegL lreg)
6102 %{
6103 predicate(CompressedOops::shift() == 0);
6104 constraint(ALLOC_IN_RC(ptr_reg));
6105 match(AddP (DecodeN reg) lreg);
6106 op_cost(0);
6107 format %{ "$reg, $lreg\t# narrow" %}
6108 interface(MEMORY_INTER) %{
6109 base($reg);
6110 index($lreg);
6111 scale(0x0);
6112 disp(0x0);
6113 %}
6114 %}
6115
6116 operand indOffIN(iRegN reg, immIOffset off)
6117 %{
6118 predicate(CompressedOops::shift() == 0);
6119 constraint(ALLOC_IN_RC(ptr_reg));
6120 match(AddP (DecodeN reg) off);
6121 op_cost(0);
6122 format %{ "[$reg, $off]\t# narrow" %}
6123 interface(MEMORY_INTER) %{
6124 base($reg);
6125 index(0xffffffff);
6126 scale(0x0);
6127 disp($off);
6128 %}
6129 %}
6130
6131 operand indOffLN(iRegN reg, immLoffset off)
6132 %{
6133 predicate(CompressedOops::shift() == 0);
6134 constraint(ALLOC_IN_RC(ptr_reg));
6135 match(AddP (DecodeN reg) off);
6136 op_cost(0);
6137 format %{ "[$reg, $off]\t# narrow" %}
6138 interface(MEMORY_INTER) %{
6139 base($reg);
6140 index(0xffffffff);
6141 scale(0x0);
6142 disp($off);
6143 %}
6144 %}
6145
6146
6147
6148 // AArch64 opto stubs need to write to the pc slot in the thread anchor
6149 operand thread_anchor_pc(thread_RegP reg, immL_pc_off off)
6150 %{
6151 constraint(ALLOC_IN_RC(ptr_reg));
6152 match(AddP reg off);
6153 op_cost(0);
6154 format %{ "[$reg, $off]" %}
6155 interface(MEMORY_INTER) %{
6156 base($reg);
6157 index(0xffffffff);
6158 scale(0x0);
6159 disp($off);
6160 %}
6161 %}
6162
6163 //----------Special Memory Operands--------------------------------------------
6164 // Stack Slot Operand - This operand is used for loading and storing temporary
6165 // values on the stack where a match requires a value to
6166 // flow through memory.
6167 operand stackSlotP(sRegP reg)
6168 %{
6169 constraint(ALLOC_IN_RC(stack_slots));
6170 op_cost(100);
6171 // No match rule because this operand is only generated in matching
6172 // match(RegP);
6173 format %{ "[$reg]" %}
6174 interface(MEMORY_INTER) %{
6175 base(0x1e); // RSP
6176 index(0x0); // No Index
6177 scale(0x0); // No Scale
6178 disp($reg); // Stack Offset
6179 %}
6180 %}
6181
6182 operand stackSlotI(sRegI reg)
6183 %{
6184 constraint(ALLOC_IN_RC(stack_slots));
6185 // No match rule because this operand is only generated in matching
6186 // match(RegI);
6187 format %{ "[$reg]" %}
6188 interface(MEMORY_INTER) %{
6189 base(0x1e); // RSP
6190 index(0x0); // No Index
6191 scale(0x0); // No Scale
6192 disp($reg); // Stack Offset
6193 %}
6194 %}
6195
6196 operand stackSlotF(sRegF reg)
6197 %{
6198 constraint(ALLOC_IN_RC(stack_slots));
6199 // No match rule because this operand is only generated in matching
6200 // match(RegF);
6201 format %{ "[$reg]" %}
6202 interface(MEMORY_INTER) %{
6203 base(0x1e); // RSP
6204 index(0x0); // No Index
6205 scale(0x0); // No Scale
6206 disp($reg); // Stack Offset
6207 %}
6208 %}
6209
6210 operand stackSlotD(sRegD reg)
6211 %{
6212 constraint(ALLOC_IN_RC(stack_slots));
6213 // No match rule because this operand is only generated in matching
6214 // match(RegD);
6215 format %{ "[$reg]" %}
6216 interface(MEMORY_INTER) %{
6217 base(0x1e); // RSP
6218 index(0x0); // No Index
6219 scale(0x0); // No Scale
6220 disp($reg); // Stack Offset
6221 %}
6222 %}
6223
6224 operand stackSlotL(sRegL reg)
6225 %{
6226 constraint(ALLOC_IN_RC(stack_slots));
6227 // No match rule because this operand is only generated in matching
6228 // match(RegL);
6229 format %{ "[$reg]" %}
6230 interface(MEMORY_INTER) %{
6231 base(0x1e); // RSP
6232 index(0x0); // No Index
6233 scale(0x0); // No Scale
6234 disp($reg); // Stack Offset
6235 %}
6236 %}
6237
6238 // Operands for expressing Control Flow
6239 // NOTE: Label is a predefined operand which should not be redefined in
6240 // the AD file. It is generically handled within the ADLC.
6241
6242 //----------Conditional Branch Operands----------------------------------------
6243 // Comparison Op - This is the operation of the comparison, and is limited to
6244 // the following set of codes:
6245 // L (<), LE (<=), G (>), GE (>=), E (==), NE (!=)
6246 //
6247 // Other attributes of the comparison, such as unsignedness, are specified
6248 // by the comparison instruction that sets a condition code flags register.
6249 // That result is represented by a flags operand whose subtype is appropriate
6250 // to the unsignedness (etc.) of the comparison.
6251 //
6252 // Later, the instruction which matches both the Comparison Op (a Bool) and
6253 // the flags (produced by the Cmp) specifies the coding of the comparison op
6254 // by matching a specific subtype of Bool operand below, such as cmpOpU.
6255
6256 // used for signed integral comparisons and fp comparisons
6257
6258 operand cmpOp()
6259 %{
6260 match(Bool);
6261
6262 format %{ "" %}
6263 interface(COND_INTER) %{
6264 equal(0x0, "eq");
6265 not_equal(0x1, "ne");
6266 less(0xb, "lt");
6267 greater_equal(0xa, "ge");
6268 less_equal(0xd, "le");
6269 greater(0xc, "gt");
6270 overflow(0x6, "vs");
6271 no_overflow(0x7, "vc");
6272 %}
6273 %}
6274
6275 // used for unsigned integral comparisons
6276
6277 operand cmpOpU()
6278 %{
6279 match(Bool);
6280
6281 format %{ "" %}
6282 interface(COND_INTER) %{
6283 equal(0x0, "eq");
6284 not_equal(0x1, "ne");
6285 less(0x3, "lo");
6286 greater_equal(0x2, "hs");
6287 less_equal(0x9, "ls");
6288 greater(0x8, "hi");
6289 overflow(0x6, "vs");
6290 no_overflow(0x7, "vc");
6291 %}
6292 %}
6293
6294 // used for certain integral comparisons which can be
6295 // converted to cbxx or tbxx instructions
6296
6297 operand cmpOpEqNe()
6298 %{
6299 match(Bool);
6300 op_cost(0);
6301 predicate(n->as_Bool()->_test._test == BoolTest::ne
6302 || n->as_Bool()->_test._test == BoolTest::eq);
6303
6304 format %{ "" %}
6305 interface(COND_INTER) %{
6306 equal(0x0, "eq");
6307 not_equal(0x1, "ne");
6308 less(0xb, "lt");
6309 greater_equal(0xa, "ge");
6310 less_equal(0xd, "le");
6311 greater(0xc, "gt");
6312 overflow(0x6, "vs");
6313 no_overflow(0x7, "vc");
6314 %}
6315 %}
6316
6317 // used for certain integral comparisons which can be
6318 // converted to cbxx or tbxx instructions
6319
6320 operand cmpOpLtGe()
6321 %{
6322 match(Bool);
6323 op_cost(0);
6324
6325 predicate(n->as_Bool()->_test._test == BoolTest::lt
6326 || n->as_Bool()->_test._test == BoolTest::ge);
6327
6328 format %{ "" %}
6329 interface(COND_INTER) %{
6330 equal(0x0, "eq");
6331 not_equal(0x1, "ne");
6332 less(0xb, "lt");
6333 greater_equal(0xa, "ge");
6334 less_equal(0xd, "le");
6335 greater(0xc, "gt");
6336 overflow(0x6, "vs");
6337 no_overflow(0x7, "vc");
6338 %}
6339 %}
6340
6341 // used for certain unsigned integral comparisons which can be
6342 // converted to cbxx or tbxx instructions
6343
6344 operand cmpOpUEqNeLtGe()
6345 %{
6346 match(Bool);
6347 op_cost(0);
6348
6349 predicate(n->as_Bool()->_test._test == BoolTest::eq
6350 || n->as_Bool()->_test._test == BoolTest::ne
6351 || n->as_Bool()->_test._test == BoolTest::lt
6352 || n->as_Bool()->_test._test == BoolTest::ge);
6353
6354 format %{ "" %}
6355 interface(COND_INTER) %{
6356 equal(0x0, "eq");
6357 not_equal(0x1, "ne");
6358 less(0xb, "lt");
6359 greater_equal(0xa, "ge");
6360 less_equal(0xd, "le");
6361 greater(0xc, "gt");
6362 overflow(0x6, "vs");
6363 no_overflow(0x7, "vc");
6364 %}
6365 %}
6366
6367 // Special operand allowing long args to int ops to be truncated for free
6368
6369 operand iRegL2I(iRegL reg) %{
6370
6371 op_cost(0);
6372
6373 match(ConvL2I reg);
6374
6375 format %{ "l2i($reg)" %}
6376
6377 interface(REG_INTER)
6378 %}
6379
6380 opclass vmem2(indirect, indIndex, indOffI2, indOffL2);
6381 opclass vmem4(indirect, indIndex, indOffI4, indOffL4);
6382 opclass vmem8(indirect, indIndex, indOffI8, indOffL8);
6383 opclass vmem16(indirect, indIndex, indOffI16, indOffL16);
6384
6385 //----------OPERAND CLASSES----------------------------------------------------
6386 // Operand Classes are groups of operands that are used as to simplify
6387 // instruction definitions by not requiring the AD writer to specify
6388 // separate instructions for every form of operand when the
6389 // instruction accepts multiple operand types with the same basic
6390 // encoding and format. The classic case of this is memory operands.
6391
6392 // memory is used to define read/write location for load/store
6393 // instruction defs. we can turn a memory op into an Address
6394
6395 opclass memory1(indirect, indIndexScaled, indIndexScaledI2L, indIndexI2L, indIndex, indOffI1, indOffL1,
6396 indirectN, indIndexScaledN, indIndexScaledI2LN, indIndexI2LN, indIndexN);
6397
6398 opclass memory2(indirect, indIndexScaled, indIndexScaledI2L, indIndexI2L, indIndex, indOffI2, indOffL2,
6399 indirectN, indIndexScaledN, indIndexScaledI2LN, indIndexI2LN, indIndexN);
6400
6401 opclass memory4(indirect, indIndexScaled, indIndexScaledI2L, indIndexI2L, indIndex, indOffI4, indOffL4,
6402 indirectN, indIndexScaledN, indIndexScaledI2LN, indIndexI2LN, indIndexN, indOffIN, indOffLN);
6403
6404 opclass memory8(indirect, indIndexScaled, indIndexScaledI2L, indIndexI2L, indIndex, indOffI8, indOffL8,
6405 indirectN, indIndexScaledN, indIndexScaledI2LN, indIndexI2LN, indIndexN, indOffIN, indOffLN);
6406
6407 // All of the memory operands. For the pipeline description.
6408 opclass memory(indirect, indIndexScaled, indIndexScaledI2L, indIndexI2L, indIndex,
6409 indOffI1, indOffL1, indOffI2, indOffL2, indOffI4, indOffL4, indOffI8, indOffL8,
6410 indirectN, indIndexScaledN, indIndexScaledI2LN, indIndexI2LN, indIndexN, indOffIN, indOffLN);
6411
6412
6413 // iRegIorL2I is used for src inputs in rules for 32 bit int (I)
6414 // operations. it allows the src to be either an iRegI or a (ConvL2I
6415 // iRegL). in the latter case the l2i normally planted for a ConvL2I
6416 // can be elided because the 32-bit instruction will just employ the
6417 // lower 32 bits anyway.
6418 //
6419 // n.b. this does not elide all L2I conversions. if the truncated
6420 // value is consumed by more than one operation then the ConvL2I
6421 // cannot be bundled into the consuming nodes so an l2i gets planted
6422 // (actually a movw $dst $src) and the downstream instructions consume
6423 // the result of the l2i as an iRegI input. That's a shame since the
6424 // movw is actually redundant but its not too costly.
6425
6426 opclass iRegIorL2I(iRegI, iRegL2I);
6427
6428 //----------PIPELINE-----------------------------------------------------------
6429 // Rules which define the behavior of the target architectures pipeline.
6430
6431 // For specific pipelines, eg A53, define the stages of that pipeline
6432 //pipe_desc(ISS, EX1, EX2, WR);
6433 #define ISS S0
6434 #define EX1 S1
6435 #define EX2 S2
6436 #define WR S3
6437
6438 // Integer ALU reg operation
6439 pipeline %{
6440
6441 attributes %{
6442 // ARM instructions are of fixed length
6443 fixed_size_instructions; // Fixed size instructions TODO does
6444 max_instructions_per_bundle = 4; // A53 = 2, A57 = 4
6445 // ARM instructions come in 32-bit word units
6446 instruction_unit_size = 4; // An instruction is 4 bytes long
6447 instruction_fetch_unit_size = 64; // The processor fetches one line
6448 instruction_fetch_units = 1; // of 64 bytes
6449
6450 // List of nop instructions
6451 nops( MachNop );
6452 %}
6453
6454 // We don't use an actual pipeline model so don't care about resources
6455 // or description. we do use pipeline classes to introduce fixed
6456 // latencies
6457
6458 //----------RESOURCES----------------------------------------------------------
6459 // Resources are the functional units available to the machine
6460
6461 resources( INS0, INS1, INS01 = INS0 | INS1,
6462 ALU0, ALU1, ALU = ALU0 | ALU1,
6463 MAC,
6464 DIV,
6465 BRANCH,
6466 LDST,
6467 NEON_FP);
6468
6469 //----------PIPELINE DESCRIPTION-----------------------------------------------
6470 // Pipeline Description specifies the stages in the machine's pipeline
6471
6472 // Define the pipeline as a generic 6 stage pipeline
6473 pipe_desc(S0, S1, S2, S3, S4, S5);
6474
6475 //----------PIPELINE CLASSES---------------------------------------------------
6476 // Pipeline Classes describe the stages in which input and output are
6477 // referenced by the hardware pipeline.
6478
6479 pipe_class fp_dop_reg_reg_s(vRegF dst, vRegF src1, vRegF src2)
6480 %{
6481 single_instruction;
6482 src1 : S1(read);
6483 src2 : S2(read);
6484 dst : S5(write);
6485 INS01 : ISS;
6486 NEON_FP : S5;
6487 %}
6488
6489 pipe_class fp_dop_reg_reg_d(vRegD dst, vRegD src1, vRegD src2)
6490 %{
6491 single_instruction;
6492 src1 : S1(read);
6493 src2 : S2(read);
6494 dst : S5(write);
6495 INS01 : ISS;
6496 NEON_FP : S5;
6497 %}
6498
6499 pipe_class fp_uop_s(vRegF dst, vRegF src)
6500 %{
6501 single_instruction;
6502 src : S1(read);
6503 dst : S5(write);
6504 INS01 : ISS;
6505 NEON_FP : S5;
6506 %}
6507
6508 pipe_class fp_uop_d(vRegD dst, vRegD src)
6509 %{
6510 single_instruction;
6511 src : S1(read);
6512 dst : S5(write);
6513 INS01 : ISS;
6514 NEON_FP : S5;
6515 %}
6516
6517 pipe_class fp_d2f(vRegF dst, vRegD src)
6518 %{
6519 single_instruction;
6520 src : S1(read);
6521 dst : S5(write);
6522 INS01 : ISS;
6523 NEON_FP : S5;
6524 %}
6525
6526 pipe_class fp_f2d(vRegD dst, vRegF src)
6527 %{
6528 single_instruction;
6529 src : S1(read);
6530 dst : S5(write);
6531 INS01 : ISS;
6532 NEON_FP : S5;
6533 %}
6534
6535 pipe_class fp_f2i(iRegINoSp dst, vRegF src)
6536 %{
6537 single_instruction;
6538 src : S1(read);
6539 dst : S5(write);
6540 INS01 : ISS;
6541 NEON_FP : S5;
6542 %}
6543
6544 pipe_class fp_f2l(iRegLNoSp dst, vRegF src)
6545 %{
6546 single_instruction;
6547 src : S1(read);
6548 dst : S5(write);
6549 INS01 : ISS;
6550 NEON_FP : S5;
6551 %}
6552
6553 pipe_class fp_i2f(vRegF dst, iRegIorL2I src)
6554 %{
6555 single_instruction;
6556 src : S1(read);
6557 dst : S5(write);
6558 INS01 : ISS;
6559 NEON_FP : S5;
6560 %}
6561
6562 pipe_class fp_l2f(vRegF dst, iRegL src)
6563 %{
6564 single_instruction;
6565 src : S1(read);
6566 dst : S5(write);
6567 INS01 : ISS;
6568 NEON_FP : S5;
6569 %}
6570
6571 pipe_class fp_d2i(iRegINoSp dst, vRegD src)
6572 %{
6573 single_instruction;
6574 src : S1(read);
6575 dst : S5(write);
6576 INS01 : ISS;
6577 NEON_FP : S5;
6578 %}
6579
6580 pipe_class fp_d2l(iRegLNoSp dst, vRegD src)
6581 %{
6582 single_instruction;
6583 src : S1(read);
6584 dst : S5(write);
6585 INS01 : ISS;
6586 NEON_FP : S5;
6587 %}
6588
6589 pipe_class fp_i2d(vRegD dst, iRegIorL2I src)
6590 %{
6591 single_instruction;
6592 src : S1(read);
6593 dst : S5(write);
6594 INS01 : ISS;
6595 NEON_FP : S5;
6596 %}
6597
6598 pipe_class fp_l2d(vRegD dst, iRegIorL2I src)
6599 %{
6600 single_instruction;
6601 src : S1(read);
6602 dst : S5(write);
6603 INS01 : ISS;
6604 NEON_FP : S5;
6605 %}
6606
6607 pipe_class fp_div_s(vRegF dst, vRegF src1, vRegF src2)
6608 %{
6609 single_instruction;
6610 src1 : S1(read);
6611 src2 : S2(read);
6612 dst : S5(write);
6613 INS0 : ISS;
6614 NEON_FP : S5;
6615 %}
6616
6617 pipe_class fp_div_d(vRegD dst, vRegD src1, vRegD src2)
6618 %{
6619 single_instruction;
6620 src1 : S1(read);
6621 src2 : S2(read);
6622 dst : S5(write);
6623 INS0 : ISS;
6624 NEON_FP : S5;
6625 %}
6626
6627 pipe_class fp_cond_reg_reg_s(vRegF dst, vRegF src1, vRegF src2, rFlagsReg cr)
6628 %{
6629 single_instruction;
6630 cr : S1(read);
6631 src1 : S1(read);
6632 src2 : S1(read);
6633 dst : S3(write);
6634 INS01 : ISS;
6635 NEON_FP : S3;
6636 %}
6637
6638 pipe_class fp_cond_reg_reg_d(vRegD dst, vRegD src1, vRegD src2, rFlagsReg cr)
6639 %{
6640 single_instruction;
6641 cr : S1(read);
6642 src1 : S1(read);
6643 src2 : S1(read);
6644 dst : S3(write);
6645 INS01 : ISS;
6646 NEON_FP : S3;
6647 %}
6648
6649 pipe_class fp_imm_s(vRegF dst)
6650 %{
6651 single_instruction;
6652 dst : S3(write);
6653 INS01 : ISS;
6654 NEON_FP : S3;
6655 %}
6656
6657 pipe_class fp_imm_d(vRegD dst)
6658 %{
6659 single_instruction;
6660 dst : S3(write);
6661 INS01 : ISS;
6662 NEON_FP : S3;
6663 %}
6664
6665 pipe_class fp_load_constant_s(vRegF dst)
6666 %{
6667 single_instruction;
6668 dst : S4(write);
6669 INS01 : ISS;
6670 NEON_FP : S4;
6671 %}
6672
6673 pipe_class fp_load_constant_d(vRegD dst)
6674 %{
6675 single_instruction;
6676 dst : S4(write);
6677 INS01 : ISS;
6678 NEON_FP : S4;
6679 %}
6680
6681 pipe_class vmul64(vecD dst, vecD src1, vecD src2)
6682 %{
6683 single_instruction;
6684 dst : S5(write);
6685 src1 : S1(read);
6686 src2 : S1(read);
6687 INS01 : ISS;
6688 NEON_FP : S5;
6689 %}
6690
6691 pipe_class vmul128(vecX dst, vecX src1, vecX src2)
6692 %{
6693 single_instruction;
6694 dst : S5(write);
6695 src1 : S1(read);
6696 src2 : S1(read);
6697 INS0 : ISS;
6698 NEON_FP : S5;
6699 %}
6700
6701 pipe_class vmla64(vecD dst, vecD src1, vecD src2)
6702 %{
6703 single_instruction;
6704 dst : S5(write);
6705 src1 : S1(read);
6706 src2 : S1(read);
6707 dst : S1(read);
6708 INS01 : ISS;
6709 NEON_FP : S5;
6710 %}
6711
6712 pipe_class vmla128(vecX dst, vecX src1, vecX src2)
6713 %{
6714 single_instruction;
6715 dst : S5(write);
6716 src1 : S1(read);
6717 src2 : S1(read);
6718 dst : S1(read);
6719 INS0 : ISS;
6720 NEON_FP : S5;
6721 %}
6722
6723 pipe_class vdop64(vecD dst, vecD src1, vecD src2)
6724 %{
6725 single_instruction;
6726 dst : S4(write);
6727 src1 : S2(read);
6728 src2 : S2(read);
6729 INS01 : ISS;
6730 NEON_FP : S4;
6731 %}
6732
6733 pipe_class vdop128(vecX dst, vecX src1, vecX src2)
6734 %{
6735 single_instruction;
6736 dst : S4(write);
6737 src1 : S2(read);
6738 src2 : S2(read);
6739 INS0 : ISS;
6740 NEON_FP : S4;
6741 %}
6742
6743 pipe_class vlogical64(vecD dst, vecD src1, vecD src2)
6744 %{
6745 single_instruction;
6746 dst : S3(write);
6747 src1 : S2(read);
6748 src2 : S2(read);
6749 INS01 : ISS;
6750 NEON_FP : S3;
6751 %}
6752
6753 pipe_class vlogical128(vecX dst, vecX src1, vecX src2)
6754 %{
6755 single_instruction;
6756 dst : S3(write);
6757 src1 : S2(read);
6758 src2 : S2(read);
6759 INS0 : ISS;
6760 NEON_FP : S3;
6761 %}
6762
6763 pipe_class vshift64(vecD dst, vecD src, vecX shift)
6764 %{
6765 single_instruction;
6766 dst : S3(write);
6767 src : S1(read);
6768 shift : S1(read);
6769 INS01 : ISS;
6770 NEON_FP : S3;
6771 %}
6772
6773 pipe_class vshift128(vecX dst, vecX src, vecX shift)
6774 %{
6775 single_instruction;
6776 dst : S3(write);
6777 src : S1(read);
6778 shift : S1(read);
6779 INS0 : ISS;
6780 NEON_FP : S3;
6781 %}
6782
6783 pipe_class vshift64_imm(vecD dst, vecD src, immI shift)
6784 %{
6785 single_instruction;
6786 dst : S3(write);
6787 src : S1(read);
6788 INS01 : ISS;
6789 NEON_FP : S3;
6790 %}
6791
6792 pipe_class vshift128_imm(vecX dst, vecX src, immI shift)
6793 %{
6794 single_instruction;
6795 dst : S3(write);
6796 src : S1(read);
6797 INS0 : ISS;
6798 NEON_FP : S3;
6799 %}
6800
6801 pipe_class vdop_fp64(vecD dst, vecD src1, vecD src2)
6802 %{
6803 single_instruction;
6804 dst : S5(write);
6805 src1 : S1(read);
6806 src2 : S1(read);
6807 INS01 : ISS;
6808 NEON_FP : S5;
6809 %}
6810
6811 pipe_class vdop_fp128(vecX dst, vecX src1, vecX src2)
6812 %{
6813 single_instruction;
6814 dst : S5(write);
6815 src1 : S1(read);
6816 src2 : S1(read);
6817 INS0 : ISS;
6818 NEON_FP : S5;
6819 %}
6820
6821 pipe_class vmuldiv_fp64(vecD dst, vecD src1, vecD src2)
6822 %{
6823 single_instruction;
6824 dst : S5(write);
6825 src1 : S1(read);
6826 src2 : S1(read);
6827 INS0 : ISS;
6828 NEON_FP : S5;
6829 %}
6830
6831 pipe_class vmuldiv_fp128(vecX dst, vecX src1, vecX src2)
6832 %{
6833 single_instruction;
6834 dst : S5(write);
6835 src1 : S1(read);
6836 src2 : S1(read);
6837 INS0 : ISS;
6838 NEON_FP : S5;
6839 %}
6840
6841 pipe_class vsqrt_fp128(vecX dst, vecX src)
6842 %{
6843 single_instruction;
6844 dst : S5(write);
6845 src : S1(read);
6846 INS0 : ISS;
6847 NEON_FP : S5;
6848 %}
6849
6850 pipe_class vunop_fp64(vecD dst, vecD src)
6851 %{
6852 single_instruction;
6853 dst : S5(write);
6854 src : S1(read);
6855 INS01 : ISS;
6856 NEON_FP : S5;
6857 %}
6858
6859 pipe_class vunop_fp128(vecX dst, vecX src)
6860 %{
6861 single_instruction;
6862 dst : S5(write);
6863 src : S1(read);
6864 INS0 : ISS;
6865 NEON_FP : S5;
6866 %}
6867
6868 pipe_class vdup_reg_reg64(vecD dst, iRegI src)
6869 %{
6870 single_instruction;
6871 dst : S3(write);
6872 src : S1(read);
6873 INS01 : ISS;
6874 NEON_FP : S3;
6875 %}
6876
6877 pipe_class vdup_reg_reg128(vecX dst, iRegI src)
6878 %{
6879 single_instruction;
6880 dst : S3(write);
6881 src : S1(read);
6882 INS01 : ISS;
6883 NEON_FP : S3;
6884 %}
6885
6886 pipe_class vdup_reg_freg64(vecD dst, vRegF src)
6887 %{
6888 single_instruction;
6889 dst : S3(write);
6890 src : S1(read);
6891 INS01 : ISS;
6892 NEON_FP : S3;
6893 %}
6894
6895 pipe_class vdup_reg_freg128(vecX dst, vRegF src)
6896 %{
6897 single_instruction;
6898 dst : S3(write);
6899 src : S1(read);
6900 INS01 : ISS;
6901 NEON_FP : S3;
6902 %}
6903
6904 pipe_class vdup_reg_dreg128(vecX dst, vRegD src)
6905 %{
6906 single_instruction;
6907 dst : S3(write);
6908 src : S1(read);
6909 INS01 : ISS;
6910 NEON_FP : S3;
6911 %}
6912
6913 pipe_class vmovi_reg_imm64(vecD dst)
6914 %{
6915 single_instruction;
6916 dst : S3(write);
6917 INS01 : ISS;
6918 NEON_FP : S3;
6919 %}
6920
6921 pipe_class vmovi_reg_imm128(vecX dst)
6922 %{
6923 single_instruction;
6924 dst : S3(write);
6925 INS0 : ISS;
6926 NEON_FP : S3;
6927 %}
6928
6929 pipe_class vload_reg_mem64(vecD dst, vmem8 mem)
6930 %{
6931 single_instruction;
6932 dst : S5(write);
6933 mem : ISS(read);
6934 INS01 : ISS;
6935 NEON_FP : S3;
6936 %}
6937
6938 pipe_class vload_reg_mem128(vecX dst, vmem16 mem)
6939 %{
6940 single_instruction;
6941 dst : S5(write);
6942 mem : ISS(read);
6943 INS01 : ISS;
6944 NEON_FP : S3;
6945 %}
6946
6947 pipe_class vstore_reg_mem64(vecD src, vmem8 mem)
6948 %{
6949 single_instruction;
6950 mem : ISS(read);
6951 src : S2(read);
6952 INS01 : ISS;
6953 NEON_FP : S3;
6954 %}
6955
6956 pipe_class vstore_reg_mem128(vecD src, vmem16 mem)
6957 %{
6958 single_instruction;
6959 mem : ISS(read);
6960 src : S2(read);
6961 INS01 : ISS;
6962 NEON_FP : S3;
6963 %}
6964
6965 //------- Integer ALU operations --------------------------
6966
6967 // Integer ALU reg-reg operation
6968 // Operands needed in EX1, result generated in EX2
6969 // Eg. ADD x0, x1, x2
6970 pipe_class ialu_reg_reg(iRegI dst, iRegI src1, iRegI src2)
6971 %{
6972 single_instruction;
6973 dst : EX2(write);
6974 src1 : EX1(read);
6975 src2 : EX1(read);
6976 INS01 : ISS; // Dual issue as instruction 0 or 1
6977 ALU : EX2;
6978 %}
6979
6980 // Integer ALU reg-reg operation with constant shift
6981 // Shifted register must be available in LATE_ISS instead of EX1
6982 // Eg. ADD x0, x1, x2, LSL #2
6983 pipe_class ialu_reg_reg_shift(iRegI dst, iRegI src1, iRegI src2, immI shift)
6984 %{
6985 single_instruction;
6986 dst : EX2(write);
6987 src1 : EX1(read);
6988 src2 : ISS(read);
6989 INS01 : ISS;
6990 ALU : EX2;
6991 %}
6992
6993 // Integer ALU reg operation with constant shift
6994 // Eg. LSL x0, x1, #shift
6995 pipe_class ialu_reg_shift(iRegI dst, iRegI src1)
6996 %{
6997 single_instruction;
6998 dst : EX2(write);
6999 src1 : ISS(read);
7000 INS01 : ISS;
7001 ALU : EX2;
7002 %}
7003
7004 // Integer ALU reg-reg operation with variable shift
7005 // Both operands must be available in LATE_ISS instead of EX1
7006 // Result is available in EX1 instead of EX2
7007 // Eg. LSLV x0, x1, x2
7008 pipe_class ialu_reg_reg_vshift(iRegI dst, iRegI src1, iRegI src2)
7009 %{
7010 single_instruction;
7011 dst : EX1(write);
7012 src1 : ISS(read);
7013 src2 : ISS(read);
7014 INS01 : ISS;
7015 ALU : EX1;
7016 %}
7017
7018 // Integer ALU reg-reg operation with extract
7019 // As for _vshift above, but result generated in EX2
7020 // Eg. EXTR x0, x1, x2, #N
7021 pipe_class ialu_reg_reg_extr(iRegI dst, iRegI src1, iRegI src2)
7022 %{
7023 single_instruction;
7024 dst : EX2(write);
7025 src1 : ISS(read);
7026 src2 : ISS(read);
7027 INS1 : ISS; // Can only dual issue as Instruction 1
7028 ALU : EX1;
7029 %}
7030
7031 // Integer ALU reg operation
7032 // Eg. NEG x0, x1
7033 pipe_class ialu_reg(iRegI dst, iRegI src)
7034 %{
7035 single_instruction;
7036 dst : EX2(write);
7037 src : EX1(read);
7038 INS01 : ISS;
7039 ALU : EX2;
7040 %}
7041
7042 // Integer ALU reg mmediate operation
7043 // Eg. ADD x0, x1, #N
7044 pipe_class ialu_reg_imm(iRegI dst, iRegI src1)
7045 %{
7046 single_instruction;
7047 dst : EX2(write);
7048 src1 : EX1(read);
7049 INS01 : ISS;
7050 ALU : EX2;
7051 %}
7052
7053 // Integer ALU immediate operation (no source operands)
7054 // Eg. MOV x0, #N
7055 pipe_class ialu_imm(iRegI dst)
7056 %{
7057 single_instruction;
7058 dst : EX1(write);
7059 INS01 : ISS;
7060 ALU : EX1;
7061 %}
7062
7063 //------- Compare operation -------------------------------
7064
7065 // Compare reg-reg
7066 // Eg. CMP x0, x1
7067 pipe_class icmp_reg_reg(rFlagsReg cr, iRegI op1, iRegI op2)
7068 %{
7069 single_instruction;
7070 // fixed_latency(16);
7071 cr : EX2(write);
7072 op1 : EX1(read);
7073 op2 : EX1(read);
7074 INS01 : ISS;
7075 ALU : EX2;
7076 %}
7077
7078 // Compare reg-reg
7079 // Eg. CMP x0, #N
7080 pipe_class icmp_reg_imm(rFlagsReg cr, iRegI op1)
7081 %{
7082 single_instruction;
7083 // fixed_latency(16);
7084 cr : EX2(write);
7085 op1 : EX1(read);
7086 INS01 : ISS;
7087 ALU : EX2;
7088 %}
7089
7090 //------- Conditional instructions ------------------------
7091
7092 // Conditional no operands
7093 // Eg. CSINC x0, zr, zr, <cond>
7094 pipe_class icond_none(iRegI dst, rFlagsReg cr)
7095 %{
7096 single_instruction;
7097 cr : EX1(read);
7098 dst : EX2(write);
7099 INS01 : ISS;
7100 ALU : EX2;
7101 %}
7102
7103 // Conditional 2 operand
7104 // EG. CSEL X0, X1, X2, <cond>
7105 pipe_class icond_reg_reg(iRegI dst, iRegI src1, iRegI src2, rFlagsReg cr)
7106 %{
7107 single_instruction;
7108 cr : EX1(read);
7109 src1 : EX1(read);
7110 src2 : EX1(read);
7111 dst : EX2(write);
7112 INS01 : ISS;
7113 ALU : EX2;
7114 %}
7115
7116 // Conditional 2 operand
7117 // EG. CSEL X0, X1, X2, <cond>
7118 pipe_class icond_reg(iRegI dst, iRegI src, rFlagsReg cr)
7119 %{
7120 single_instruction;
7121 cr : EX1(read);
7122 src : EX1(read);
7123 dst : EX2(write);
7124 INS01 : ISS;
7125 ALU : EX2;
7126 %}
7127
7128 //------- Multiply pipeline operations --------------------
7129
7130 // Multiply reg-reg
7131 // Eg. MUL w0, w1, w2
7132 pipe_class imul_reg_reg(iRegI dst, iRegI src1, iRegI src2)
7133 %{
7134 single_instruction;
7135 dst : WR(write);
7136 src1 : ISS(read);
7137 src2 : ISS(read);
7138 INS01 : ISS;
7139 MAC : WR;
7140 %}
7141
7142 // Multiply accumulate
7143 // Eg. MADD w0, w1, w2, w3
7144 pipe_class imac_reg_reg(iRegI dst, iRegI src1, iRegI src2, iRegI src3)
7145 %{
7146 single_instruction;
7147 dst : WR(write);
7148 src1 : ISS(read);
7149 src2 : ISS(read);
7150 src3 : ISS(read);
7151 INS01 : ISS;
7152 MAC : WR;
7153 %}
7154
7155 // Eg. MUL w0, w1, w2
7156 pipe_class lmul_reg_reg(iRegI dst, iRegI src1, iRegI src2)
7157 %{
7158 single_instruction;
7159 fixed_latency(3); // Maximum latency for 64 bit mul
7160 dst : WR(write);
7161 src1 : ISS(read);
7162 src2 : ISS(read);
7163 INS01 : ISS;
7164 MAC : WR;
7165 %}
7166
7167 // Multiply accumulate
7168 // Eg. MADD w0, w1, w2, w3
7169 pipe_class lmac_reg_reg(iRegI dst, iRegI src1, iRegI src2, iRegI src3)
7170 %{
7171 single_instruction;
7172 fixed_latency(3); // Maximum latency for 64 bit mul
7173 dst : WR(write);
7174 src1 : ISS(read);
7175 src2 : ISS(read);
7176 src3 : ISS(read);
7177 INS01 : ISS;
7178 MAC : WR;
7179 %}
7180
7181 //------- Divide pipeline operations --------------------
7182
7183 // Eg. SDIV w0, w1, w2
7184 pipe_class idiv_reg_reg(iRegI dst, iRegI src1, iRegI src2)
7185 %{
7186 single_instruction;
7187 fixed_latency(8); // Maximum latency for 32 bit divide
7188 dst : WR(write);
7189 src1 : ISS(read);
7190 src2 : ISS(read);
7191 INS0 : ISS; // Can only dual issue as instruction 0
7192 DIV : WR;
7193 %}
7194
7195 // Eg. SDIV x0, x1, x2
7196 pipe_class ldiv_reg_reg(iRegI dst, iRegI src1, iRegI src2)
7197 %{
7198 single_instruction;
7199 fixed_latency(16); // Maximum latency for 64 bit divide
7200 dst : WR(write);
7201 src1 : ISS(read);
7202 src2 : ISS(read);
7203 INS0 : ISS; // Can only dual issue as instruction 0
7204 DIV : WR;
7205 %}
7206
7207 //------- Load pipeline operations ------------------------
7208
7209 // Load - prefetch
7210 // Eg. PFRM <mem>
7211 pipe_class iload_prefetch(memory mem)
7212 %{
7213 single_instruction;
7214 mem : ISS(read);
7215 INS01 : ISS;
7216 LDST : WR;
7217 %}
7218
7219 // Load - reg, mem
7220 // Eg. LDR x0, <mem>
7221 pipe_class iload_reg_mem(iRegI dst, memory mem)
7222 %{
7223 single_instruction;
7224 dst : WR(write);
7225 mem : ISS(read);
7226 INS01 : ISS;
7227 LDST : WR;
7228 %}
7229
7230 // Load - reg, reg
7231 // Eg. LDR x0, [sp, x1]
7232 pipe_class iload_reg_reg(iRegI dst, iRegI src)
7233 %{
7234 single_instruction;
7235 dst : WR(write);
7236 src : ISS(read);
7237 INS01 : ISS;
7238 LDST : WR;
7239 %}
7240
7241 //------- Store pipeline operations -----------------------
7242
7243 // Store - zr, mem
7244 // Eg. STR zr, <mem>
7245 pipe_class istore_mem(memory mem)
7246 %{
7247 single_instruction;
7248 mem : ISS(read);
7249 INS01 : ISS;
7250 LDST : WR;
7251 %}
7252
7253 // Store - reg, mem
7254 // Eg. STR x0, <mem>
7255 pipe_class istore_reg_mem(iRegI src, memory mem)
7256 %{
7257 single_instruction;
7258 mem : ISS(read);
7259 src : EX2(read);
7260 INS01 : ISS;
7261 LDST : WR;
7262 %}
7263
7264 // Store - reg, reg
7265 // Eg. STR x0, [sp, x1]
7266 pipe_class istore_reg_reg(iRegI dst, iRegI src)
7267 %{
7268 single_instruction;
7269 dst : ISS(read);
7270 src : EX2(read);
7271 INS01 : ISS;
7272 LDST : WR;
7273 %}
7274
7275 //------- Store pipeline operations -----------------------
7276
7277 // Branch
7278 pipe_class pipe_branch()
7279 %{
7280 single_instruction;
7281 INS01 : ISS;
7282 BRANCH : EX1;
7283 %}
7284
7285 // Conditional branch
7286 pipe_class pipe_branch_cond(rFlagsReg cr)
7287 %{
7288 single_instruction;
7289 cr : EX1(read);
7290 INS01 : ISS;
7291 BRANCH : EX1;
7292 %}
7293
7294 // Compare & Branch
7295 // EG. CBZ/CBNZ
7296 pipe_class pipe_cmp_branch(iRegI op1)
7297 %{
7298 single_instruction;
7299 op1 : EX1(read);
7300 INS01 : ISS;
7301 BRANCH : EX1;
7302 %}
7303
7304 //------- Synchronisation operations ----------------------
7305
7306 // Any operation requiring serialization.
7307 // EG. DMB/Atomic Ops/Load Acquire/Str Release
7308 pipe_class pipe_serial()
7309 %{
7310 single_instruction;
7311 force_serialization;
7312 fixed_latency(16);
7313 INS01 : ISS(2); // Cannot dual issue with any other instruction
7314 LDST : WR;
7315 %}
7316
7317 // Generic big/slow expanded idiom - also serialized
7318 pipe_class pipe_slow()
7319 %{
7320 instruction_count(10);
7321 multiple_bundles;
7322 force_serialization;
7323 fixed_latency(16);
7324 INS01 : ISS(2); // Cannot dual issue with any other instruction
7325 LDST : WR;
7326 %}
7327
7328 // Empty pipeline class
7329 pipe_class pipe_class_empty()
7330 %{
7331 single_instruction;
7332 fixed_latency(0);
7333 %}
7334
7335 // Default pipeline class.
7336 pipe_class pipe_class_default()
7337 %{
7338 single_instruction;
7339 fixed_latency(2);
7340 %}
7341
7342 // Pipeline class for compares.
7343 pipe_class pipe_class_compare()
7344 %{
7345 single_instruction;
7346 fixed_latency(16);
7347 %}
7348
7349 // Pipeline class for memory operations.
7350 pipe_class pipe_class_memory()
7351 %{
7352 single_instruction;
7353 fixed_latency(16);
7354 %}
7355
7356 // Pipeline class for call.
7357 pipe_class pipe_class_call()
7358 %{
7359 single_instruction;
7360 fixed_latency(100);
7361 %}
7362
7363 // Define the class for the Nop node.
7364 define %{
7365 MachNop = pipe_class_empty;
7366 %}
7367
7368 %}
7369 //----------INSTRUCTIONS-------------------------------------------------------
7370 //
7371 // match -- States which machine-independent subtree may be replaced
7372 // by this instruction.
7373 // ins_cost -- The estimated cost of this instruction is used by instruction
7374 // selection to identify a minimum cost tree of machine
7375 // instructions that matches a tree of machine-independent
7376 // instructions.
7377 // format -- A string providing the disassembly for this instruction.
7378 // The value of an instruction's operand may be inserted
7379 // by referring to it with a '$' prefix.
7380 // opcode -- Three instruction opcodes may be provided. These are referred
7381 // to within an encode class as $primary, $secondary, and $tertiary
7382 // rrspectively. The primary opcode is commonly used to
7383 // indicate the type of machine instruction, while secondary
7384 // and tertiary are often used for prefix options or addressing
7385 // modes.
7386 // ins_encode -- A list of encode classes with parameters. The encode class
7387 // name must have been defined in an 'enc_class' specification
7388 // in the encode section of the architecture description.
7389
7390 // ============================================================================
7391 // Memory (Load/Store) Instructions
7392
7393 // Load Instructions
7394
7395 // Load Byte (8 bit signed)
7396 instruct loadB(iRegINoSp dst, memory1 mem)
7397 %{
7398 match(Set dst (LoadB mem));
7399 predicate(!needs_acquiring_load(n));
7400
7401 ins_cost(4 * INSN_COST);
7402 format %{ "ldrsbw $dst, $mem\t# byte" %}
7403
7404 ins_encode(aarch64_enc_ldrsbw(dst, mem));
7405
7406 ins_pipe(iload_reg_mem);
7407 %}
7408
7409 // Load Byte (8 bit signed) into long
7410 instruct loadB2L(iRegLNoSp dst, memory1 mem)
7411 %{
7412 match(Set dst (ConvI2L (LoadB mem)));
7413 predicate(!needs_acquiring_load(n->in(1)));
7414
7415 ins_cost(4 * INSN_COST);
7416 format %{ "ldrsb $dst, $mem\t# byte" %}
7417
7418 ins_encode(aarch64_enc_ldrsb(dst, mem));
7419
7420 ins_pipe(iload_reg_mem);
7421 %}
7422
7423 // Load Byte (8 bit unsigned)
7424 instruct loadUB(iRegINoSp dst, memory1 mem)
7425 %{
7426 match(Set dst (LoadUB mem));
7427 predicate(!needs_acquiring_load(n));
7428
7429 ins_cost(4 * INSN_COST);
7430 format %{ "ldrbw $dst, $mem\t# byte" %}
7431
7432 ins_encode(aarch64_enc_ldrb(dst, mem));
7433
7434 ins_pipe(iload_reg_mem);
7435 %}
7436
7437 // Load Byte (8 bit unsigned) into long
7438 instruct loadUB2L(iRegLNoSp dst, memory1 mem)
7439 %{
7440 match(Set dst (ConvI2L (LoadUB mem)));
7441 predicate(!needs_acquiring_load(n->in(1)));
7442
7443 ins_cost(4 * INSN_COST);
7444 format %{ "ldrb $dst, $mem\t# byte" %}
7445
7446 ins_encode(aarch64_enc_ldrb(dst, mem));
7447
7448 ins_pipe(iload_reg_mem);
7449 %}
7450
7451 // Load Short (16 bit signed)
7452 instruct loadS(iRegINoSp dst, memory2 mem)
7453 %{
7454 match(Set dst (LoadS mem));
7455 predicate(!needs_acquiring_load(n));
7456
7457 ins_cost(4 * INSN_COST);
7458 format %{ "ldrshw $dst, $mem\t# short" %}
7459
7460 ins_encode(aarch64_enc_ldrshw(dst, mem));
7461
7462 ins_pipe(iload_reg_mem);
7463 %}
7464
7465 // Load Short (16 bit signed) into long
7466 instruct loadS2L(iRegLNoSp dst, memory2 mem)
7467 %{
7468 match(Set dst (ConvI2L (LoadS mem)));
7469 predicate(!needs_acquiring_load(n->in(1)));
7470
7471 ins_cost(4 * INSN_COST);
7472 format %{ "ldrsh $dst, $mem\t# short" %}
7473
7474 ins_encode(aarch64_enc_ldrsh(dst, mem));
7475
7476 ins_pipe(iload_reg_mem);
7477 %}
7478
7479 // Load Char (16 bit unsigned)
7480 instruct loadUS(iRegINoSp dst, memory2 mem)
7481 %{
7482 match(Set dst (LoadUS mem));
7483 predicate(!needs_acquiring_load(n));
7484
7485 ins_cost(4 * INSN_COST);
7486 format %{ "ldrh $dst, $mem\t# short" %}
7487
7488 ins_encode(aarch64_enc_ldrh(dst, mem));
7489
7490 ins_pipe(iload_reg_mem);
7491 %}
7492
7493 // Load Short/Char (16 bit unsigned) into long
7494 instruct loadUS2L(iRegLNoSp dst, memory2 mem)
7495 %{
7496 match(Set dst (ConvI2L (LoadUS mem)));
7497 predicate(!needs_acquiring_load(n->in(1)));
7498
7499 ins_cost(4 * INSN_COST);
7500 format %{ "ldrh $dst, $mem\t# short" %}
7501
7502 ins_encode(aarch64_enc_ldrh(dst, mem));
7503
7504 ins_pipe(iload_reg_mem);
7505 %}
7506
7507 // Load Integer (32 bit signed)
7508 instruct loadI(iRegINoSp dst, memory4 mem)
7509 %{
7510 match(Set dst (LoadI mem));
7511 predicate(!needs_acquiring_load(n));
7512
7513 ins_cost(4 * INSN_COST);
7514 format %{ "ldrw $dst, $mem\t# int" %}
7515
7516 ins_encode(aarch64_enc_ldrw(dst, mem));
7517
7518 ins_pipe(iload_reg_mem);
7519 %}
7520
7521 // Load Integer (32 bit signed) into long
7522 instruct loadI2L(iRegLNoSp dst, memory4 mem)
7523 %{
7524 match(Set dst (ConvI2L (LoadI mem)));
7525 predicate(!needs_acquiring_load(n->in(1)));
7526
7527 ins_cost(4 * INSN_COST);
7528 format %{ "ldrsw $dst, $mem\t# int" %}
7529
7530 ins_encode(aarch64_enc_ldrsw(dst, mem));
7531
7532 ins_pipe(iload_reg_mem);
7533 %}
7534
7535 // Load Integer (32 bit unsigned) into long
7536 instruct loadUI2L(iRegLNoSp dst, memory4 mem, immL_32bits mask)
7537 %{
7538 match(Set dst (AndL (ConvI2L (LoadI mem)) mask));
7539 predicate(!needs_acquiring_load(n->in(1)->in(1)->as_Load()));
7540
7541 ins_cost(4 * INSN_COST);
7542 format %{ "ldrw $dst, $mem\t# int" %}
7543
7544 ins_encode(aarch64_enc_ldrw(dst, mem));
7545
7546 ins_pipe(iload_reg_mem);
7547 %}
7548
7549 // Load Long (64 bit signed)
7550 instruct loadL(iRegLNoSp dst, memory8 mem)
7551 %{
7552 match(Set dst (LoadL mem));
7553 predicate(!needs_acquiring_load(n));
7554
7555 ins_cost(4 * INSN_COST);
7556 format %{ "ldr $dst, $mem\t# int" %}
7557
7558 ins_encode(aarch64_enc_ldr(dst, mem));
7559
7560 ins_pipe(iload_reg_mem);
7561 %}
7562
7563 // Load Range
7564 instruct loadRange(iRegINoSp dst, memory4 mem)
7565 %{
7566 match(Set dst (LoadRange mem));
7567
7568 ins_cost(4 * INSN_COST);
7569 format %{ "ldrw $dst, $mem\t# range" %}
7570
7571 ins_encode(aarch64_enc_ldrw(dst, mem));
7572
7573 ins_pipe(iload_reg_mem);
7574 %}
7575
7576 // Load Pointer
7577 instruct loadP(iRegPNoSp dst, memory8 mem)
7578 %{
7579 match(Set dst (LoadP mem));
7580 predicate(!needs_acquiring_load(n) && (n->as_Load()->barrier_data() == 0));
7581
7582 ins_cost(4 * INSN_COST);
7583 format %{ "ldr $dst, $mem\t# ptr" %}
7584
7585 ins_encode(aarch64_enc_ldr(dst, mem));
7586
7587 ins_pipe(iload_reg_mem);
7588 %}
7589
7590 // Load Compressed Pointer
7591 instruct loadN(iRegNNoSp dst, memory4 mem)
7592 %{
7593 match(Set dst (LoadN mem));
7594 predicate(!needs_acquiring_load(n));
7595
7596 ins_cost(4 * INSN_COST);
7597 format %{ "ldrw $dst, $mem\t# compressed ptr" %}
7598
7599 ins_encode(aarch64_enc_ldrw(dst, mem));
7600
7601 ins_pipe(iload_reg_mem);
7602 %}
7603
7604 // Load Klass Pointer
7605 instruct loadKlass(iRegPNoSp dst, memory8 mem)
7606 %{
7607 match(Set dst (LoadKlass mem));
7608 predicate(!needs_acquiring_load(n));
7609
7610 ins_cost(4 * INSN_COST);
7611 format %{ "ldr $dst, $mem\t# class" %}
7612
7613 ins_encode(aarch64_enc_ldr(dst, mem));
7614
7615 ins_pipe(iload_reg_mem);
7616 %}
7617
7618 // Load Narrow Klass Pointer
7619 instruct loadNKlass(iRegNNoSp dst, memory4 mem)
7620 %{
7621 match(Set dst (LoadNKlass mem));
7622 predicate(!needs_acquiring_load(n));
7623
7624 ins_cost(4 * INSN_COST);
7625 format %{ "ldrw $dst, $mem\t# compressed class ptr" %}
7626
7627 ins_encode(aarch64_enc_ldrw(dst, mem));
7628
7629 ins_pipe(iload_reg_mem);
7630 %}
7631
7632 // Load Float
7633 instruct loadF(vRegF dst, memory4 mem)
7634 %{
7635 match(Set dst (LoadF mem));
7636 predicate(!needs_acquiring_load(n));
7637
7638 ins_cost(4 * INSN_COST);
7639 format %{ "ldrs $dst, $mem\t# float" %}
7640
7641 ins_encode( aarch64_enc_ldrs(dst, mem) );
7642
7643 ins_pipe(pipe_class_memory);
7644 %}
7645
7646 // Load Double
7647 instruct loadD(vRegD dst, memory8 mem)
7648 %{
7649 match(Set dst (LoadD mem));
7650 predicate(!needs_acquiring_load(n));
7651
7652 ins_cost(4 * INSN_COST);
7653 format %{ "ldrd $dst, $mem\t# double" %}
7654
7655 ins_encode( aarch64_enc_ldrd(dst, mem) );
7656
7657 ins_pipe(pipe_class_memory);
7658 %}
7659
7660
7661 // Load Int Constant
7662 instruct loadConI(iRegINoSp dst, immI src)
7663 %{
7664 match(Set dst src);
7665
7666 ins_cost(INSN_COST);
7667 format %{ "mov $dst, $src\t# int" %}
7668
7669 ins_encode( aarch64_enc_movw_imm(dst, src) );
7670
7671 ins_pipe(ialu_imm);
7672 %}
7673
7674 // Load Long Constant
7675 instruct loadConL(iRegLNoSp dst, immL src)
7676 %{
7677 match(Set dst src);
7678
7679 ins_cost(INSN_COST);
7680 format %{ "mov $dst, $src\t# long" %}
7681
7682 ins_encode( aarch64_enc_mov_imm(dst, src) );
7683
7684 ins_pipe(ialu_imm);
7685 %}
7686
7687 // Load Pointer Constant
7688
7689 instruct loadConP(iRegPNoSp dst, immP con)
7690 %{
7691 match(Set dst con);
7692
7693 ins_cost(INSN_COST * 4);
7694 format %{
7695 "mov $dst, $con\t# ptr\n\t"
7696 %}
7697
7698 ins_encode(aarch64_enc_mov_p(dst, con));
7699
7700 ins_pipe(ialu_imm);
7701 %}
7702
7703 // Load Null Pointer Constant
7704
7705 instruct loadConP0(iRegPNoSp dst, immP0 con)
7706 %{
7707 match(Set dst con);
7708
7709 ins_cost(INSN_COST);
7710 format %{ "mov $dst, $con\t# NULL ptr" %}
7711
7712 ins_encode(aarch64_enc_mov_p0(dst, con));
7713
7714 ins_pipe(ialu_imm);
7715 %}
7716
7717 // Load Pointer Constant One
7718
7719 instruct loadConP1(iRegPNoSp dst, immP_1 con)
7720 %{
7721 match(Set dst con);
7722
7723 ins_cost(INSN_COST);
7724 format %{ "mov $dst, $con\t# NULL ptr" %}
7725
7726 ins_encode(aarch64_enc_mov_p1(dst, con));
7727
7728 ins_pipe(ialu_imm);
7729 %}
7730
7731 // Load Byte Map Base Constant
7732
7733 instruct loadByteMapBase(iRegPNoSp dst, immByteMapBase con)
7734 %{
7735 match(Set dst con);
7736
7737 ins_cost(INSN_COST);
7738 format %{ "adr $dst, $con\t# Byte Map Base" %}
7739
7740 ins_encode(aarch64_enc_mov_byte_map_base(dst, con));
7741
7742 ins_pipe(ialu_imm);
7743 %}
7744
7745 // Load Narrow Pointer Constant
7746
7747 instruct loadConN(iRegNNoSp dst, immN con)
7748 %{
7749 match(Set dst con);
7750
7751 ins_cost(INSN_COST * 4);
7752 format %{ "mov $dst, $con\t# compressed ptr" %}
7753
7754 ins_encode(aarch64_enc_mov_n(dst, con));
7755
7756 ins_pipe(ialu_imm);
7757 %}
7758
7759 // Load Narrow Null Pointer Constant
7760
7761 instruct loadConN0(iRegNNoSp dst, immN0 con)
7762 %{
7763 match(Set dst con);
7764
7765 ins_cost(INSN_COST);
7766 format %{ "mov $dst, $con\t# compressed NULL ptr" %}
7767
7768 ins_encode(aarch64_enc_mov_n0(dst, con));
7769
7770 ins_pipe(ialu_imm);
7771 %}
7772
7773 // Load Narrow Klass Constant
7774
7775 instruct loadConNKlass(iRegNNoSp dst, immNKlass con)
7776 %{
7777 match(Set dst con);
7778
7779 ins_cost(INSN_COST);
7780 format %{ "mov $dst, $con\t# compressed klass ptr" %}
7781
7782 ins_encode(aarch64_enc_mov_nk(dst, con));
7783
7784 ins_pipe(ialu_imm);
7785 %}
7786
7787 // Load Packed Float Constant
7788
7789 instruct loadConF_packed(vRegF dst, immFPacked con) %{
7790 match(Set dst con);
7791 ins_cost(INSN_COST * 4);
7792 format %{ "fmovs $dst, $con"%}
7793 ins_encode %{
7794 __ fmovs(as_FloatRegister($dst$$reg), (double)$con$$constant);
7795 %}
7796
7797 ins_pipe(fp_imm_s);
7798 %}
7799
7800 // Load Float Constant
7801
7802 instruct loadConF(vRegF dst, immF con) %{
7803 match(Set dst con);
7804
7805 ins_cost(INSN_COST * 4);
7806
7807 format %{
7808 "ldrs $dst, [$constantaddress]\t# load from constant table: float=$con\n\t"
7809 %}
7810
7811 ins_encode %{
7812 __ ldrs(as_FloatRegister($dst$$reg), $constantaddress($con));
7813 %}
7814
7815 ins_pipe(fp_load_constant_s);
7816 %}
7817
7818 // Load Packed Double Constant
7819
7820 instruct loadConD_packed(vRegD dst, immDPacked con) %{
7821 match(Set dst con);
7822 ins_cost(INSN_COST);
7823 format %{ "fmovd $dst, $con"%}
7824 ins_encode %{
7825 __ fmovd(as_FloatRegister($dst$$reg), $con$$constant);
7826 %}
7827
7828 ins_pipe(fp_imm_d);
7829 %}
7830
7831 // Load Double Constant
7832
7833 instruct loadConD(vRegD dst, immD con) %{
7834 match(Set dst con);
7835
7836 ins_cost(INSN_COST * 5);
7837 format %{
7838 "ldrd $dst, [$constantaddress]\t# load from constant table: float=$con\n\t"
7839 %}
7840
7841 ins_encode %{
7842 __ ldrd(as_FloatRegister($dst$$reg), $constantaddress($con));
7843 %}
7844
7845 ins_pipe(fp_load_constant_d);
7846 %}
7847
7848 // Store Instructions
7849
7850 // Store CMS card-mark Immediate
7851 instruct storeimmCM0(immI0 zero, memory1 mem)
7852 %{
7853 match(Set mem (StoreCM mem zero));
7854
7855 ins_cost(INSN_COST);
7856 format %{ "storestore (elided)\n\t"
7857 "strb zr, $mem\t# byte" %}
7858
7859 ins_encode(aarch64_enc_strb0(mem));
7860
7861 ins_pipe(istore_mem);
7862 %}
7863
7864 // Store CMS card-mark Immediate with intervening StoreStore
7865 // needed when using CMS with no conditional card marking
7866 instruct storeimmCM0_ordered(immI0 zero, memory1 mem)
7867 %{
7868 match(Set mem (StoreCM mem zero));
7869
7870 ins_cost(INSN_COST * 2);
7871 format %{ "storestore\n\t"
7872 "dmb ishst"
7873 "\n\tstrb zr, $mem\t# byte" %}
7874
7875 ins_encode(aarch64_enc_strb0_ordered(mem));
7876
7877 ins_pipe(istore_mem);
7878 %}
7879
7880 // Store Byte
7881 instruct storeB(iRegIorL2I src, memory1 mem)
7882 %{
7883 match(Set mem (StoreB mem src));
7884 predicate(!needs_releasing_store(n));
7885
7886 ins_cost(INSN_COST);
7887 format %{ "strb $src, $mem\t# byte" %}
7888
7889 ins_encode(aarch64_enc_strb(src, mem));
7890
7891 ins_pipe(istore_reg_mem);
7892 %}
7893
7894
7895 instruct storeimmB0(immI0 zero, memory1 mem)
7896 %{
7897 match(Set mem (StoreB mem zero));
7898 predicate(!needs_releasing_store(n));
7899
7900 ins_cost(INSN_COST);
7901 format %{ "strb rscractch2, $mem\t# byte" %}
7902
7903 ins_encode(aarch64_enc_strb0(mem));
7904
7905 ins_pipe(istore_mem);
7906 %}
7907
7908 // Store Char/Short
7909 instruct storeC(iRegIorL2I src, memory2 mem)
7910 %{
7911 match(Set mem (StoreC mem src));
7912 predicate(!needs_releasing_store(n));
7913
7914 ins_cost(INSN_COST);
7915 format %{ "strh $src, $mem\t# short" %}
7916
7917 ins_encode(aarch64_enc_strh(src, mem));
7918
7919 ins_pipe(istore_reg_mem);
7920 %}
7921
7922 instruct storeimmC0(immI0 zero, memory2 mem)
7923 %{
7924 match(Set mem (StoreC mem zero));
7925 predicate(!needs_releasing_store(n));
7926
7927 ins_cost(INSN_COST);
7928 format %{ "strh zr, $mem\t# short" %}
7929
7930 ins_encode(aarch64_enc_strh0(mem));
7931
7932 ins_pipe(istore_mem);
7933 %}
7934
7935 // Store Integer
7936
7937 instruct storeI(iRegIorL2I src, memory4 mem)
7938 %{
7939 match(Set mem(StoreI mem src));
7940 predicate(!needs_releasing_store(n));
7941
7942 ins_cost(INSN_COST);
7943 format %{ "strw $src, $mem\t# int" %}
7944
7945 ins_encode(aarch64_enc_strw(src, mem));
7946
7947 ins_pipe(istore_reg_mem);
7948 %}
7949
7950 instruct storeimmI0(immI0 zero, memory4 mem)
7951 %{
7952 match(Set mem(StoreI mem zero));
7953 predicate(!needs_releasing_store(n));
7954
7955 ins_cost(INSN_COST);
7956 format %{ "strw zr, $mem\t# int" %}
7957
7958 ins_encode(aarch64_enc_strw0(mem));
7959
7960 ins_pipe(istore_mem);
7961 %}
7962
7963 // Store Long (64 bit signed)
7964 instruct storeL(iRegL src, memory8 mem)
7965 %{
7966 match(Set mem (StoreL mem src));
7967 predicate(!needs_releasing_store(n));
7968
7969 ins_cost(INSN_COST);
7970 format %{ "str $src, $mem\t# int" %}
7971
7972 ins_encode(aarch64_enc_str(src, mem));
7973
7974 ins_pipe(istore_reg_mem);
7975 %}
7976
7977 // Store Long (64 bit signed)
7978 instruct storeimmL0(immL0 zero, memory8 mem)
7979 %{
7980 match(Set mem (StoreL mem zero));
7981 predicate(!needs_releasing_store(n));
7982
7983 ins_cost(INSN_COST);
7984 format %{ "str zr, $mem\t# int" %}
7985
7986 ins_encode(aarch64_enc_str0(mem));
7987
7988 ins_pipe(istore_mem);
7989 %}
7990
7991 // Store Pointer
7992 instruct storeP(iRegP src, memory8 mem)
7993 %{
7994 match(Set mem (StoreP mem src));
7995 predicate(!needs_releasing_store(n));
7996
7997 ins_cost(INSN_COST);
7998 format %{ "str $src, $mem\t# ptr" %}
7999
8000 ins_encode(aarch64_enc_str(src, mem));
8001
8002 ins_pipe(istore_reg_mem);
8003 %}
8004
8005 // Store Pointer
8006 instruct storeimmP0(immP0 zero, memory8 mem)
8007 %{
8008 match(Set mem (StoreP mem zero));
8009 predicate(!needs_releasing_store(n));
8010
8011 ins_cost(INSN_COST);
8012 format %{ "str zr, $mem\t# ptr" %}
8013
8014 ins_encode(aarch64_enc_str0(mem));
8015
8016 ins_pipe(istore_mem);
8017 %}
8018
8019 // Store Compressed Pointer
8020 instruct storeN(iRegN src, memory4 mem)
8021 %{
8022 match(Set mem (StoreN mem src));
8023 predicate(!needs_releasing_store(n));
8024
8025 ins_cost(INSN_COST);
8026 format %{ "strw $src, $mem\t# compressed ptr" %}
8027
8028 ins_encode(aarch64_enc_strw(src, mem));
8029
8030 ins_pipe(istore_reg_mem);
8031 %}
8032
8033 instruct storeImmN0(immN0 zero, memory4 mem)
8034 %{
8035 match(Set mem (StoreN mem zero));
8036 predicate(!needs_releasing_store(n));
8037
8038 ins_cost(INSN_COST);
8039 format %{ "strw zr, $mem\t# compressed ptr" %}
8040
8041 ins_encode(aarch64_enc_strw0(mem));
8042
8043 ins_pipe(istore_mem);
8044 %}
8045
8046 // Store Float
8047 instruct storeF(vRegF src, memory4 mem)
8048 %{
8049 match(Set mem (StoreF mem src));
8050 predicate(!needs_releasing_store(n));
8051
8052 ins_cost(INSN_COST);
8053 format %{ "strs $src, $mem\t# float" %}
8054
8055 ins_encode( aarch64_enc_strs(src, mem) );
8056
8057 ins_pipe(pipe_class_memory);
8058 %}
8059
8060 // TODO
8061 // implement storeImmF0 and storeFImmPacked
8062
8063 // Store Double
8064 instruct storeD(vRegD src, memory8 mem)
8065 %{
8066 match(Set mem (StoreD mem src));
8067 predicate(!needs_releasing_store(n));
8068
8069 ins_cost(INSN_COST);
8070 format %{ "strd $src, $mem\t# double" %}
8071
8072 ins_encode( aarch64_enc_strd(src, mem) );
8073
8074 ins_pipe(pipe_class_memory);
8075 %}
8076
8077 // Store Compressed Klass Pointer
8078 instruct storeNKlass(iRegN src, memory4 mem)
8079 %{
8080 predicate(!needs_releasing_store(n));
8081 match(Set mem (StoreNKlass mem src));
8082
8083 ins_cost(INSN_COST);
8084 format %{ "strw $src, $mem\t# compressed klass ptr" %}
8085
8086 ins_encode(aarch64_enc_strw(src, mem));
8087
8088 ins_pipe(istore_reg_mem);
8089 %}
8090
8091 // TODO
8092 // implement storeImmD0 and storeDImmPacked
8093
8094 // prefetch instructions
8095 // Must be safe to execute with invalid address (cannot fault).
8096
8097 instruct prefetchalloc( memory8 mem ) %{
8098 match(PrefetchAllocation mem);
8099
8100 ins_cost(INSN_COST);
8101 format %{ "prfm $mem, PSTL1KEEP\t# Prefetch into level 1 cache write keep" %}
8102
8103 ins_encode( aarch64_enc_prefetchw(mem) );
8104
8105 ins_pipe(iload_prefetch);
8106 %}
8107
8108 // ---------------- volatile loads and stores ----------------
8109
8110 // Load Byte (8 bit signed)
8111 instruct loadB_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
8112 %{
8113 match(Set dst (LoadB mem));
8114
8115 ins_cost(VOLATILE_REF_COST);
8116 format %{ "ldarsb $dst, $mem\t# byte" %}
8117
8118 ins_encode(aarch64_enc_ldarsb(dst, mem));
8119
8120 ins_pipe(pipe_serial);
8121 %}
8122
8123 // Load Byte (8 bit signed) into long
8124 instruct loadB2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
8125 %{
8126 match(Set dst (ConvI2L (LoadB mem)));
8127
8128 ins_cost(VOLATILE_REF_COST);
8129 format %{ "ldarsb $dst, $mem\t# byte" %}
8130
8131 ins_encode(aarch64_enc_ldarsb(dst, mem));
8132
8133 ins_pipe(pipe_serial);
8134 %}
8135
8136 // Load Byte (8 bit unsigned)
8137 instruct loadUB_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
8138 %{
8139 match(Set dst (LoadUB mem));
8140
8141 ins_cost(VOLATILE_REF_COST);
8142 format %{ "ldarb $dst, $mem\t# byte" %}
8143
8144 ins_encode(aarch64_enc_ldarb(dst, mem));
8145
8146 ins_pipe(pipe_serial);
8147 %}
8148
8149 // Load Byte (8 bit unsigned) into long
8150 instruct loadUB2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
8151 %{
8152 match(Set dst (ConvI2L (LoadUB mem)));
8153
8154 ins_cost(VOLATILE_REF_COST);
8155 format %{ "ldarb $dst, $mem\t# byte" %}
8156
8157 ins_encode(aarch64_enc_ldarb(dst, mem));
8158
8159 ins_pipe(pipe_serial);
8160 %}
8161
8162 // Load Short (16 bit signed)
8163 instruct loadS_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
8164 %{
8165 match(Set dst (LoadS mem));
8166
8167 ins_cost(VOLATILE_REF_COST);
8168 format %{ "ldarshw $dst, $mem\t# short" %}
8169
8170 ins_encode(aarch64_enc_ldarshw(dst, mem));
8171
8172 ins_pipe(pipe_serial);
8173 %}
8174
8175 instruct loadUS_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
8176 %{
8177 match(Set dst (LoadUS mem));
8178
8179 ins_cost(VOLATILE_REF_COST);
8180 format %{ "ldarhw $dst, $mem\t# short" %}
8181
8182 ins_encode(aarch64_enc_ldarhw(dst, mem));
8183
8184 ins_pipe(pipe_serial);
8185 %}
8186
8187 // Load Short/Char (16 bit unsigned) into long
8188 instruct loadUS2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
8189 %{
8190 match(Set dst (ConvI2L (LoadUS mem)));
8191
8192 ins_cost(VOLATILE_REF_COST);
8193 format %{ "ldarh $dst, $mem\t# short" %}
8194
8195 ins_encode(aarch64_enc_ldarh(dst, mem));
8196
8197 ins_pipe(pipe_serial);
8198 %}
8199
8200 // Load Short/Char (16 bit signed) into long
8201 instruct loadS2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
8202 %{
8203 match(Set dst (ConvI2L (LoadS mem)));
8204
8205 ins_cost(VOLATILE_REF_COST);
8206 format %{ "ldarh $dst, $mem\t# short" %}
8207
8208 ins_encode(aarch64_enc_ldarsh(dst, mem));
8209
8210 ins_pipe(pipe_serial);
8211 %}
8212
8213 // Load Integer (32 bit signed)
8214 instruct loadI_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
8215 %{
8216 match(Set dst (LoadI mem));
8217
8218 ins_cost(VOLATILE_REF_COST);
8219 format %{ "ldarw $dst, $mem\t# int" %}
8220
8221 ins_encode(aarch64_enc_ldarw(dst, mem));
8222
8223 ins_pipe(pipe_serial);
8224 %}
8225
8226 // Load Integer (32 bit unsigned) into long
8227 instruct loadUI2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem, immL_32bits mask)
8228 %{
8229 match(Set dst (AndL (ConvI2L (LoadI mem)) mask));
8230
8231 ins_cost(VOLATILE_REF_COST);
8232 format %{ "ldarw $dst, $mem\t# int" %}
8233
8234 ins_encode(aarch64_enc_ldarw(dst, mem));
8235
8236 ins_pipe(pipe_serial);
8237 %}
8238
8239 // Load Long (64 bit signed)
8240 instruct loadL_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
8241 %{
8242 match(Set dst (LoadL mem));
8243
8244 ins_cost(VOLATILE_REF_COST);
8245 format %{ "ldar $dst, $mem\t# int" %}
8246
8247 ins_encode(aarch64_enc_ldar(dst, mem));
8248
8249 ins_pipe(pipe_serial);
8250 %}
8251
8252 // Load Pointer
8253 instruct loadP_volatile(iRegPNoSp dst, /* sync_memory*/indirect mem)
8254 %{
8255 match(Set dst (LoadP mem));
8256 predicate(n->as_Load()->barrier_data() == 0);
8257
8258 ins_cost(VOLATILE_REF_COST);
8259 format %{ "ldar $dst, $mem\t# ptr" %}
8260
8261 ins_encode(aarch64_enc_ldar(dst, mem));
8262
8263 ins_pipe(pipe_serial);
8264 %}
8265
8266 // Load Compressed Pointer
8267 instruct loadN_volatile(iRegNNoSp dst, /* sync_memory*/indirect mem)
8268 %{
8269 match(Set dst (LoadN mem));
8270
8271 ins_cost(VOLATILE_REF_COST);
8272 format %{ "ldarw $dst, $mem\t# compressed ptr" %}
8273
8274 ins_encode(aarch64_enc_ldarw(dst, mem));
8275
8276 ins_pipe(pipe_serial);
8277 %}
8278
8279 // Load Float
8280 instruct loadF_volatile(vRegF dst, /* sync_memory*/indirect mem)
8281 %{
8282 match(Set dst (LoadF mem));
8283
8284 ins_cost(VOLATILE_REF_COST);
8285 format %{ "ldars $dst, $mem\t# float" %}
8286
8287 ins_encode( aarch64_enc_fldars(dst, mem) );
8288
8289 ins_pipe(pipe_serial);
8290 %}
8291
8292 // Load Double
8293 instruct loadD_volatile(vRegD dst, /* sync_memory*/indirect mem)
8294 %{
8295 match(Set dst (LoadD mem));
8296
8297 ins_cost(VOLATILE_REF_COST);
8298 format %{ "ldard $dst, $mem\t# double" %}
8299
8300 ins_encode( aarch64_enc_fldard(dst, mem) );
8301
8302 ins_pipe(pipe_serial);
8303 %}
8304
8305 // Store Byte
8306 instruct storeB_volatile(iRegIorL2I src, /* sync_memory*/indirect mem)
8307 %{
8308 match(Set mem (StoreB mem src));
8309
8310 ins_cost(VOLATILE_REF_COST);
8311 format %{ "stlrb $src, $mem\t# byte" %}
8312
8313 ins_encode(aarch64_enc_stlrb(src, mem));
8314
8315 ins_pipe(pipe_class_memory);
8316 %}
8317
8318 instruct storeimmB0_volatile(immI0 zero, /* sync_memory*/indirect mem)
8319 %{
8320 match(Set mem (StoreB mem zero));
8321
8322 ins_cost(VOLATILE_REF_COST);
8323 format %{ "stlrb zr, $mem\t# byte" %}
8324
8325 ins_encode(aarch64_enc_stlrb0(mem));
8326
8327 ins_pipe(pipe_class_memory);
8328 %}
8329
8330 // Store Char/Short
8331 instruct storeC_volatile(iRegIorL2I src, /* sync_memory*/indirect mem)
8332 %{
8333 match(Set mem (StoreC mem src));
8334
8335 ins_cost(VOLATILE_REF_COST);
8336 format %{ "stlrh $src, $mem\t# short" %}
8337
8338 ins_encode(aarch64_enc_stlrh(src, mem));
8339
8340 ins_pipe(pipe_class_memory);
8341 %}
8342
8343 instruct storeimmC0_volatile(immI0 zero, /* sync_memory*/indirect mem)
8344 %{
8345 match(Set mem (StoreC mem zero));
8346
8347 ins_cost(VOLATILE_REF_COST);
8348 format %{ "stlrh zr, $mem\t# short" %}
8349
8350 ins_encode(aarch64_enc_stlrh0(mem));
8351
8352 ins_pipe(pipe_class_memory);
8353 %}
8354
8355 // Store Integer
8356
8357 instruct storeI_volatile(iRegIorL2I src, /* sync_memory*/indirect mem)
8358 %{
8359 match(Set mem(StoreI mem src));
8360
8361 ins_cost(VOLATILE_REF_COST);
8362 format %{ "stlrw $src, $mem\t# int" %}
8363
8364 ins_encode(aarch64_enc_stlrw(src, mem));
8365
8366 ins_pipe(pipe_class_memory);
8367 %}
8368
8369 instruct storeimmI0_volatile(immI0 zero, /* sync_memory*/indirect mem)
8370 %{
8371 match(Set mem(StoreI mem zero));
8372
8373 ins_cost(VOLATILE_REF_COST);
8374 format %{ "stlrw zr, $mem\t# int" %}
8375
8376 ins_encode(aarch64_enc_stlrw0(mem));
8377
8378 ins_pipe(pipe_class_memory);
8379 %}
8380
8381 // Store Long (64 bit signed)
8382 instruct storeL_volatile(iRegL src, /* sync_memory*/indirect mem)
8383 %{
8384 match(Set mem (StoreL mem src));
8385
8386 ins_cost(VOLATILE_REF_COST);
8387 format %{ "stlr $src, $mem\t# int" %}
8388
8389 ins_encode(aarch64_enc_stlr(src, mem));
8390
8391 ins_pipe(pipe_class_memory);
8392 %}
8393
8394 instruct storeimmL0_volatile(immL0 zero, /* sync_memory*/indirect mem)
8395 %{
8396 match(Set mem (StoreL mem zero));
8397
8398 ins_cost(VOLATILE_REF_COST);
8399 format %{ "stlr zr, $mem\t# int" %}
8400
8401 ins_encode(aarch64_enc_stlr0(mem));
8402
8403 ins_pipe(pipe_class_memory);
8404 %}
8405
8406 // Store Pointer
8407 instruct storeP_volatile(iRegP src, /* sync_memory*/indirect mem)
8408 %{
8409 match(Set mem (StoreP mem src));
8410
8411 ins_cost(VOLATILE_REF_COST);
8412 format %{ "stlr $src, $mem\t# ptr" %}
8413
8414 ins_encode(aarch64_enc_stlr(src, mem));
8415
8416 ins_pipe(pipe_class_memory);
8417 %}
8418
8419 instruct storeimmP0_volatile(immP0 zero, /* sync_memory*/indirect mem)
8420 %{
8421 match(Set mem (StoreP mem zero));
8422
8423 ins_cost(VOLATILE_REF_COST);
8424 format %{ "stlr zr, $mem\t# ptr" %}
8425
8426 ins_encode(aarch64_enc_stlr0(mem));
8427
8428 ins_pipe(pipe_class_memory);
8429 %}
8430
8431 // Store Compressed Pointer
8432 instruct storeN_volatile(iRegN src, /* sync_memory*/indirect mem)
8433 %{
8434 match(Set mem (StoreN mem src));
8435
8436 ins_cost(VOLATILE_REF_COST);
8437 format %{ "stlrw $src, $mem\t# compressed ptr" %}
8438
8439 ins_encode(aarch64_enc_stlrw(src, mem));
8440
8441 ins_pipe(pipe_class_memory);
8442 %}
8443
8444 instruct storeimmN0_volatile(immN0 zero, /* sync_memory*/indirect mem)
8445 %{
8446 match(Set mem (StoreN mem zero));
8447
8448 ins_cost(VOLATILE_REF_COST);
8449 format %{ "stlrw zr, $mem\t# compressed ptr" %}
8450
8451 ins_encode(aarch64_enc_stlrw0(mem));
8452
8453 ins_pipe(pipe_class_memory);
8454 %}
8455
8456 // Store Float
8457 instruct storeF_volatile(vRegF src, /* sync_memory*/indirect mem)
8458 %{
8459 match(Set mem (StoreF mem src));
8460
8461 ins_cost(VOLATILE_REF_COST);
8462 format %{ "stlrs $src, $mem\t# float" %}
8463
8464 ins_encode( aarch64_enc_fstlrs(src, mem) );
8465
8466 ins_pipe(pipe_class_memory);
8467 %}
8468
8469 // TODO
8470 // implement storeImmF0 and storeFImmPacked
8471
8472 // Store Double
8473 instruct storeD_volatile(vRegD src, /* sync_memory*/indirect mem)
8474 %{
8475 match(Set mem (StoreD mem src));
8476
8477 ins_cost(VOLATILE_REF_COST);
8478 format %{ "stlrd $src, $mem\t# double" %}
8479
8480 ins_encode( aarch64_enc_fstlrd(src, mem) );
8481
8482 ins_pipe(pipe_class_memory);
8483 %}
8484
8485 // ---------------- end of volatile loads and stores ----------------
8486
8487 instruct cacheWB(indirect addr)
8488 %{
8489 predicate(VM_Version::supports_data_cache_line_flush());
8490 match(CacheWB addr);
8491
8492 ins_cost(100);
8493 format %{"cache wb $addr" %}
8494 ins_encode %{
8495 assert($addr->index_position() < 0, "should be");
8496 assert($addr$$disp == 0, "should be");
8497 __ cache_wb(Address($addr$$base$$Register, 0));
8498 %}
8499 ins_pipe(pipe_slow); // XXX
8500 %}
8501
8502 instruct cacheWBPreSync()
8503 %{
8504 predicate(VM_Version::supports_data_cache_line_flush());
8505 match(CacheWBPreSync);
8506
8507 ins_cost(100);
8508 format %{"cache wb presync" %}
8509 ins_encode %{
8510 __ cache_wbsync(true);
8511 %}
8512 ins_pipe(pipe_slow); // XXX
8513 %}
8514
8515 instruct cacheWBPostSync()
8516 %{
8517 predicate(VM_Version::supports_data_cache_line_flush());
8518 match(CacheWBPostSync);
8519
8520 ins_cost(100);
8521 format %{"cache wb postsync" %}
8522 ins_encode %{
8523 __ cache_wbsync(false);
8524 %}
8525 ins_pipe(pipe_slow); // XXX
8526 %}
8527
8528 // ============================================================================
8529 // BSWAP Instructions
8530
8531 instruct bytes_reverse_int(iRegINoSp dst, iRegIorL2I src) %{
8532 match(Set dst (ReverseBytesI src));
8533
8534 ins_cost(INSN_COST);
8535 format %{ "revw $dst, $src" %}
8536
8537 ins_encode %{
8538 __ revw(as_Register($dst$$reg), as_Register($src$$reg));
8539 %}
8540
8541 ins_pipe(ialu_reg);
8542 %}
8543
8544 instruct bytes_reverse_long(iRegLNoSp dst, iRegL src) %{
8545 match(Set dst (ReverseBytesL src));
8546
8547 ins_cost(INSN_COST);
8548 format %{ "rev $dst, $src" %}
8549
8550 ins_encode %{
8551 __ rev(as_Register($dst$$reg), as_Register($src$$reg));
8552 %}
8553
8554 ins_pipe(ialu_reg);
8555 %}
8556
8557 instruct bytes_reverse_unsigned_short(iRegINoSp dst, iRegIorL2I src) %{
8558 match(Set dst (ReverseBytesUS src));
8559
8560 ins_cost(INSN_COST);
8561 format %{ "rev16w $dst, $src" %}
8562
8563 ins_encode %{
8564 __ rev16w(as_Register($dst$$reg), as_Register($src$$reg));
8565 %}
8566
8567 ins_pipe(ialu_reg);
8568 %}
8569
8570 instruct bytes_reverse_short(iRegINoSp dst, iRegIorL2I src) %{
8571 match(Set dst (ReverseBytesS src));
8572
8573 ins_cost(INSN_COST);
8574 format %{ "rev16w $dst, $src\n\t"
8575 "sbfmw $dst, $dst, #0, #15" %}
8576
8577 ins_encode %{
8578 __ rev16w(as_Register($dst$$reg), as_Register($src$$reg));
8579 __ sbfmw(as_Register($dst$$reg), as_Register($dst$$reg), 0U, 15U);
8580 %}
8581
8582 ins_pipe(ialu_reg);
8583 %}
8584
8585 // ============================================================================
8586 // Zero Count Instructions
8587
8588 instruct countLeadingZerosI(iRegINoSp dst, iRegIorL2I src) %{
8589 match(Set dst (CountLeadingZerosI src));
8590
8591 ins_cost(INSN_COST);
8592 format %{ "clzw $dst, $src" %}
8593 ins_encode %{
8594 __ clzw(as_Register($dst$$reg), as_Register($src$$reg));
8595 %}
8596
8597 ins_pipe(ialu_reg);
8598 %}
8599
8600 instruct countLeadingZerosL(iRegINoSp dst, iRegL src) %{
8601 match(Set dst (CountLeadingZerosL src));
8602
8603 ins_cost(INSN_COST);
8604 format %{ "clz $dst, $src" %}
8605 ins_encode %{
8606 __ clz(as_Register($dst$$reg), as_Register($src$$reg));
8607 %}
8608
8609 ins_pipe(ialu_reg);
8610 %}
8611
8612 instruct countTrailingZerosI(iRegINoSp dst, iRegIorL2I src) %{
8613 match(Set dst (CountTrailingZerosI src));
8614
8615 ins_cost(INSN_COST * 2);
8616 format %{ "rbitw $dst, $src\n\t"
8617 "clzw $dst, $dst" %}
8618 ins_encode %{
8619 __ rbitw(as_Register($dst$$reg), as_Register($src$$reg));
8620 __ clzw(as_Register($dst$$reg), as_Register($dst$$reg));
8621 %}
8622
8623 ins_pipe(ialu_reg);
8624 %}
8625
8626 instruct countTrailingZerosL(iRegINoSp dst, iRegL src) %{
8627 match(Set dst (CountTrailingZerosL src));
8628
8629 ins_cost(INSN_COST * 2);
8630 format %{ "rbit $dst, $src\n\t"
8631 "clz $dst, $dst" %}
8632 ins_encode %{
8633 __ rbit(as_Register($dst$$reg), as_Register($src$$reg));
8634 __ clz(as_Register($dst$$reg), as_Register($dst$$reg));
8635 %}
8636
8637 ins_pipe(ialu_reg);
8638 %}
8639
8640 //---------- Population Count Instructions -------------------------------------
8641 //
8642
8643 instruct popCountI(iRegINoSp dst, iRegIorL2I src, vRegF tmp) %{
8644 predicate(UsePopCountInstruction);
8645 match(Set dst (PopCountI src));
8646 effect(TEMP tmp);
8647 ins_cost(INSN_COST * 13);
8648
8649 format %{ "movw $src, $src\n\t"
8650 "mov $tmp, $src\t# vector (1D)\n\t"
8651 "cnt $tmp, $tmp\t# vector (8B)\n\t"
8652 "addv $tmp, $tmp\t# vector (8B)\n\t"
8653 "mov $dst, $tmp\t# vector (1D)" %}
8654 ins_encode %{
8655 __ movw($src$$Register, $src$$Register); // ensure top 32 bits 0
8656 __ mov($tmp$$FloatRegister, __ D, 0, $src$$Register);
8657 __ cnt($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
8658 __ addv($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
8659 __ mov($dst$$Register, $tmp$$FloatRegister, __ D, 0);
8660 %}
8661
8662 ins_pipe(pipe_class_default);
8663 %}
8664
8665 instruct popCountI_mem(iRegINoSp dst, memory4 mem, vRegF tmp) %{
8666 predicate(UsePopCountInstruction);
8667 match(Set dst (PopCountI (LoadI mem)));
8668 effect(TEMP tmp);
8669 ins_cost(INSN_COST * 13);
8670
8671 format %{ "ldrs $tmp, $mem\n\t"
8672 "cnt $tmp, $tmp\t# vector (8B)\n\t"
8673 "addv $tmp, $tmp\t# vector (8B)\n\t"
8674 "mov $dst, $tmp\t# vector (1D)" %}
8675 ins_encode %{
8676 FloatRegister tmp_reg = as_FloatRegister($tmp$$reg);
8677 loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrs, tmp_reg, $mem->opcode(),
8678 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
8679 __ cnt($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
8680 __ addv($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
8681 __ mov($dst$$Register, $tmp$$FloatRegister, __ D, 0);
8682 %}
8683
8684 ins_pipe(pipe_class_default);
8685 %}
8686
8687 // Note: Long.bitCount(long) returns an int.
8688 instruct popCountL(iRegINoSp dst, iRegL src, vRegD tmp) %{
8689 predicate(UsePopCountInstruction);
8690 match(Set dst (PopCountL src));
8691 effect(TEMP tmp);
8692 ins_cost(INSN_COST * 13);
8693
8694 format %{ "mov $tmp, $src\t# vector (1D)\n\t"
8695 "cnt $tmp, $tmp\t# vector (8B)\n\t"
8696 "addv $tmp, $tmp\t# vector (8B)\n\t"
8697 "mov $dst, $tmp\t# vector (1D)" %}
8698 ins_encode %{
8699 __ mov($tmp$$FloatRegister, __ D, 0, $src$$Register);
8700 __ cnt($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
8701 __ addv($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
8702 __ mov($dst$$Register, $tmp$$FloatRegister, __ D, 0);
8703 %}
8704
8705 ins_pipe(pipe_class_default);
8706 %}
8707
8708 instruct popCountL_mem(iRegINoSp dst, memory8 mem, vRegD tmp) %{
8709 predicate(UsePopCountInstruction);
8710 match(Set dst (PopCountL (LoadL mem)));
8711 effect(TEMP tmp);
8712 ins_cost(INSN_COST * 13);
8713
8714 format %{ "ldrd $tmp, $mem\n\t"
8715 "cnt $tmp, $tmp\t# vector (8B)\n\t"
8716 "addv $tmp, $tmp\t# vector (8B)\n\t"
8717 "mov $dst, $tmp\t# vector (1D)" %}
8718 ins_encode %{
8719 FloatRegister tmp_reg = as_FloatRegister($tmp$$reg);
8720 loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrd, tmp_reg, $mem->opcode(),
8721 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
8722 __ cnt($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
8723 __ addv($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
8724 __ mov($dst$$Register, $tmp$$FloatRegister, __ D, 0);
8725 %}
8726
8727 ins_pipe(pipe_class_default);
8728 %}
8729
8730 // ============================================================================
8731 // MemBar Instruction
8732
8733 instruct load_fence() %{
8734 match(LoadFence);
8735 ins_cost(VOLATILE_REF_COST);
8736
8737 format %{ "load_fence" %}
8738
8739 ins_encode %{
8740 __ membar(Assembler::LoadLoad|Assembler::LoadStore);
8741 %}
8742 ins_pipe(pipe_serial);
8743 %}
8744
8745 instruct unnecessary_membar_acquire() %{
8746 predicate(unnecessary_acquire(n));
8747 match(MemBarAcquire);
8748 ins_cost(0);
8749
8750 format %{ "membar_acquire (elided)" %}
8751
8752 ins_encode %{
8753 __ block_comment("membar_acquire (elided)");
8754 %}
8755
8756 ins_pipe(pipe_class_empty);
8757 %}
8758
8759 instruct membar_acquire() %{
8760 match(MemBarAcquire);
8761 ins_cost(VOLATILE_REF_COST);
8762
8763 format %{ "membar_acquire\n\t"
8764 "dmb ish" %}
8765
8766 ins_encode %{
8767 __ block_comment("membar_acquire");
8768 __ membar(Assembler::LoadLoad|Assembler::LoadStore);
8769 %}
8770
8771 ins_pipe(pipe_serial);
8772 %}
8773
8774
8775 instruct membar_acquire_lock() %{
8776 match(MemBarAcquireLock);
8777 ins_cost(VOLATILE_REF_COST);
8778
8779 format %{ "membar_acquire_lock (elided)" %}
8780
8781 ins_encode %{
8782 __ block_comment("membar_acquire_lock (elided)");
8783 %}
8784
8785 ins_pipe(pipe_serial);
8786 %}
8787
8788 instruct store_fence() %{
8789 match(StoreFence);
8790 ins_cost(VOLATILE_REF_COST);
8791
8792 format %{ "store_fence" %}
8793
8794 ins_encode %{
8795 __ membar(Assembler::LoadStore|Assembler::StoreStore);
8796 %}
8797 ins_pipe(pipe_serial);
8798 %}
8799
8800 instruct unnecessary_membar_release() %{
8801 predicate(unnecessary_release(n));
8802 match(MemBarRelease);
8803 ins_cost(0);
8804
8805 format %{ "membar_release (elided)" %}
8806
8807 ins_encode %{
8808 __ block_comment("membar_release (elided)");
8809 %}
8810 ins_pipe(pipe_serial);
8811 %}
8812
8813 instruct membar_release() %{
8814 match(MemBarRelease);
8815 ins_cost(VOLATILE_REF_COST);
8816
8817 format %{ "membar_release\n\t"
8818 "dmb ish" %}
8819
8820 ins_encode %{
8821 __ block_comment("membar_release");
8822 __ membar(Assembler::LoadStore|Assembler::StoreStore);
8823 %}
8824 ins_pipe(pipe_serial);
8825 %}
8826
8827 instruct membar_storestore() %{
8828 match(MemBarStoreStore);
8829 match(StoreStoreFence);
8830 ins_cost(VOLATILE_REF_COST);
8831
8832 format %{ "MEMBAR-store-store" %}
8833
8834 ins_encode %{
8835 __ membar(Assembler::StoreStore);
8836 %}
8837 ins_pipe(pipe_serial);
8838 %}
8839
8840 instruct membar_release_lock() %{
8841 match(MemBarReleaseLock);
8842 ins_cost(VOLATILE_REF_COST);
8843
8844 format %{ "membar_release_lock (elided)" %}
8845
8846 ins_encode %{
8847 __ block_comment("membar_release_lock (elided)");
8848 %}
8849
8850 ins_pipe(pipe_serial);
8851 %}
8852
8853 instruct unnecessary_membar_volatile() %{
8854 predicate(unnecessary_volatile(n));
8855 match(MemBarVolatile);
8856 ins_cost(0);
8857
8858 format %{ "membar_volatile (elided)" %}
8859
8860 ins_encode %{
8861 __ block_comment("membar_volatile (elided)");
8862 %}
8863
8864 ins_pipe(pipe_serial);
8865 %}
8866
8867 instruct membar_volatile() %{
8868 match(MemBarVolatile);
8869 ins_cost(VOLATILE_REF_COST*100);
8870
8871 format %{ "membar_volatile\n\t"
8872 "dmb ish"%}
8873
8874 ins_encode %{
8875 __ block_comment("membar_volatile");
8876 __ membar(Assembler::StoreLoad);
8877 %}
8878
8879 ins_pipe(pipe_serial);
8880 %}
8881
8882 // ============================================================================
8883 // Cast/Convert Instructions
8884
8885 instruct castX2P(iRegPNoSp dst, iRegL src) %{
8886 match(Set dst (CastX2P src));
8887
8888 ins_cost(INSN_COST);
8889 format %{ "mov $dst, $src\t# long -> ptr" %}
8890
8891 ins_encode %{
8892 if ($dst$$reg != $src$$reg) {
8893 __ mov(as_Register($dst$$reg), as_Register($src$$reg));
8894 }
8895 %}
8896
8897 ins_pipe(ialu_reg);
8898 %}
8899
8900 instruct castP2X(iRegLNoSp dst, iRegP src) %{
8901 match(Set dst (CastP2X src));
8902
8903 ins_cost(INSN_COST);
8904 format %{ "mov $dst, $src\t# ptr -> long" %}
8905
8906 ins_encode %{
8907 if ($dst$$reg != $src$$reg) {
8908 __ mov(as_Register($dst$$reg), as_Register($src$$reg));
8909 }
8910 %}
8911
8912 ins_pipe(ialu_reg);
8913 %}
8914
8915 // Convert oop into int for vectors alignment masking
8916 instruct convP2I(iRegINoSp dst, iRegP src) %{
8917 match(Set dst (ConvL2I (CastP2X src)));
8918
8919 ins_cost(INSN_COST);
8920 format %{ "movw $dst, $src\t# ptr -> int" %}
8921 ins_encode %{
8922 __ movw($dst$$Register, $src$$Register);
8923 %}
8924
8925 ins_pipe(ialu_reg);
8926 %}
8927
8928 // Convert compressed oop into int for vectors alignment masking
8929 // in case of 32bit oops (heap < 4Gb).
8930 instruct convN2I(iRegINoSp dst, iRegN src)
8931 %{
8932 predicate(CompressedOops::shift() == 0);
8933 match(Set dst (ConvL2I (CastP2X (DecodeN src))));
8934
8935 ins_cost(INSN_COST);
8936 format %{ "mov dst, $src\t# compressed ptr -> int" %}
8937 ins_encode %{
8938 __ movw($dst$$Register, $src$$Register);
8939 %}
8940
8941 ins_pipe(ialu_reg);
8942 %}
8943
8944
8945 // Convert oop pointer into compressed form
8946 instruct encodeHeapOop(iRegNNoSp dst, iRegP src, rFlagsReg cr) %{
8947 predicate(n->bottom_type()->make_ptr()->ptr() != TypePtr::NotNull);
8948 match(Set dst (EncodeP src));
8949 effect(KILL cr);
8950 ins_cost(INSN_COST * 3);
8951 format %{ "encode_heap_oop $dst, $src" %}
8952 ins_encode %{
8953 Register s = $src$$Register;
8954 Register d = $dst$$Register;
8955 __ encode_heap_oop(d, s);
8956 %}
8957 ins_pipe(ialu_reg);
8958 %}
8959
8960 instruct encodeHeapOop_not_null(iRegNNoSp dst, iRegP src, rFlagsReg cr) %{
8961 predicate(n->bottom_type()->make_ptr()->ptr() == TypePtr::NotNull);
8962 match(Set dst (EncodeP src));
8963 ins_cost(INSN_COST * 3);
8964 format %{ "encode_heap_oop_not_null $dst, $src" %}
8965 ins_encode %{
8966 __ encode_heap_oop_not_null($dst$$Register, $src$$Register);
8967 %}
8968 ins_pipe(ialu_reg);
8969 %}
8970
8971 instruct decodeHeapOop(iRegPNoSp dst, iRegN src, rFlagsReg cr) %{
8972 predicate(n->bottom_type()->is_ptr()->ptr() != TypePtr::NotNull &&
8973 n->bottom_type()->is_ptr()->ptr() != TypePtr::Constant);
8974 match(Set dst (DecodeN src));
8975 ins_cost(INSN_COST * 3);
8976 format %{ "decode_heap_oop $dst, $src" %}
8977 ins_encode %{
8978 Register s = $src$$Register;
8979 Register d = $dst$$Register;
8980 __ decode_heap_oop(d, s);
8981 %}
8982 ins_pipe(ialu_reg);
8983 %}
8984
8985 instruct decodeHeapOop_not_null(iRegPNoSp dst, iRegN src, rFlagsReg cr) %{
8986 predicate(n->bottom_type()->is_ptr()->ptr() == TypePtr::NotNull ||
8987 n->bottom_type()->is_ptr()->ptr() == TypePtr::Constant);
8988 match(Set dst (DecodeN src));
8989 ins_cost(INSN_COST * 3);
8990 format %{ "decode_heap_oop_not_null $dst, $src" %}
8991 ins_encode %{
8992 Register s = $src$$Register;
8993 Register d = $dst$$Register;
8994 __ decode_heap_oop_not_null(d, s);
8995 %}
8996 ins_pipe(ialu_reg);
8997 %}
8998
8999 // n.b. AArch64 implementations of encode_klass_not_null and
9000 // decode_klass_not_null do not modify the flags register so, unlike
9001 // Intel, we don't kill CR as a side effect here
9002
9003 instruct encodeKlass_not_null(iRegNNoSp dst, iRegP src) %{
9004 match(Set dst (EncodePKlass src));
9005
9006 ins_cost(INSN_COST * 3);
9007 format %{ "encode_klass_not_null $dst,$src" %}
9008
9009 ins_encode %{
9010 Register src_reg = as_Register($src$$reg);
9011 Register dst_reg = as_Register($dst$$reg);
9012 __ encode_klass_not_null(dst_reg, src_reg);
9013 %}
9014
9015 ins_pipe(ialu_reg);
9016 %}
9017
9018 instruct decodeKlass_not_null(iRegPNoSp dst, iRegN src) %{
9019 match(Set dst (DecodeNKlass src));
9020
9021 ins_cost(INSN_COST * 3);
9022 format %{ "decode_klass_not_null $dst,$src" %}
9023
9024 ins_encode %{
9025 Register src_reg = as_Register($src$$reg);
9026 Register dst_reg = as_Register($dst$$reg);
9027 if (dst_reg != src_reg) {
9028 __ decode_klass_not_null(dst_reg, src_reg);
9029 } else {
9030 __ decode_klass_not_null(dst_reg);
9031 }
9032 %}
9033
9034 ins_pipe(ialu_reg);
9035 %}
9036
9037 instruct checkCastPP(iRegPNoSp dst)
9038 %{
9039 match(Set dst (CheckCastPP dst));
9040
9041 size(0);
9042 format %{ "# checkcastPP of $dst" %}
9043 ins_encode(/* empty encoding */);
9044 ins_pipe(pipe_class_empty);
9045 %}
9046
9047 instruct castPP(iRegPNoSp dst)
9048 %{
9049 match(Set dst (CastPP dst));
9050
9051 size(0);
9052 format %{ "# castPP of $dst" %}
9053 ins_encode(/* empty encoding */);
9054 ins_pipe(pipe_class_empty);
9055 %}
9056
9057 instruct castII(iRegI dst)
9058 %{
9059 match(Set dst (CastII dst));
9060
9061 size(0);
9062 format %{ "# castII of $dst" %}
9063 ins_encode(/* empty encoding */);
9064 ins_cost(0);
9065 ins_pipe(pipe_class_empty);
9066 %}
9067
9068 instruct castLL(iRegL dst)
9069 %{
9070 match(Set dst (CastLL dst));
9071
9072 size(0);
9073 format %{ "# castLL of $dst" %}
9074 ins_encode(/* empty encoding */);
9075 ins_cost(0);
9076 ins_pipe(pipe_class_empty);
9077 %}
9078
9079 instruct castFF(vRegF dst)
9080 %{
9081 match(Set dst (CastFF dst));
9082
9083 size(0);
9084 format %{ "# castFF of $dst" %}
9085 ins_encode(/* empty encoding */);
9086 ins_cost(0);
9087 ins_pipe(pipe_class_empty);
9088 %}
9089
9090 instruct castDD(vRegD dst)
9091 %{
9092 match(Set dst (CastDD dst));
9093
9094 size(0);
9095 format %{ "# castDD of $dst" %}
9096 ins_encode(/* empty encoding */);
9097 ins_cost(0);
9098 ins_pipe(pipe_class_empty);
9099 %}
9100
9101 instruct castVVD(vecD dst)
9102 %{
9103 match(Set dst (CastVV dst));
9104
9105 size(0);
9106 format %{ "# castVV of $dst" %}
9107 ins_encode(/* empty encoding */);
9108 ins_cost(0);
9109 ins_pipe(pipe_class_empty);
9110 %}
9111
9112 instruct castVVX(vecX dst)
9113 %{
9114 match(Set dst (CastVV dst));
9115
9116 size(0);
9117 format %{ "# castVV of $dst" %}
9118 ins_encode(/* empty encoding */);
9119 ins_cost(0);
9120 ins_pipe(pipe_class_empty);
9121 %}
9122
9123 instruct castVV(vReg dst)
9124 %{
9125 match(Set dst (CastVV dst));
9126
9127 size(0);
9128 format %{ "# castVV of $dst" %}
9129 ins_encode(/* empty encoding */);
9130 ins_cost(0);
9131 ins_pipe(pipe_class_empty);
9132 %}
9133
9134 instruct castVVMask(pRegGov dst)
9135 %{
9136 match(Set dst (CastVV dst));
9137
9138 size(0);
9139 format %{ "# castVV of $dst" %}
9140 ins_encode(/* empty encoding */);
9141 ins_cost(0);
9142 ins_pipe(pipe_class_empty);
9143 %}
9144
9145 // ============================================================================
9146 // Atomic operation instructions
9147 //
9148 // Intel and SPARC both implement Ideal Node LoadPLocked and
9149 // Store{PIL}Conditional instructions using a normal load for the
9150 // LoadPLocked and a CAS for the Store{PIL}Conditional.
9151 //
9152 // The ideal code appears only to use LoadPLocked/StorePLocked as a
9153 // pair to lock object allocations from Eden space when not using
9154 // TLABs.
9155 //
9156 // There does not appear to be a Load{IL}Locked Ideal Node and the
9157 // Ideal code appears to use Store{IL}Conditional as an alias for CAS
9158 // and to use StoreIConditional only for 32-bit and StoreLConditional
9159 // only for 64-bit.
9160 //
9161 // We implement LoadPLocked and StorePLocked instructions using,
9162 // respectively the AArch64 hw load-exclusive and store-conditional
9163 // instructions. Whereas we must implement each of
9164 // Store{IL}Conditional using a CAS which employs a pair of
9165 // instructions comprising a load-exclusive followed by a
9166 // store-conditional.
9167
9168
9169 // Locked-load (linked load) of the current heap-top
9170 // used when updating the eden heap top
9171 // implemented using ldaxr on AArch64
9172
9173 instruct loadPLocked(iRegPNoSp dst, indirect mem)
9174 %{
9175 match(Set dst (LoadPLocked mem));
9176
9177 ins_cost(VOLATILE_REF_COST);
9178
9179 format %{ "ldaxr $dst, $mem\t# ptr linked acquire" %}
9180
9181 ins_encode(aarch64_enc_ldaxr(dst, mem));
9182
9183 ins_pipe(pipe_serial);
9184 %}
9185
9186 // Conditional-store of the updated heap-top.
9187 // Used during allocation of the shared heap.
9188 // Sets flag (EQ) on success.
9189 // implemented using stlxr on AArch64.
9190
9191 instruct storePConditional(memory8 heap_top_ptr, iRegP oldval, iRegP newval, rFlagsReg cr)
9192 %{
9193 match(Set cr (StorePConditional heap_top_ptr (Binary oldval newval)));
9194
9195 ins_cost(VOLATILE_REF_COST);
9196
9197 // TODO
9198 // do we need to do a store-conditional release or can we just use a
9199 // plain store-conditional?
9200
9201 format %{
9202 "stlxr rscratch1, $newval, $heap_top_ptr\t# ptr cond release"
9203 "cmpw rscratch1, zr\t# EQ on successful write"
9204 %}
9205
9206 ins_encode(aarch64_enc_stlxr(newval, heap_top_ptr));
9207
9208 ins_pipe(pipe_serial);
9209 %}
9210
9211 instruct storeLConditional(indirect mem, iRegLNoSp oldval, iRegLNoSp newval, rFlagsReg cr)
9212 %{
9213 match(Set cr (StoreLConditional mem (Binary oldval newval)));
9214
9215 ins_cost(VOLATILE_REF_COST);
9216
9217 format %{
9218 "cmpxchg rscratch1, $mem, $oldval, $newval, $mem\t# if $mem == $oldval then $mem <-- $newval"
9219 "cmpw rscratch1, zr\t# EQ on successful write"
9220 %}
9221
9222 ins_encode(aarch64_enc_cmpxchg_acq(mem, oldval, newval));
9223
9224 ins_pipe(pipe_slow);
9225 %}
9226
9227 // storeIConditional also has acquire semantics, for no better reason
9228 // than matching storeLConditional. At the time of writing this
9229 // comment storeIConditional was not used anywhere by AArch64.
9230 instruct storeIConditional(indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr)
9231 %{
9232 match(Set cr (StoreIConditional mem (Binary oldval newval)));
9233
9234 ins_cost(VOLATILE_REF_COST);
9235
9236 format %{
9237 "cmpxchgw rscratch1, $mem, $oldval, $newval, $mem\t# if $mem == $oldval then $mem <-- $newval"
9238 "cmpw rscratch1, zr\t# EQ on successful write"
9239 %}
9240
9241 ins_encode(aarch64_enc_cmpxchgw_acq(mem, oldval, newval));
9242
9243 ins_pipe(pipe_slow);
9244 %}
9245
9246 // standard CompareAndSwapX when we are using barriers
9247 // these have higher priority than the rules selected by a predicate
9248
9249 // XXX No flag versions for CompareAndSwap{I,L,P,N} because matcher
9250 // can't match them
9251
9252 instruct compareAndSwapB(iRegINoSp res, indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr) %{
9253
9254 match(Set res (CompareAndSwapB mem (Binary oldval newval)));
9255 ins_cost(2 * VOLATILE_REF_COST);
9256
9257 effect(KILL cr);
9258
9259 format %{
9260 "cmpxchgb $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval"
9261 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
9262 %}
9263
9264 ins_encode(aarch64_enc_cmpxchgb(mem, oldval, newval),
9265 aarch64_enc_cset_eq(res));
9266
9267 ins_pipe(pipe_slow);
9268 %}
9269
9270 instruct compareAndSwapS(iRegINoSp res, indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr) %{
9271
9272 match(Set res (CompareAndSwapS mem (Binary oldval newval)));
9273 ins_cost(2 * VOLATILE_REF_COST);
9274
9275 effect(KILL cr);
9276
9277 format %{
9278 "cmpxchgs $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval"
9279 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
9280 %}
9281
9282 ins_encode(aarch64_enc_cmpxchgs(mem, oldval, newval),
9283 aarch64_enc_cset_eq(res));
9284
9285 ins_pipe(pipe_slow);
9286 %}
9287
9288 instruct compareAndSwapI(iRegINoSp res, indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr) %{
9289
9290 match(Set res (CompareAndSwapI mem (Binary oldval newval)));
9291 ins_cost(2 * VOLATILE_REF_COST);
9292
9293 effect(KILL cr);
9294
9295 format %{
9296 "cmpxchgw $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval"
9297 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
9298 %}
9299
9300 ins_encode(aarch64_enc_cmpxchgw(mem, oldval, newval),
9301 aarch64_enc_cset_eq(res));
9302
9303 ins_pipe(pipe_slow);
9304 %}
9305
9306 instruct compareAndSwapL(iRegINoSp res, indirect mem, iRegLNoSp oldval, iRegLNoSp newval, rFlagsReg cr) %{
9307
9308 match(Set res (CompareAndSwapL mem (Binary oldval newval)));
9309 ins_cost(2 * VOLATILE_REF_COST);
9310
9311 effect(KILL cr);
9312
9313 format %{
9314 "cmpxchg $mem, $oldval, $newval\t# (long) if $mem == $oldval then $mem <-- $newval"
9315 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
9316 %}
9317
9318 ins_encode(aarch64_enc_cmpxchg(mem, oldval, newval),
9319 aarch64_enc_cset_eq(res));
9320
9321 ins_pipe(pipe_slow);
9322 %}
9323
9324 instruct compareAndSwapP(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
9325
9326 match(Set res (CompareAndSwapP mem (Binary oldval newval)));
9327 predicate(n->as_LoadStore()->barrier_data() == 0);
9328 ins_cost(2 * VOLATILE_REF_COST);
9329
9330 effect(KILL cr);
9331
9332 format %{
9333 "cmpxchg $mem, $oldval, $newval\t# (ptr) if $mem == $oldval then $mem <-- $newval"
9334 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
9335 %}
9336
9337 ins_encode(aarch64_enc_cmpxchg(mem, oldval, newval),
9338 aarch64_enc_cset_eq(res));
9339
9340 ins_pipe(pipe_slow);
9341 %}
9342
9343 instruct compareAndSwapN(iRegINoSp res, indirect mem, iRegNNoSp oldval, iRegNNoSp newval, rFlagsReg cr) %{
9344
9345 match(Set res (CompareAndSwapN mem (Binary oldval newval)));
9346 ins_cost(2 * VOLATILE_REF_COST);
9347
9348 effect(KILL cr);
9349
9350 format %{
9351 "cmpxchgw $mem, $oldval, $newval\t# (narrow oop) if $mem == $oldval then $mem <-- $newval"
9352 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
9353 %}
9354
9355 ins_encode(aarch64_enc_cmpxchgw(mem, oldval, newval),
9356 aarch64_enc_cset_eq(res));
9357
9358 ins_pipe(pipe_slow);
9359 %}
9360
9361 // alternative CompareAndSwapX when we are eliding barriers
9362
9363 instruct compareAndSwapBAcq(iRegINoSp res, indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr) %{
9364
9365 predicate(needs_acquiring_load_exclusive(n));
9366 match(Set res (CompareAndSwapB mem (Binary oldval newval)));
9367 ins_cost(VOLATILE_REF_COST);
9368
9369 effect(KILL cr);
9370
9371 format %{
9372 "cmpxchgb_acq $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval"
9373 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
9374 %}
9375
9376 ins_encode(aarch64_enc_cmpxchgb_acq(mem, oldval, newval),
9377 aarch64_enc_cset_eq(res));
9378
9379 ins_pipe(pipe_slow);
9380 %}
9381
9382 instruct compareAndSwapSAcq(iRegINoSp res, indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr) %{
9383
9384 predicate(needs_acquiring_load_exclusive(n));
9385 match(Set res (CompareAndSwapS mem (Binary oldval newval)));
9386 ins_cost(VOLATILE_REF_COST);
9387
9388 effect(KILL cr);
9389
9390 format %{
9391 "cmpxchgs_acq $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval"
9392 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
9393 %}
9394
9395 ins_encode(aarch64_enc_cmpxchgs_acq(mem, oldval, newval),
9396 aarch64_enc_cset_eq(res));
9397
9398 ins_pipe(pipe_slow);
9399 %}
9400
9401 instruct compareAndSwapIAcq(iRegINoSp res, indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr) %{
9402
9403 predicate(needs_acquiring_load_exclusive(n));
9404 match(Set res (CompareAndSwapI mem (Binary oldval newval)));
9405 ins_cost(VOLATILE_REF_COST);
9406
9407 effect(KILL cr);
9408
9409 format %{
9410 "cmpxchgw_acq $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval"
9411 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
9412 %}
9413
9414 ins_encode(aarch64_enc_cmpxchgw_acq(mem, oldval, newval),
9415 aarch64_enc_cset_eq(res));
9416
9417 ins_pipe(pipe_slow);
9418 %}
9419
9420 instruct compareAndSwapLAcq(iRegINoSp res, indirect mem, iRegLNoSp oldval, iRegLNoSp newval, rFlagsReg cr) %{
9421
9422 predicate(needs_acquiring_load_exclusive(n));
9423 match(Set res (CompareAndSwapL mem (Binary oldval newval)));
9424 ins_cost(VOLATILE_REF_COST);
9425
9426 effect(KILL cr);
9427
9428 format %{
9429 "cmpxchg_acq $mem, $oldval, $newval\t# (long) if $mem == $oldval then $mem <-- $newval"
9430 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
9431 %}
9432
9433 ins_encode(aarch64_enc_cmpxchg_acq(mem, oldval, newval),
9434 aarch64_enc_cset_eq(res));
9435
9436 ins_pipe(pipe_slow);
9437 %}
9438
9439 instruct compareAndSwapPAcq(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
9440
9441 predicate(needs_acquiring_load_exclusive(n) && (n->as_LoadStore()->barrier_data() == 0));
9442 match(Set res (CompareAndSwapP mem (Binary oldval newval)));
9443 ins_cost(VOLATILE_REF_COST);
9444
9445 effect(KILL cr);
9446
9447 format %{
9448 "cmpxchg_acq $mem, $oldval, $newval\t# (ptr) if $mem == $oldval then $mem <-- $newval"
9449 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
9450 %}
9451
9452 ins_encode(aarch64_enc_cmpxchg_acq(mem, oldval, newval),
9453 aarch64_enc_cset_eq(res));
9454
9455 ins_pipe(pipe_slow);
9456 %}
9457
9458 instruct compareAndSwapNAcq(iRegINoSp res, indirect mem, iRegNNoSp oldval, iRegNNoSp newval, rFlagsReg cr) %{
9459
9460 predicate(needs_acquiring_load_exclusive(n));
9461 match(Set res (CompareAndSwapN mem (Binary oldval newval)));
9462 ins_cost(VOLATILE_REF_COST);
9463
9464 effect(KILL cr);
9465
9466 format %{
9467 "cmpxchgw_acq $mem, $oldval, $newval\t# (narrow oop) if $mem == $oldval then $mem <-- $newval"
9468 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
9469 %}
9470
9471 ins_encode(aarch64_enc_cmpxchgw_acq(mem, oldval, newval),
9472 aarch64_enc_cset_eq(res));
9473
9474 ins_pipe(pipe_slow);
9475 %}
9476
9477
9478 // ---------------------------------------------------------------------
9479
9480 // BEGIN This section of the file is automatically generated. Do not edit --------------
9481
9482 // Sundry CAS operations. Note that release is always true,
9483 // regardless of the memory ordering of the CAS. This is because we
9484 // need the volatile case to be sequentially consistent but there is
9485 // no trailing StoreLoad barrier emitted by C2. Unfortunately we
9486 // can't check the type of memory ordering here, so we always emit a
9487 // STLXR.
9488
9489 // This section is generated from aarch64_ad_cas.m4
9490
9491
9492
9493 // This pattern is generated automatically from cas.m4.
9494 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9495 instruct compareAndExchangeB(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
9496
9497 match(Set res (CompareAndExchangeB mem (Binary oldval newval)));
9498 ins_cost(2 * VOLATILE_REF_COST);
9499 effect(TEMP_DEF res, KILL cr);
9500 format %{
9501 "cmpxchgb $res = $mem, $oldval, $newval\t# (byte, weak) if $mem == $oldval then $mem <-- $newval"
9502 %}
9503 ins_encode %{
9504 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
9505 Assembler::byte, /*acquire*/ false, /*release*/ true,
9506 /*weak*/ false, $res$$Register);
9507 __ sxtbw($res$$Register, $res$$Register);
9508 %}
9509 ins_pipe(pipe_slow);
9510 %}
9511
9512 // This pattern is generated automatically from cas.m4.
9513 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9514 instruct compareAndExchangeS(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
9515
9516 match(Set res (CompareAndExchangeS mem (Binary oldval newval)));
9517 ins_cost(2 * VOLATILE_REF_COST);
9518 effect(TEMP_DEF res, KILL cr);
9519 format %{
9520 "cmpxchgs $res = $mem, $oldval, $newval\t# (short, weak) if $mem == $oldval then $mem <-- $newval"
9521 %}
9522 ins_encode %{
9523 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
9524 Assembler::halfword, /*acquire*/ false, /*release*/ true,
9525 /*weak*/ false, $res$$Register);
9526 __ sxthw($res$$Register, $res$$Register);
9527 %}
9528 ins_pipe(pipe_slow);
9529 %}
9530
9531 // This pattern is generated automatically from cas.m4.
9532 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9533 instruct compareAndExchangeI(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
9534
9535 match(Set res (CompareAndExchangeI mem (Binary oldval newval)));
9536 ins_cost(2 * VOLATILE_REF_COST);
9537 effect(TEMP_DEF res, KILL cr);
9538 format %{
9539 "cmpxchgw $res = $mem, $oldval, $newval\t# (int, weak) if $mem == $oldval then $mem <-- $newval"
9540 %}
9541 ins_encode %{
9542 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
9543 Assembler::word, /*acquire*/ false, /*release*/ true,
9544 /*weak*/ false, $res$$Register);
9545 %}
9546 ins_pipe(pipe_slow);
9547 %}
9548
9549 // This pattern is generated automatically from cas.m4.
9550 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9551 instruct compareAndExchangeL(iRegLNoSp res, indirect mem, iRegL oldval, iRegL newval, rFlagsReg cr) %{
9552
9553 match(Set res (CompareAndExchangeL mem (Binary oldval newval)));
9554 ins_cost(2 * VOLATILE_REF_COST);
9555 effect(TEMP_DEF res, KILL cr);
9556 format %{
9557 "cmpxchg $res = $mem, $oldval, $newval\t# (long, weak) if $mem == $oldval then $mem <-- $newval"
9558 %}
9559 ins_encode %{
9560 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
9561 Assembler::xword, /*acquire*/ false, /*release*/ true,
9562 /*weak*/ false, $res$$Register);
9563 %}
9564 ins_pipe(pipe_slow);
9565 %}
9566
9567 // This pattern is generated automatically from cas.m4.
9568 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9569 instruct compareAndExchangeN(iRegNNoSp res, indirect mem, iRegN oldval, iRegN newval, rFlagsReg cr) %{
9570
9571 match(Set res (CompareAndExchangeN mem (Binary oldval newval)));
9572 ins_cost(2 * VOLATILE_REF_COST);
9573 effect(TEMP_DEF res, KILL cr);
9574 format %{
9575 "cmpxchgw $res = $mem, $oldval, $newval\t# (narrow oop, weak) if $mem == $oldval then $mem <-- $newval"
9576 %}
9577 ins_encode %{
9578 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
9579 Assembler::word, /*acquire*/ false, /*release*/ true,
9580 /*weak*/ false, $res$$Register);
9581 %}
9582 ins_pipe(pipe_slow);
9583 %}
9584
9585 // This pattern is generated automatically from cas.m4.
9586 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9587 instruct compareAndExchangeP(iRegPNoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
9588 predicate(n->as_LoadStore()->barrier_data() == 0);
9589 match(Set res (CompareAndExchangeP mem (Binary oldval newval)));
9590 ins_cost(2 * VOLATILE_REF_COST);
9591 effect(TEMP_DEF res, KILL cr);
9592 format %{
9593 "cmpxchg $res = $mem, $oldval, $newval\t# (ptr, weak) if $mem == $oldval then $mem <-- $newval"
9594 %}
9595 ins_encode %{
9596 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
9597 Assembler::xword, /*acquire*/ false, /*release*/ true,
9598 /*weak*/ false, $res$$Register);
9599 %}
9600 ins_pipe(pipe_slow);
9601 %}
9602
9603 // This pattern is generated automatically from cas.m4.
9604 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9605 instruct compareAndExchangeBAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
9606 predicate(needs_acquiring_load_exclusive(n));
9607 match(Set res (CompareAndExchangeB mem (Binary oldval newval)));
9608 ins_cost(VOLATILE_REF_COST);
9609 effect(TEMP_DEF res, KILL cr);
9610 format %{
9611 "cmpxchgb_acq $res = $mem, $oldval, $newval\t# (byte, weak) if $mem == $oldval then $mem <-- $newval"
9612 %}
9613 ins_encode %{
9614 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
9615 Assembler::byte, /*acquire*/ true, /*release*/ true,
9616 /*weak*/ false, $res$$Register);
9617 __ sxtbw($res$$Register, $res$$Register);
9618 %}
9619 ins_pipe(pipe_slow);
9620 %}
9621
9622 // This pattern is generated automatically from cas.m4.
9623 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9624 instruct compareAndExchangeSAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
9625 predicate(needs_acquiring_load_exclusive(n));
9626 match(Set res (CompareAndExchangeS mem (Binary oldval newval)));
9627 ins_cost(VOLATILE_REF_COST);
9628 effect(TEMP_DEF res, KILL cr);
9629 format %{
9630 "cmpxchgs_acq $res = $mem, $oldval, $newval\t# (short, weak) if $mem == $oldval then $mem <-- $newval"
9631 %}
9632 ins_encode %{
9633 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
9634 Assembler::halfword, /*acquire*/ true, /*release*/ true,
9635 /*weak*/ false, $res$$Register);
9636 __ sxthw($res$$Register, $res$$Register);
9637 %}
9638 ins_pipe(pipe_slow);
9639 %}
9640
9641 // This pattern is generated automatically from cas.m4.
9642 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9643 instruct compareAndExchangeIAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
9644 predicate(needs_acquiring_load_exclusive(n));
9645 match(Set res (CompareAndExchangeI mem (Binary oldval newval)));
9646 ins_cost(VOLATILE_REF_COST);
9647 effect(TEMP_DEF res, KILL cr);
9648 format %{
9649 "cmpxchgw_acq $res = $mem, $oldval, $newval\t# (int, weak) if $mem == $oldval then $mem <-- $newval"
9650 %}
9651 ins_encode %{
9652 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
9653 Assembler::word, /*acquire*/ true, /*release*/ true,
9654 /*weak*/ false, $res$$Register);
9655 %}
9656 ins_pipe(pipe_slow);
9657 %}
9658
9659 // This pattern is generated automatically from cas.m4.
9660 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9661 instruct compareAndExchangeLAcq(iRegLNoSp res, indirect mem, iRegL oldval, iRegL newval, rFlagsReg cr) %{
9662 predicate(needs_acquiring_load_exclusive(n));
9663 match(Set res (CompareAndExchangeL mem (Binary oldval newval)));
9664 ins_cost(VOLATILE_REF_COST);
9665 effect(TEMP_DEF res, KILL cr);
9666 format %{
9667 "cmpxchg_acq $res = $mem, $oldval, $newval\t# (long, weak) if $mem == $oldval then $mem <-- $newval"
9668 %}
9669 ins_encode %{
9670 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
9671 Assembler::xword, /*acquire*/ true, /*release*/ true,
9672 /*weak*/ false, $res$$Register);
9673 %}
9674 ins_pipe(pipe_slow);
9675 %}
9676
9677 // This pattern is generated automatically from cas.m4.
9678 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9679 instruct compareAndExchangeNAcq(iRegNNoSp res, indirect mem, iRegN oldval, iRegN newval, rFlagsReg cr) %{
9680 predicate(needs_acquiring_load_exclusive(n));
9681 match(Set res (CompareAndExchangeN mem (Binary oldval newval)));
9682 ins_cost(VOLATILE_REF_COST);
9683 effect(TEMP_DEF res, KILL cr);
9684 format %{
9685 "cmpxchgw_acq $res = $mem, $oldval, $newval\t# (narrow oop, weak) if $mem == $oldval then $mem <-- $newval"
9686 %}
9687 ins_encode %{
9688 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
9689 Assembler::word, /*acquire*/ true, /*release*/ true,
9690 /*weak*/ false, $res$$Register);
9691 %}
9692 ins_pipe(pipe_slow);
9693 %}
9694
9695 // This pattern is generated automatically from cas.m4.
9696 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9697 instruct compareAndExchangePAcq(iRegPNoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
9698 predicate(needs_acquiring_load_exclusive(n) && (n->as_LoadStore()->barrier_data() == 0));
9699 match(Set res (CompareAndExchangeP mem (Binary oldval newval)));
9700 ins_cost(VOLATILE_REF_COST);
9701 effect(TEMP_DEF res, KILL cr);
9702 format %{
9703 "cmpxchg_acq $res = $mem, $oldval, $newval\t# (ptr, weak) if $mem == $oldval then $mem <-- $newval"
9704 %}
9705 ins_encode %{
9706 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
9707 Assembler::xword, /*acquire*/ true, /*release*/ true,
9708 /*weak*/ false, $res$$Register);
9709 %}
9710 ins_pipe(pipe_slow);
9711 %}
9712
9713 // This pattern is generated automatically from cas.m4.
9714 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9715 instruct weakCompareAndSwapB(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
9716
9717 match(Set res (WeakCompareAndSwapB mem (Binary oldval newval)));
9718 ins_cost(2 * VOLATILE_REF_COST);
9719 effect(KILL cr);
9720 format %{
9721 "cmpxchgb $res = $mem, $oldval, $newval\t# (byte, weak) if $mem == $oldval then $mem <-- $newval"
9722 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
9723 %}
9724 ins_encode %{
9725 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
9726 Assembler::byte, /*acquire*/ false, /*release*/ true,
9727 /*weak*/ true, noreg);
9728 __ csetw($res$$Register, Assembler::EQ);
9729 %}
9730 ins_pipe(pipe_slow);
9731 %}
9732
9733 // This pattern is generated automatically from cas.m4.
9734 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9735 instruct weakCompareAndSwapS(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
9736
9737 match(Set res (WeakCompareAndSwapS mem (Binary oldval newval)));
9738 ins_cost(2 * VOLATILE_REF_COST);
9739 effect(KILL cr);
9740 format %{
9741 "cmpxchgs $res = $mem, $oldval, $newval\t# (short, weak) if $mem == $oldval then $mem <-- $newval"
9742 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
9743 %}
9744 ins_encode %{
9745 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
9746 Assembler::halfword, /*acquire*/ false, /*release*/ true,
9747 /*weak*/ true, noreg);
9748 __ csetw($res$$Register, Assembler::EQ);
9749 %}
9750 ins_pipe(pipe_slow);
9751 %}
9752
9753 // This pattern is generated automatically from cas.m4.
9754 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9755 instruct weakCompareAndSwapI(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
9756
9757 match(Set res (WeakCompareAndSwapI mem (Binary oldval newval)));
9758 ins_cost(2 * VOLATILE_REF_COST);
9759 effect(KILL cr);
9760 format %{
9761 "cmpxchgw $res = $mem, $oldval, $newval\t# (int, weak) if $mem == $oldval then $mem <-- $newval"
9762 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
9763 %}
9764 ins_encode %{
9765 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
9766 Assembler::word, /*acquire*/ false, /*release*/ true,
9767 /*weak*/ true, noreg);
9768 __ csetw($res$$Register, Assembler::EQ);
9769 %}
9770 ins_pipe(pipe_slow);
9771 %}
9772
9773 // This pattern is generated automatically from cas.m4.
9774 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9775 instruct weakCompareAndSwapL(iRegINoSp res, indirect mem, iRegL oldval, iRegL newval, rFlagsReg cr) %{
9776
9777 match(Set res (WeakCompareAndSwapL mem (Binary oldval newval)));
9778 ins_cost(2 * VOLATILE_REF_COST);
9779 effect(KILL cr);
9780 format %{
9781 "cmpxchg $res = $mem, $oldval, $newval\t# (long, weak) if $mem == $oldval then $mem <-- $newval"
9782 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
9783 %}
9784 ins_encode %{
9785 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
9786 Assembler::xword, /*acquire*/ false, /*release*/ true,
9787 /*weak*/ true, noreg);
9788 __ csetw($res$$Register, Assembler::EQ);
9789 %}
9790 ins_pipe(pipe_slow);
9791 %}
9792
9793 // This pattern is generated automatically from cas.m4.
9794 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9795 instruct weakCompareAndSwapN(iRegINoSp res, indirect mem, iRegN oldval, iRegN newval, rFlagsReg cr) %{
9796
9797 match(Set res (WeakCompareAndSwapN mem (Binary oldval newval)));
9798 ins_cost(2 * VOLATILE_REF_COST);
9799 effect(KILL cr);
9800 format %{
9801 "cmpxchgw $res = $mem, $oldval, $newval\t# (narrow oop, weak) if $mem == $oldval then $mem <-- $newval"
9802 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
9803 %}
9804 ins_encode %{
9805 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
9806 Assembler::word, /*acquire*/ false, /*release*/ true,
9807 /*weak*/ true, noreg);
9808 __ csetw($res$$Register, Assembler::EQ);
9809 %}
9810 ins_pipe(pipe_slow);
9811 %}
9812
9813 // This pattern is generated automatically from cas.m4.
9814 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9815 instruct weakCompareAndSwapP(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
9816 predicate(n->as_LoadStore()->barrier_data() == 0);
9817 match(Set res (WeakCompareAndSwapP mem (Binary oldval newval)));
9818 ins_cost(2 * VOLATILE_REF_COST);
9819 effect(KILL cr);
9820 format %{
9821 "cmpxchg $res = $mem, $oldval, $newval\t# (ptr, weak) if $mem == $oldval then $mem <-- $newval"
9822 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
9823 %}
9824 ins_encode %{
9825 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
9826 Assembler::xword, /*acquire*/ false, /*release*/ true,
9827 /*weak*/ true, noreg);
9828 __ csetw($res$$Register, Assembler::EQ);
9829 %}
9830 ins_pipe(pipe_slow);
9831 %}
9832
9833 // This pattern is generated automatically from cas.m4.
9834 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9835 instruct weakCompareAndSwapBAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
9836 predicate(needs_acquiring_load_exclusive(n));
9837 match(Set res (WeakCompareAndSwapB mem (Binary oldval newval)));
9838 ins_cost(VOLATILE_REF_COST);
9839 effect(KILL cr);
9840 format %{
9841 "cmpxchgb_acq $res = $mem, $oldval, $newval\t# (byte, weak) if $mem == $oldval then $mem <-- $newval"
9842 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
9843 %}
9844 ins_encode %{
9845 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
9846 Assembler::byte, /*acquire*/ true, /*release*/ true,
9847 /*weak*/ true, noreg);
9848 __ csetw($res$$Register, Assembler::EQ);
9849 %}
9850 ins_pipe(pipe_slow);
9851 %}
9852
9853 // This pattern is generated automatically from cas.m4.
9854 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9855 instruct weakCompareAndSwapSAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
9856 predicate(needs_acquiring_load_exclusive(n));
9857 match(Set res (WeakCompareAndSwapS mem (Binary oldval newval)));
9858 ins_cost(VOLATILE_REF_COST);
9859 effect(KILL cr);
9860 format %{
9861 "cmpxchgs_acq $res = $mem, $oldval, $newval\t# (short, weak) if $mem == $oldval then $mem <-- $newval"
9862 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
9863 %}
9864 ins_encode %{
9865 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
9866 Assembler::halfword, /*acquire*/ true, /*release*/ true,
9867 /*weak*/ true, noreg);
9868 __ csetw($res$$Register, Assembler::EQ);
9869 %}
9870 ins_pipe(pipe_slow);
9871 %}
9872
9873 // This pattern is generated automatically from cas.m4.
9874 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9875 instruct weakCompareAndSwapIAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
9876 predicate(needs_acquiring_load_exclusive(n));
9877 match(Set res (WeakCompareAndSwapI mem (Binary oldval newval)));
9878 ins_cost(VOLATILE_REF_COST);
9879 effect(KILL cr);
9880 format %{
9881 "cmpxchgw_acq $res = $mem, $oldval, $newval\t# (int, weak) if $mem == $oldval then $mem <-- $newval"
9882 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
9883 %}
9884 ins_encode %{
9885 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
9886 Assembler::word, /*acquire*/ true, /*release*/ true,
9887 /*weak*/ true, noreg);
9888 __ csetw($res$$Register, Assembler::EQ);
9889 %}
9890 ins_pipe(pipe_slow);
9891 %}
9892
9893 // This pattern is generated automatically from cas.m4.
9894 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9895 instruct weakCompareAndSwapLAcq(iRegINoSp res, indirect mem, iRegL oldval, iRegL newval, rFlagsReg cr) %{
9896 predicate(needs_acquiring_load_exclusive(n));
9897 match(Set res (WeakCompareAndSwapL mem (Binary oldval newval)));
9898 ins_cost(VOLATILE_REF_COST);
9899 effect(KILL cr);
9900 format %{
9901 "cmpxchg_acq $res = $mem, $oldval, $newval\t# (long, weak) if $mem == $oldval then $mem <-- $newval"
9902 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
9903 %}
9904 ins_encode %{
9905 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
9906 Assembler::xword, /*acquire*/ true, /*release*/ true,
9907 /*weak*/ true, noreg);
9908 __ csetw($res$$Register, Assembler::EQ);
9909 %}
9910 ins_pipe(pipe_slow);
9911 %}
9912
9913 // This pattern is generated automatically from cas.m4.
9914 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9915 instruct weakCompareAndSwapNAcq(iRegINoSp res, indirect mem, iRegN oldval, iRegN newval, rFlagsReg cr) %{
9916 predicate(needs_acquiring_load_exclusive(n));
9917 match(Set res (WeakCompareAndSwapN mem (Binary oldval newval)));
9918 ins_cost(VOLATILE_REF_COST);
9919 effect(KILL cr);
9920 format %{
9921 "cmpxchgw_acq $res = $mem, $oldval, $newval\t# (narrow oop, weak) if $mem == $oldval then $mem <-- $newval"
9922 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
9923 %}
9924 ins_encode %{
9925 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
9926 Assembler::word, /*acquire*/ true, /*release*/ true,
9927 /*weak*/ true, noreg);
9928 __ csetw($res$$Register, Assembler::EQ);
9929 %}
9930 ins_pipe(pipe_slow);
9931 %}
9932
9933 // This pattern is generated automatically from cas.m4.
9934 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9935 instruct weakCompareAndSwapPAcq(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
9936 predicate(needs_acquiring_load_exclusive(n) && (n->as_LoadStore()->barrier_data() == 0));
9937 match(Set res (WeakCompareAndSwapP mem (Binary oldval newval)));
9938 ins_cost(VOLATILE_REF_COST);
9939 effect(KILL cr);
9940 format %{
9941 "cmpxchg_acq $res = $mem, $oldval, $newval\t# (ptr, weak) if $mem == $oldval then $mem <-- $newval"
9942 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
9943 %}
9944 ins_encode %{
9945 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
9946 Assembler::xword, /*acquire*/ true, /*release*/ true,
9947 /*weak*/ true, noreg);
9948 __ csetw($res$$Register, Assembler::EQ);
9949 %}
9950 ins_pipe(pipe_slow);
9951 %}
9952
9953 // END This section of the file is automatically generated. Do not edit --------------
9954 // ---------------------------------------------------------------------
9955
9956 instruct get_and_setI(indirect mem, iRegI newv, iRegINoSp prev) %{
9957 match(Set prev (GetAndSetI mem newv));
9958 ins_cost(2 * VOLATILE_REF_COST);
9959 format %{ "atomic_xchgw $prev, $newv, [$mem]" %}
9960 ins_encode %{
9961 __ atomic_xchgw($prev$$Register, $newv$$Register, as_Register($mem$$base));
9962 %}
9963 ins_pipe(pipe_serial);
9964 %}
9965
9966 instruct get_and_setL(indirect mem, iRegL newv, iRegLNoSp prev) %{
9967 match(Set prev (GetAndSetL mem newv));
9968 ins_cost(2 * VOLATILE_REF_COST);
9969 format %{ "atomic_xchg $prev, $newv, [$mem]" %}
9970 ins_encode %{
9971 __ atomic_xchg($prev$$Register, $newv$$Register, as_Register($mem$$base));
9972 %}
9973 ins_pipe(pipe_serial);
9974 %}
9975
9976 instruct get_and_setN(indirect mem, iRegN newv, iRegINoSp prev) %{
9977 match(Set prev (GetAndSetN mem newv));
9978 ins_cost(2 * VOLATILE_REF_COST);
9979 format %{ "atomic_xchgw $prev, $newv, [$mem]" %}
9980 ins_encode %{
9981 __ atomic_xchgw($prev$$Register, $newv$$Register, as_Register($mem$$base));
9982 %}
9983 ins_pipe(pipe_serial);
9984 %}
9985
9986 instruct get_and_setP(indirect mem, iRegP newv, iRegPNoSp prev) %{
9987 predicate(n->as_LoadStore()->barrier_data() == 0);
9988 match(Set prev (GetAndSetP mem newv));
9989 ins_cost(2 * VOLATILE_REF_COST);
9990 format %{ "atomic_xchg $prev, $newv, [$mem]" %}
9991 ins_encode %{
9992 __ atomic_xchg($prev$$Register, $newv$$Register, as_Register($mem$$base));
9993 %}
9994 ins_pipe(pipe_serial);
9995 %}
9996
9997 instruct get_and_setIAcq(indirect mem, iRegI newv, iRegINoSp prev) %{
9998 predicate(needs_acquiring_load_exclusive(n));
9999 match(Set prev (GetAndSetI mem newv));
10000 ins_cost(VOLATILE_REF_COST);
10001 format %{ "atomic_xchgw_acq $prev, $newv, [$mem]" %}
10002 ins_encode %{
10003 __ atomic_xchgalw($prev$$Register, $newv$$Register, as_Register($mem$$base));
10004 %}
10005 ins_pipe(pipe_serial);
10006 %}
10007
10008 instruct get_and_setLAcq(indirect mem, iRegL newv, iRegLNoSp prev) %{
10009 predicate(needs_acquiring_load_exclusive(n));
10010 match(Set prev (GetAndSetL mem newv));
10011 ins_cost(VOLATILE_REF_COST);
10012 format %{ "atomic_xchg_acq $prev, $newv, [$mem]" %}
10013 ins_encode %{
10014 __ atomic_xchgal($prev$$Register, $newv$$Register, as_Register($mem$$base));
10015 %}
10016 ins_pipe(pipe_serial);
10017 %}
10018
10019 instruct get_and_setNAcq(indirect mem, iRegN newv, iRegINoSp prev) %{
10020 predicate(needs_acquiring_load_exclusive(n));
10021 match(Set prev (GetAndSetN mem newv));
10022 ins_cost(VOLATILE_REF_COST);
10023 format %{ "atomic_xchgw_acq $prev, $newv, [$mem]" %}
10024 ins_encode %{
10025 __ atomic_xchgalw($prev$$Register, $newv$$Register, as_Register($mem$$base));
10026 %}
10027 ins_pipe(pipe_serial);
10028 %}
10029
10030 instruct get_and_setPAcq(indirect mem, iRegP newv, iRegPNoSp prev) %{
10031 predicate(needs_acquiring_load_exclusive(n) && (n->as_LoadStore()->barrier_data() == 0));
10032 match(Set prev (GetAndSetP mem newv));
10033 ins_cost(VOLATILE_REF_COST);
10034 format %{ "atomic_xchg_acq $prev, $newv, [$mem]" %}
10035 ins_encode %{
10036 __ atomic_xchgal($prev$$Register, $newv$$Register, as_Register($mem$$base));
10037 %}
10038 ins_pipe(pipe_serial);
10039 %}
10040
10041
10042 instruct get_and_addL(indirect mem, iRegLNoSp newval, iRegL incr) %{
10043 match(Set newval (GetAndAddL mem incr));
10044 ins_cost(2 * VOLATILE_REF_COST + 1);
10045 format %{ "get_and_addL $newval, [$mem], $incr" %}
10046 ins_encode %{
10047 __ atomic_add($newval$$Register, $incr$$Register, as_Register($mem$$base));
10048 %}
10049 ins_pipe(pipe_serial);
10050 %}
10051
10052 instruct get_and_addL_no_res(indirect mem, Universe dummy, iRegL incr) %{
10053 predicate(n->as_LoadStore()->result_not_used());
10054 match(Set dummy (GetAndAddL mem incr));
10055 ins_cost(2 * VOLATILE_REF_COST);
10056 format %{ "get_and_addL [$mem], $incr" %}
10057 ins_encode %{
10058 __ atomic_add(noreg, $incr$$Register, as_Register($mem$$base));
10059 %}
10060 ins_pipe(pipe_serial);
10061 %}
10062
10063 instruct get_and_addLi(indirect mem, iRegLNoSp newval, immLAddSub incr) %{
10064 match(Set newval (GetAndAddL mem incr));
10065 ins_cost(2 * VOLATILE_REF_COST + 1);
10066 format %{ "get_and_addL $newval, [$mem], $incr" %}
10067 ins_encode %{
10068 __ atomic_add($newval$$Register, $incr$$constant, as_Register($mem$$base));
10069 %}
10070 ins_pipe(pipe_serial);
10071 %}
10072
10073 instruct get_and_addLi_no_res(indirect mem, Universe dummy, immLAddSub incr) %{
10074 predicate(n->as_LoadStore()->result_not_used());
10075 match(Set dummy (GetAndAddL mem incr));
10076 ins_cost(2 * VOLATILE_REF_COST);
10077 format %{ "get_and_addL [$mem], $incr" %}
10078 ins_encode %{
10079 __ atomic_add(noreg, $incr$$constant, as_Register($mem$$base));
10080 %}
10081 ins_pipe(pipe_serial);
10082 %}
10083
10084 instruct get_and_addI(indirect mem, iRegINoSp newval, iRegIorL2I incr) %{
10085 match(Set newval (GetAndAddI mem incr));
10086 ins_cost(2 * VOLATILE_REF_COST + 1);
10087 format %{ "get_and_addI $newval, [$mem], $incr" %}
10088 ins_encode %{
10089 __ atomic_addw($newval$$Register, $incr$$Register, as_Register($mem$$base));
10090 %}
10091 ins_pipe(pipe_serial);
10092 %}
10093
10094 instruct get_and_addI_no_res(indirect mem, Universe dummy, iRegIorL2I incr) %{
10095 predicate(n->as_LoadStore()->result_not_used());
10096 match(Set dummy (GetAndAddI mem incr));
10097 ins_cost(2 * VOLATILE_REF_COST);
10098 format %{ "get_and_addI [$mem], $incr" %}
10099 ins_encode %{
10100 __ atomic_addw(noreg, $incr$$Register, as_Register($mem$$base));
10101 %}
10102 ins_pipe(pipe_serial);
10103 %}
10104
10105 instruct get_and_addIi(indirect mem, iRegINoSp newval, immIAddSub incr) %{
10106 match(Set newval (GetAndAddI mem incr));
10107 ins_cost(2 * VOLATILE_REF_COST + 1);
10108 format %{ "get_and_addI $newval, [$mem], $incr" %}
10109 ins_encode %{
10110 __ atomic_addw($newval$$Register, $incr$$constant, as_Register($mem$$base));
10111 %}
10112 ins_pipe(pipe_serial);
10113 %}
10114
10115 instruct get_and_addIi_no_res(indirect mem, Universe dummy, immIAddSub incr) %{
10116 predicate(n->as_LoadStore()->result_not_used());
10117 match(Set dummy (GetAndAddI mem incr));
10118 ins_cost(2 * VOLATILE_REF_COST);
10119 format %{ "get_and_addI [$mem], $incr" %}
10120 ins_encode %{
10121 __ atomic_addw(noreg, $incr$$constant, as_Register($mem$$base));
10122 %}
10123 ins_pipe(pipe_serial);
10124 %}
10125
10126 instruct get_and_addLAcq(indirect mem, iRegLNoSp newval, iRegL incr) %{
10127 predicate(needs_acquiring_load_exclusive(n));
10128 match(Set newval (GetAndAddL mem incr));
10129 ins_cost(VOLATILE_REF_COST + 1);
10130 format %{ "get_and_addL_acq $newval, [$mem], $incr" %}
10131 ins_encode %{
10132 __ atomic_addal($newval$$Register, $incr$$Register, as_Register($mem$$base));
10133 %}
10134 ins_pipe(pipe_serial);
10135 %}
10136
10137 instruct get_and_addL_no_resAcq(indirect mem, Universe dummy, iRegL incr) %{
10138 predicate(n->as_LoadStore()->result_not_used() && needs_acquiring_load_exclusive(n));
10139 match(Set dummy (GetAndAddL mem incr));
10140 ins_cost(VOLATILE_REF_COST);
10141 format %{ "get_and_addL_acq [$mem], $incr" %}
10142 ins_encode %{
10143 __ atomic_addal(noreg, $incr$$Register, as_Register($mem$$base));
10144 %}
10145 ins_pipe(pipe_serial);
10146 %}
10147
10148 instruct get_and_addLiAcq(indirect mem, iRegLNoSp newval, immLAddSub incr) %{
10149 predicate(needs_acquiring_load_exclusive(n));
10150 match(Set newval (GetAndAddL mem incr));
10151 ins_cost(VOLATILE_REF_COST + 1);
10152 format %{ "get_and_addL_acq $newval, [$mem], $incr" %}
10153 ins_encode %{
10154 __ atomic_addal($newval$$Register, $incr$$constant, as_Register($mem$$base));
10155 %}
10156 ins_pipe(pipe_serial);
10157 %}
10158
10159 instruct get_and_addLi_no_resAcq(indirect mem, Universe dummy, immLAddSub incr) %{
10160 predicate(n->as_LoadStore()->result_not_used() && needs_acquiring_load_exclusive(n));
10161 match(Set dummy (GetAndAddL mem incr));
10162 ins_cost(VOLATILE_REF_COST);
10163 format %{ "get_and_addL_acq [$mem], $incr" %}
10164 ins_encode %{
10165 __ atomic_addal(noreg, $incr$$constant, as_Register($mem$$base));
10166 %}
10167 ins_pipe(pipe_serial);
10168 %}
10169
10170 instruct get_and_addIAcq(indirect mem, iRegINoSp newval, iRegIorL2I incr) %{
10171 predicate(needs_acquiring_load_exclusive(n));
10172 match(Set newval (GetAndAddI mem incr));
10173 ins_cost(VOLATILE_REF_COST + 1);
10174 format %{ "get_and_addI_acq $newval, [$mem], $incr" %}
10175 ins_encode %{
10176 __ atomic_addalw($newval$$Register, $incr$$Register, as_Register($mem$$base));
10177 %}
10178 ins_pipe(pipe_serial);
10179 %}
10180
10181 instruct get_and_addI_no_resAcq(indirect mem, Universe dummy, iRegIorL2I incr) %{
10182 predicate(n->as_LoadStore()->result_not_used() && needs_acquiring_load_exclusive(n));
10183 match(Set dummy (GetAndAddI mem incr));
10184 ins_cost(VOLATILE_REF_COST);
10185 format %{ "get_and_addI_acq [$mem], $incr" %}
10186 ins_encode %{
10187 __ atomic_addalw(noreg, $incr$$Register, as_Register($mem$$base));
10188 %}
10189 ins_pipe(pipe_serial);
10190 %}
10191
10192 instruct get_and_addIiAcq(indirect mem, iRegINoSp newval, immIAddSub incr) %{
10193 predicate(needs_acquiring_load_exclusive(n));
10194 match(Set newval (GetAndAddI mem incr));
10195 ins_cost(VOLATILE_REF_COST + 1);
10196 format %{ "get_and_addI_acq $newval, [$mem], $incr" %}
10197 ins_encode %{
10198 __ atomic_addalw($newval$$Register, $incr$$constant, as_Register($mem$$base));
10199 %}
10200 ins_pipe(pipe_serial);
10201 %}
10202
10203 instruct get_and_addIi_no_resAcq(indirect mem, Universe dummy, immIAddSub incr) %{
10204 predicate(n->as_LoadStore()->result_not_used() && needs_acquiring_load_exclusive(n));
10205 match(Set dummy (GetAndAddI mem incr));
10206 ins_cost(VOLATILE_REF_COST);
10207 format %{ "get_and_addI_acq [$mem], $incr" %}
10208 ins_encode %{
10209 __ atomic_addalw(noreg, $incr$$constant, as_Register($mem$$base));
10210 %}
10211 ins_pipe(pipe_serial);
10212 %}
10213
10214 // Manifest a CmpL result in an integer register.
10215 // (src1 < src2) ? -1 : ((src1 > src2) ? 1 : 0)
10216 instruct cmpL3_reg_reg(iRegINoSp dst, iRegL src1, iRegL src2, rFlagsReg flags)
10217 %{
10218 match(Set dst (CmpL3 src1 src2));
10219 effect(KILL flags);
10220
10221 ins_cost(INSN_COST * 6);
10222 format %{
10223 "cmp $src1, $src2"
10224 "csetw $dst, ne"
10225 "cnegw $dst, lt"
10226 %}
10227 // format %{ "CmpL3 $dst, $src1, $src2" %}
10228 ins_encode %{
10229 __ cmp($src1$$Register, $src2$$Register);
10230 __ csetw($dst$$Register, Assembler::NE);
10231 __ cnegw($dst$$Register, $dst$$Register, Assembler::LT);
10232 %}
10233
10234 ins_pipe(pipe_class_default);
10235 %}
10236
10237 instruct cmpL3_reg_imm(iRegINoSp dst, iRegL src1, immLAddSub src2, rFlagsReg flags)
10238 %{
10239 match(Set dst (CmpL3 src1 src2));
10240 effect(KILL flags);
10241
10242 ins_cost(INSN_COST * 6);
10243 format %{
10244 "cmp $src1, $src2"
10245 "csetw $dst, ne"
10246 "cnegw $dst, lt"
10247 %}
10248 ins_encode %{
10249 int32_t con = (int32_t)$src2$$constant;
10250 if (con < 0) {
10251 __ adds(zr, $src1$$Register, -con);
10252 } else {
10253 __ subs(zr, $src1$$Register, con);
10254 }
10255 __ csetw($dst$$Register, Assembler::NE);
10256 __ cnegw($dst$$Register, $dst$$Register, Assembler::LT);
10257 %}
10258
10259 ins_pipe(pipe_class_default);
10260 %}
10261
10262 // ============================================================================
10263 // Conditional Move Instructions
10264
10265 // n.b. we have identical rules for both a signed compare op (cmpOp)
10266 // and an unsigned compare op (cmpOpU). it would be nice if we could
10267 // define an op class which merged both inputs and use it to type the
10268 // argument to a single rule. unfortunatelyt his fails because the
10269 // opclass does not live up to the COND_INTER interface of its
10270 // component operands. When the generic code tries to negate the
10271 // operand it ends up running the generci Machoper::negate method
10272 // which throws a ShouldNotHappen. So, we have to provide two flavours
10273 // of each rule, one for a cmpOp and a second for a cmpOpU (sigh).
10274
10275 instruct cmovI_reg_reg(cmpOp cmp, rFlagsReg cr, iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10276 match(Set dst (CMoveI (Binary cmp cr) (Binary src1 src2)));
10277
10278 ins_cost(INSN_COST * 2);
10279 format %{ "cselw $dst, $src2, $src1 $cmp\t# signed, int" %}
10280
10281 ins_encode %{
10282 __ cselw(as_Register($dst$$reg),
10283 as_Register($src2$$reg),
10284 as_Register($src1$$reg),
10285 (Assembler::Condition)$cmp$$cmpcode);
10286 %}
10287
10288 ins_pipe(icond_reg_reg);
10289 %}
10290
10291 instruct cmovUI_reg_reg(cmpOpU cmp, rFlagsRegU cr, iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10292 match(Set dst (CMoveI (Binary cmp cr) (Binary src1 src2)));
10293
10294 ins_cost(INSN_COST * 2);
10295 format %{ "cselw $dst, $src2, $src1 $cmp\t# unsigned, int" %}
10296
10297 ins_encode %{
10298 __ cselw(as_Register($dst$$reg),
10299 as_Register($src2$$reg),
10300 as_Register($src1$$reg),
10301 (Assembler::Condition)$cmp$$cmpcode);
10302 %}
10303
10304 ins_pipe(icond_reg_reg);
10305 %}
10306
10307 // special cases where one arg is zero
10308
10309 // n.b. this is selected in preference to the rule above because it
10310 // avoids loading constant 0 into a source register
10311
10312 // TODO
10313 // we ought only to be able to cull one of these variants as the ideal
10314 // transforms ought always to order the zero consistently (to left/right?)
10315
10316 instruct cmovI_zero_reg(cmpOp cmp, rFlagsReg cr, iRegINoSp dst, immI0 zero, iRegIorL2I src) %{
10317 match(Set dst (CMoveI (Binary cmp cr) (Binary zero src)));
10318
10319 ins_cost(INSN_COST * 2);
10320 format %{ "cselw $dst, $src, zr $cmp\t# signed, int" %}
10321
10322 ins_encode %{
10323 __ cselw(as_Register($dst$$reg),
10324 as_Register($src$$reg),
10325 zr,
10326 (Assembler::Condition)$cmp$$cmpcode);
10327 %}
10328
10329 ins_pipe(icond_reg);
10330 %}
10331
10332 instruct cmovUI_zero_reg(cmpOpU cmp, rFlagsRegU cr, iRegINoSp dst, immI0 zero, iRegIorL2I src) %{
10333 match(Set dst (CMoveI (Binary cmp cr) (Binary zero src)));
10334
10335 ins_cost(INSN_COST * 2);
10336 format %{ "cselw $dst, $src, zr $cmp\t# unsigned, int" %}
10337
10338 ins_encode %{
10339 __ cselw(as_Register($dst$$reg),
10340 as_Register($src$$reg),
10341 zr,
10342 (Assembler::Condition)$cmp$$cmpcode);
10343 %}
10344
10345 ins_pipe(icond_reg);
10346 %}
10347
10348 instruct cmovI_reg_zero(cmpOp cmp, rFlagsReg cr, iRegINoSp dst, iRegIorL2I src, immI0 zero) %{
10349 match(Set dst (CMoveI (Binary cmp cr) (Binary src zero)));
10350
10351 ins_cost(INSN_COST * 2);
10352 format %{ "cselw $dst, zr, $src $cmp\t# signed, int" %}
10353
10354 ins_encode %{
10355 __ cselw(as_Register($dst$$reg),
10356 zr,
10357 as_Register($src$$reg),
10358 (Assembler::Condition)$cmp$$cmpcode);
10359 %}
10360
10361 ins_pipe(icond_reg);
10362 %}
10363
10364 instruct cmovUI_reg_zero(cmpOpU cmp, rFlagsRegU cr, iRegINoSp dst, iRegIorL2I src, immI0 zero) %{
10365 match(Set dst (CMoveI (Binary cmp cr) (Binary src zero)));
10366
10367 ins_cost(INSN_COST * 2);
10368 format %{ "cselw $dst, zr, $src $cmp\t# unsigned, int" %}
10369
10370 ins_encode %{
10371 __ cselw(as_Register($dst$$reg),
10372 zr,
10373 as_Register($src$$reg),
10374 (Assembler::Condition)$cmp$$cmpcode);
10375 %}
10376
10377 ins_pipe(icond_reg);
10378 %}
10379
10380 // special case for creating a boolean 0 or 1
10381
10382 // n.b. this is selected in preference to the rule above because it
10383 // avoids loading constants 0 and 1 into a source register
10384
10385 instruct cmovI_reg_zero_one(cmpOp cmp, rFlagsReg cr, iRegINoSp dst, immI0 zero, immI_1 one) %{
10386 match(Set dst (CMoveI (Binary cmp cr) (Binary one zero)));
10387
10388 ins_cost(INSN_COST * 2);
10389 format %{ "csincw $dst, zr, zr $cmp\t# signed, int" %}
10390
10391 ins_encode %{
10392 // equivalently
10393 // cset(as_Register($dst$$reg),
10394 // negate_condition((Assembler::Condition)$cmp$$cmpcode));
10395 __ csincw(as_Register($dst$$reg),
10396 zr,
10397 zr,
10398 (Assembler::Condition)$cmp$$cmpcode);
10399 %}
10400
10401 ins_pipe(icond_none);
10402 %}
10403
10404 instruct cmovUI_reg_zero_one(cmpOpU cmp, rFlagsRegU cr, iRegINoSp dst, immI0 zero, immI_1 one) %{
10405 match(Set dst (CMoveI (Binary cmp cr) (Binary one zero)));
10406
10407 ins_cost(INSN_COST * 2);
10408 format %{ "csincw $dst, zr, zr $cmp\t# unsigned, int" %}
10409
10410 ins_encode %{
10411 // equivalently
10412 // cset(as_Register($dst$$reg),
10413 // negate_condition((Assembler::Condition)$cmp$$cmpcode));
10414 __ csincw(as_Register($dst$$reg),
10415 zr,
10416 zr,
10417 (Assembler::Condition)$cmp$$cmpcode);
10418 %}
10419
10420 ins_pipe(icond_none);
10421 %}
10422
10423 instruct cmovL_reg_reg(cmpOp cmp, rFlagsReg cr, iRegLNoSp dst, iRegL src1, iRegL src2) %{
10424 match(Set dst (CMoveL (Binary cmp cr) (Binary src1 src2)));
10425
10426 ins_cost(INSN_COST * 2);
10427 format %{ "csel $dst, $src2, $src1 $cmp\t# signed, long" %}
10428
10429 ins_encode %{
10430 __ csel(as_Register($dst$$reg),
10431 as_Register($src2$$reg),
10432 as_Register($src1$$reg),
10433 (Assembler::Condition)$cmp$$cmpcode);
10434 %}
10435
10436 ins_pipe(icond_reg_reg);
10437 %}
10438
10439 instruct cmovUL_reg_reg(cmpOpU cmp, rFlagsRegU cr, iRegLNoSp dst, iRegL src1, iRegL src2) %{
10440 match(Set dst (CMoveL (Binary cmp cr) (Binary src1 src2)));
10441
10442 ins_cost(INSN_COST * 2);
10443 format %{ "csel $dst, $src2, $src1 $cmp\t# unsigned, long" %}
10444
10445 ins_encode %{
10446 __ csel(as_Register($dst$$reg),
10447 as_Register($src2$$reg),
10448 as_Register($src1$$reg),
10449 (Assembler::Condition)$cmp$$cmpcode);
10450 %}
10451
10452 ins_pipe(icond_reg_reg);
10453 %}
10454
10455 // special cases where one arg is zero
10456
10457 instruct cmovL_reg_zero(cmpOp cmp, rFlagsReg cr, iRegLNoSp dst, iRegL src, immL0 zero) %{
10458 match(Set dst (CMoveL (Binary cmp cr) (Binary src zero)));
10459
10460 ins_cost(INSN_COST * 2);
10461 format %{ "csel $dst, zr, $src $cmp\t# signed, long" %}
10462
10463 ins_encode %{
10464 __ csel(as_Register($dst$$reg),
10465 zr,
10466 as_Register($src$$reg),
10467 (Assembler::Condition)$cmp$$cmpcode);
10468 %}
10469
10470 ins_pipe(icond_reg);
10471 %}
10472
10473 instruct cmovUL_reg_zero(cmpOpU cmp, rFlagsRegU cr, iRegLNoSp dst, iRegL src, immL0 zero) %{
10474 match(Set dst (CMoveL (Binary cmp cr) (Binary src zero)));
10475
10476 ins_cost(INSN_COST * 2);
10477 format %{ "csel $dst, zr, $src $cmp\t# unsigned, long" %}
10478
10479 ins_encode %{
10480 __ csel(as_Register($dst$$reg),
10481 zr,
10482 as_Register($src$$reg),
10483 (Assembler::Condition)$cmp$$cmpcode);
10484 %}
10485
10486 ins_pipe(icond_reg);
10487 %}
10488
10489 instruct cmovL_zero_reg(cmpOp cmp, rFlagsReg cr, iRegLNoSp dst, immL0 zero, iRegL src) %{
10490 match(Set dst (CMoveL (Binary cmp cr) (Binary zero src)));
10491
10492 ins_cost(INSN_COST * 2);
10493 format %{ "csel $dst, $src, zr $cmp\t# signed, long" %}
10494
10495 ins_encode %{
10496 __ csel(as_Register($dst$$reg),
10497 as_Register($src$$reg),
10498 zr,
10499 (Assembler::Condition)$cmp$$cmpcode);
10500 %}
10501
10502 ins_pipe(icond_reg);
10503 %}
10504
10505 instruct cmovUL_zero_reg(cmpOpU cmp, rFlagsRegU cr, iRegLNoSp dst, immL0 zero, iRegL src) %{
10506 match(Set dst (CMoveL (Binary cmp cr) (Binary zero src)));
10507
10508 ins_cost(INSN_COST * 2);
10509 format %{ "csel $dst, $src, zr $cmp\t# unsigned, long" %}
10510
10511 ins_encode %{
10512 __ csel(as_Register($dst$$reg),
10513 as_Register($src$$reg),
10514 zr,
10515 (Assembler::Condition)$cmp$$cmpcode);
10516 %}
10517
10518 ins_pipe(icond_reg);
10519 %}
10520
10521 instruct cmovP_reg_reg(cmpOp cmp, rFlagsReg cr, iRegPNoSp dst, iRegP src1, iRegP src2) %{
10522 match(Set dst (CMoveP (Binary cmp cr) (Binary src1 src2)));
10523
10524 ins_cost(INSN_COST * 2);
10525 format %{ "csel $dst, $src2, $src1 $cmp\t# signed, ptr" %}
10526
10527 ins_encode %{
10528 __ csel(as_Register($dst$$reg),
10529 as_Register($src2$$reg),
10530 as_Register($src1$$reg),
10531 (Assembler::Condition)$cmp$$cmpcode);
10532 %}
10533
10534 ins_pipe(icond_reg_reg);
10535 %}
10536
10537 instruct cmovUP_reg_reg(cmpOpU cmp, rFlagsRegU cr, iRegPNoSp dst, iRegP src1, iRegP src2) %{
10538 match(Set dst (CMoveP (Binary cmp cr) (Binary src1 src2)));
10539
10540 ins_cost(INSN_COST * 2);
10541 format %{ "csel $dst, $src2, $src1 $cmp\t# unsigned, ptr" %}
10542
10543 ins_encode %{
10544 __ csel(as_Register($dst$$reg),
10545 as_Register($src2$$reg),
10546 as_Register($src1$$reg),
10547 (Assembler::Condition)$cmp$$cmpcode);
10548 %}
10549
10550 ins_pipe(icond_reg_reg);
10551 %}
10552
10553 // special cases where one arg is zero
10554
10555 instruct cmovP_reg_zero(cmpOp cmp, rFlagsReg cr, iRegPNoSp dst, iRegP src, immP0 zero) %{
10556 match(Set dst (CMoveP (Binary cmp cr) (Binary src zero)));
10557
10558 ins_cost(INSN_COST * 2);
10559 format %{ "csel $dst, zr, $src $cmp\t# signed, ptr" %}
10560
10561 ins_encode %{
10562 __ csel(as_Register($dst$$reg),
10563 zr,
10564 as_Register($src$$reg),
10565 (Assembler::Condition)$cmp$$cmpcode);
10566 %}
10567
10568 ins_pipe(icond_reg);
10569 %}
10570
10571 instruct cmovUP_reg_zero(cmpOpU cmp, rFlagsRegU cr, iRegPNoSp dst, iRegP src, immP0 zero) %{
10572 match(Set dst (CMoveP (Binary cmp cr) (Binary src zero)));
10573
10574 ins_cost(INSN_COST * 2);
10575 format %{ "csel $dst, zr, $src $cmp\t# unsigned, ptr" %}
10576
10577 ins_encode %{
10578 __ csel(as_Register($dst$$reg),
10579 zr,
10580 as_Register($src$$reg),
10581 (Assembler::Condition)$cmp$$cmpcode);
10582 %}
10583
10584 ins_pipe(icond_reg);
10585 %}
10586
10587 instruct cmovP_zero_reg(cmpOp cmp, rFlagsReg cr, iRegPNoSp dst, immP0 zero, iRegP src) %{
10588 match(Set dst (CMoveP (Binary cmp cr) (Binary zero src)));
10589
10590 ins_cost(INSN_COST * 2);
10591 format %{ "csel $dst, $src, zr $cmp\t# signed, ptr" %}
10592
10593 ins_encode %{
10594 __ csel(as_Register($dst$$reg),
10595 as_Register($src$$reg),
10596 zr,
10597 (Assembler::Condition)$cmp$$cmpcode);
10598 %}
10599
10600 ins_pipe(icond_reg);
10601 %}
10602
10603 instruct cmovUP_zero_reg(cmpOpU cmp, rFlagsRegU cr, iRegPNoSp dst, immP0 zero, iRegP src) %{
10604 match(Set dst (CMoveP (Binary cmp cr) (Binary zero src)));
10605
10606 ins_cost(INSN_COST * 2);
10607 format %{ "csel $dst, $src, zr $cmp\t# unsigned, ptr" %}
10608
10609 ins_encode %{
10610 __ csel(as_Register($dst$$reg),
10611 as_Register($src$$reg),
10612 zr,
10613 (Assembler::Condition)$cmp$$cmpcode);
10614 %}
10615
10616 ins_pipe(icond_reg);
10617 %}
10618
10619 instruct cmovN_reg_reg(cmpOp cmp, rFlagsReg cr, iRegNNoSp dst, iRegN src1, iRegN src2) %{
10620 match(Set dst (CMoveN (Binary cmp cr) (Binary src1 src2)));
10621
10622 ins_cost(INSN_COST * 2);
10623 format %{ "cselw $dst, $src2, $src1 $cmp\t# signed, compressed ptr" %}
10624
10625 ins_encode %{
10626 __ cselw(as_Register($dst$$reg),
10627 as_Register($src2$$reg),
10628 as_Register($src1$$reg),
10629 (Assembler::Condition)$cmp$$cmpcode);
10630 %}
10631
10632 ins_pipe(icond_reg_reg);
10633 %}
10634
10635 instruct cmovUN_reg_reg(cmpOpU cmp, rFlagsRegU cr, iRegNNoSp dst, iRegN src1, iRegN src2) %{
10636 match(Set dst (CMoveN (Binary cmp cr) (Binary src1 src2)));
10637
10638 ins_cost(INSN_COST * 2);
10639 format %{ "cselw $dst, $src2, $src1 $cmp\t# signed, compressed ptr" %}
10640
10641 ins_encode %{
10642 __ cselw(as_Register($dst$$reg),
10643 as_Register($src2$$reg),
10644 as_Register($src1$$reg),
10645 (Assembler::Condition)$cmp$$cmpcode);
10646 %}
10647
10648 ins_pipe(icond_reg_reg);
10649 %}
10650
10651 // special cases where one arg is zero
10652
10653 instruct cmovN_reg_zero(cmpOp cmp, rFlagsReg cr, iRegNNoSp dst, iRegN src, immN0 zero) %{
10654 match(Set dst (CMoveN (Binary cmp cr) (Binary src zero)));
10655
10656 ins_cost(INSN_COST * 2);
10657 format %{ "cselw $dst, zr, $src $cmp\t# signed, compressed ptr" %}
10658
10659 ins_encode %{
10660 __ cselw(as_Register($dst$$reg),
10661 zr,
10662 as_Register($src$$reg),
10663 (Assembler::Condition)$cmp$$cmpcode);
10664 %}
10665
10666 ins_pipe(icond_reg);
10667 %}
10668
10669 instruct cmovUN_reg_zero(cmpOpU cmp, rFlagsRegU cr, iRegNNoSp dst, iRegN src, immN0 zero) %{
10670 match(Set dst (CMoveN (Binary cmp cr) (Binary src zero)));
10671
10672 ins_cost(INSN_COST * 2);
10673 format %{ "cselw $dst, zr, $src $cmp\t# unsigned, compressed ptr" %}
10674
10675 ins_encode %{
10676 __ cselw(as_Register($dst$$reg),
10677 zr,
10678 as_Register($src$$reg),
10679 (Assembler::Condition)$cmp$$cmpcode);
10680 %}
10681
10682 ins_pipe(icond_reg);
10683 %}
10684
10685 instruct cmovN_zero_reg(cmpOp cmp, rFlagsReg cr, iRegNNoSp dst, immN0 zero, iRegN src) %{
10686 match(Set dst (CMoveN (Binary cmp cr) (Binary zero src)));
10687
10688 ins_cost(INSN_COST * 2);
10689 format %{ "cselw $dst, $src, zr $cmp\t# signed, compressed ptr" %}
10690
10691 ins_encode %{
10692 __ cselw(as_Register($dst$$reg),
10693 as_Register($src$$reg),
10694 zr,
10695 (Assembler::Condition)$cmp$$cmpcode);
10696 %}
10697
10698 ins_pipe(icond_reg);
10699 %}
10700
10701 instruct cmovUN_zero_reg(cmpOpU cmp, rFlagsRegU cr, iRegNNoSp dst, immN0 zero, iRegN src) %{
10702 match(Set dst (CMoveN (Binary cmp cr) (Binary zero src)));
10703
10704 ins_cost(INSN_COST * 2);
10705 format %{ "cselw $dst, $src, zr $cmp\t# unsigned, compressed ptr" %}
10706
10707 ins_encode %{
10708 __ cselw(as_Register($dst$$reg),
10709 as_Register($src$$reg),
10710 zr,
10711 (Assembler::Condition)$cmp$$cmpcode);
10712 %}
10713
10714 ins_pipe(icond_reg);
10715 %}
10716
10717 instruct cmovF_reg(cmpOp cmp, rFlagsReg cr, vRegF dst, vRegF src1, vRegF src2)
10718 %{
10719 match(Set dst (CMoveF (Binary cmp cr) (Binary src1 src2)));
10720
10721 ins_cost(INSN_COST * 3);
10722
10723 format %{ "fcsels $dst, $src1, $src2, $cmp\t# signed cmove float\n\t" %}
10724 ins_encode %{
10725 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
10726 __ fcsels(as_FloatRegister($dst$$reg),
10727 as_FloatRegister($src2$$reg),
10728 as_FloatRegister($src1$$reg),
10729 cond);
10730 %}
10731
10732 ins_pipe(fp_cond_reg_reg_s);
10733 %}
10734
10735 instruct cmovUF_reg(cmpOpU cmp, rFlagsRegU cr, vRegF dst, vRegF src1, vRegF src2)
10736 %{
10737 match(Set dst (CMoveF (Binary cmp cr) (Binary src1 src2)));
10738
10739 ins_cost(INSN_COST * 3);
10740
10741 format %{ "fcsels $dst, $src1, $src2, $cmp\t# unsigned cmove float\n\t" %}
10742 ins_encode %{
10743 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
10744 __ fcsels(as_FloatRegister($dst$$reg),
10745 as_FloatRegister($src2$$reg),
10746 as_FloatRegister($src1$$reg),
10747 cond);
10748 %}
10749
10750 ins_pipe(fp_cond_reg_reg_s);
10751 %}
10752
10753 instruct cmovD_reg(cmpOp cmp, rFlagsReg cr, vRegD dst, vRegD src1, vRegD src2)
10754 %{
10755 match(Set dst (CMoveD (Binary cmp cr) (Binary src1 src2)));
10756
10757 ins_cost(INSN_COST * 3);
10758
10759 format %{ "fcseld $dst, $src1, $src2, $cmp\t# signed cmove float\n\t" %}
10760 ins_encode %{
10761 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
10762 __ fcseld(as_FloatRegister($dst$$reg),
10763 as_FloatRegister($src2$$reg),
10764 as_FloatRegister($src1$$reg),
10765 cond);
10766 %}
10767
10768 ins_pipe(fp_cond_reg_reg_d);
10769 %}
10770
10771 instruct cmovUD_reg(cmpOpU cmp, rFlagsRegU cr, vRegD dst, vRegD src1, vRegD src2)
10772 %{
10773 match(Set dst (CMoveD (Binary cmp cr) (Binary src1 src2)));
10774
10775 ins_cost(INSN_COST * 3);
10776
10777 format %{ "fcseld $dst, $src1, $src2, $cmp\t# unsigned cmove float\n\t" %}
10778 ins_encode %{
10779 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
10780 __ fcseld(as_FloatRegister($dst$$reg),
10781 as_FloatRegister($src2$$reg),
10782 as_FloatRegister($src1$$reg),
10783 cond);
10784 %}
10785
10786 ins_pipe(fp_cond_reg_reg_d);
10787 %}
10788
10789 // ============================================================================
10790 // Arithmetic Instructions
10791 //
10792
10793 // Integer Addition
10794
10795 // TODO
10796 // these currently employ operations which do not set CR and hence are
10797 // not flagged as killing CR but we would like to isolate the cases
10798 // where we want to set flags from those where we don't. need to work
10799 // out how to do that.
10800
10801 instruct addI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10802 match(Set dst (AddI src1 src2));
10803
10804 ins_cost(INSN_COST);
10805 format %{ "addw $dst, $src1, $src2" %}
10806
10807 ins_encode %{
10808 __ addw(as_Register($dst$$reg),
10809 as_Register($src1$$reg),
10810 as_Register($src2$$reg));
10811 %}
10812
10813 ins_pipe(ialu_reg_reg);
10814 %}
10815
10816 instruct addI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immIAddSub src2) %{
10817 match(Set dst (AddI src1 src2));
10818
10819 ins_cost(INSN_COST);
10820 format %{ "addw $dst, $src1, $src2" %}
10821
10822 // use opcode to indicate that this is an add not a sub
10823 opcode(0x0);
10824
10825 ins_encode(aarch64_enc_addsubw_imm(dst, src1, src2));
10826
10827 ins_pipe(ialu_reg_imm);
10828 %}
10829
10830 instruct addI_reg_imm_i2l(iRegINoSp dst, iRegL src1, immIAddSub src2) %{
10831 match(Set dst (AddI (ConvL2I src1) src2));
10832
10833 ins_cost(INSN_COST);
10834 format %{ "addw $dst, $src1, $src2" %}
10835
10836 // use opcode to indicate that this is an add not a sub
10837 opcode(0x0);
10838
10839 ins_encode(aarch64_enc_addsubw_imm(dst, src1, src2));
10840
10841 ins_pipe(ialu_reg_imm);
10842 %}
10843
10844 // Pointer Addition
10845 instruct addP_reg_reg(iRegPNoSp dst, iRegP src1, iRegL src2) %{
10846 match(Set dst (AddP src1 src2));
10847
10848 ins_cost(INSN_COST);
10849 format %{ "add $dst, $src1, $src2\t# ptr" %}
10850
10851 ins_encode %{
10852 __ add(as_Register($dst$$reg),
10853 as_Register($src1$$reg),
10854 as_Register($src2$$reg));
10855 %}
10856
10857 ins_pipe(ialu_reg_reg);
10858 %}
10859
10860 instruct addP_reg_reg_ext(iRegPNoSp dst, iRegP src1, iRegIorL2I src2) %{
10861 match(Set dst (AddP src1 (ConvI2L src2)));
10862
10863 ins_cost(1.9 * INSN_COST);
10864 format %{ "add $dst, $src1, $src2, sxtw\t# ptr" %}
10865
10866 ins_encode %{
10867 __ add(as_Register($dst$$reg),
10868 as_Register($src1$$reg),
10869 as_Register($src2$$reg), ext::sxtw);
10870 %}
10871
10872 ins_pipe(ialu_reg_reg);
10873 %}
10874
10875 instruct addP_reg_reg_lsl(iRegPNoSp dst, iRegP src1, iRegL src2, immIScale scale) %{
10876 match(Set dst (AddP src1 (LShiftL src2 scale)));
10877
10878 ins_cost(1.9 * INSN_COST);
10879 format %{ "add $dst, $src1, $src2, LShiftL $scale\t# ptr" %}
10880
10881 ins_encode %{
10882 __ lea(as_Register($dst$$reg),
10883 Address(as_Register($src1$$reg), as_Register($src2$$reg),
10884 Address::lsl($scale$$constant)));
10885 %}
10886
10887 ins_pipe(ialu_reg_reg_shift);
10888 %}
10889
10890 instruct addP_reg_reg_ext_shift(iRegPNoSp dst, iRegP src1, iRegIorL2I src2, immIScale scale) %{
10891 match(Set dst (AddP src1 (LShiftL (ConvI2L src2) scale)));
10892
10893 ins_cost(1.9 * INSN_COST);
10894 format %{ "add $dst, $src1, $src2, I2L $scale\t# ptr" %}
10895
10896 ins_encode %{
10897 __ lea(as_Register($dst$$reg),
10898 Address(as_Register($src1$$reg), as_Register($src2$$reg),
10899 Address::sxtw($scale$$constant)));
10900 %}
10901
10902 ins_pipe(ialu_reg_reg_shift);
10903 %}
10904
10905 instruct lshift_ext(iRegLNoSp dst, iRegIorL2I src, immI scale, rFlagsReg cr) %{
10906 match(Set dst (LShiftL (ConvI2L src) scale));
10907
10908 ins_cost(INSN_COST);
10909 format %{ "sbfiz $dst, $src, $scale & 63, -$scale & 63\t" %}
10910
10911 ins_encode %{
10912 __ sbfiz(as_Register($dst$$reg),
10913 as_Register($src$$reg),
10914 $scale$$constant & 63, MIN2(32, (int)((-$scale$$constant) & 63)));
10915 %}
10916
10917 ins_pipe(ialu_reg_shift);
10918 %}
10919
10920 // Pointer Immediate Addition
10921 // n.b. this needs to be more expensive than using an indirect memory
10922 // operand
10923 instruct addP_reg_imm(iRegPNoSp dst, iRegP src1, immLAddSub src2) %{
10924 match(Set dst (AddP src1 src2));
10925
10926 ins_cost(INSN_COST);
10927 format %{ "add $dst, $src1, $src2\t# ptr" %}
10928
10929 // use opcode to indicate that this is an add not a sub
10930 opcode(0x0);
10931
10932 ins_encode( aarch64_enc_addsub_imm(dst, src1, src2) );
10933
10934 ins_pipe(ialu_reg_imm);
10935 %}
10936
10937 // Long Addition
10938 instruct addL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
10939
10940 match(Set dst (AddL src1 src2));
10941
10942 ins_cost(INSN_COST);
10943 format %{ "add $dst, $src1, $src2" %}
10944
10945 ins_encode %{
10946 __ add(as_Register($dst$$reg),
10947 as_Register($src1$$reg),
10948 as_Register($src2$$reg));
10949 %}
10950
10951 ins_pipe(ialu_reg_reg);
10952 %}
10953
10954 // No constant pool entries requiredLong Immediate Addition.
10955 instruct addL_reg_imm(iRegLNoSp dst, iRegL src1, immLAddSub src2) %{
10956 match(Set dst (AddL src1 src2));
10957
10958 ins_cost(INSN_COST);
10959 format %{ "add $dst, $src1, $src2" %}
10960
10961 // use opcode to indicate that this is an add not a sub
10962 opcode(0x0);
10963
10964 ins_encode( aarch64_enc_addsub_imm(dst, src1, src2) );
10965
10966 ins_pipe(ialu_reg_imm);
10967 %}
10968
10969 // Integer Subtraction
10970 instruct subI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10971 match(Set dst (SubI src1 src2));
10972
10973 ins_cost(INSN_COST);
10974 format %{ "subw $dst, $src1, $src2" %}
10975
10976 ins_encode %{
10977 __ subw(as_Register($dst$$reg),
10978 as_Register($src1$$reg),
10979 as_Register($src2$$reg));
10980 %}
10981
10982 ins_pipe(ialu_reg_reg);
10983 %}
10984
10985 // Immediate Subtraction
10986 instruct subI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immIAddSub src2) %{
10987 match(Set dst (SubI src1 src2));
10988
10989 ins_cost(INSN_COST);
10990 format %{ "subw $dst, $src1, $src2" %}
10991
10992 // use opcode to indicate that this is a sub not an add
10993 opcode(0x1);
10994
10995 ins_encode(aarch64_enc_addsubw_imm(dst, src1, src2));
10996
10997 ins_pipe(ialu_reg_imm);
10998 %}
10999
11000 // Long Subtraction
11001 instruct subL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
11002
11003 match(Set dst (SubL src1 src2));
11004
11005 ins_cost(INSN_COST);
11006 format %{ "sub $dst, $src1, $src2" %}
11007
11008 ins_encode %{
11009 __ sub(as_Register($dst$$reg),
11010 as_Register($src1$$reg),
11011 as_Register($src2$$reg));
11012 %}
11013
11014 ins_pipe(ialu_reg_reg);
11015 %}
11016
11017 // No constant pool entries requiredLong Immediate Subtraction.
11018 instruct subL_reg_imm(iRegLNoSp dst, iRegL src1, immLAddSub src2) %{
11019 match(Set dst (SubL src1 src2));
11020
11021 ins_cost(INSN_COST);
11022 format %{ "sub$dst, $src1, $src2" %}
11023
11024 // use opcode to indicate that this is a sub not an add
11025 opcode(0x1);
11026
11027 ins_encode( aarch64_enc_addsub_imm(dst, src1, src2) );
11028
11029 ins_pipe(ialu_reg_imm);
11030 %}
11031
11032 // Integer Negation (special case for sub)
11033
11034 instruct negI_reg(iRegINoSp dst, iRegIorL2I src, immI0 zero, rFlagsReg cr) %{
11035 match(Set dst (SubI zero src));
11036
11037 ins_cost(INSN_COST);
11038 format %{ "negw $dst, $src\t# int" %}
11039
11040 ins_encode %{
11041 __ negw(as_Register($dst$$reg),
11042 as_Register($src$$reg));
11043 %}
11044
11045 ins_pipe(ialu_reg);
11046 %}
11047
11048 // Long Negation
11049
11050 instruct negL_reg(iRegLNoSp dst, iRegL src, immL0 zero, rFlagsReg cr) %{
11051 match(Set dst (SubL zero src));
11052
11053 ins_cost(INSN_COST);
11054 format %{ "neg $dst, $src\t# long" %}
11055
11056 ins_encode %{
11057 __ neg(as_Register($dst$$reg),
11058 as_Register($src$$reg));
11059 %}
11060
11061 ins_pipe(ialu_reg);
11062 %}
11063
11064 // Integer Multiply
11065
11066 instruct mulI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
11067 match(Set dst (MulI src1 src2));
11068
11069 ins_cost(INSN_COST * 3);
11070 format %{ "mulw $dst, $src1, $src2" %}
11071
11072 ins_encode %{
11073 __ mulw(as_Register($dst$$reg),
11074 as_Register($src1$$reg),
11075 as_Register($src2$$reg));
11076 %}
11077
11078 ins_pipe(imul_reg_reg);
11079 %}
11080
11081 instruct smulI(iRegLNoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
11082 match(Set dst (MulL (ConvI2L src1) (ConvI2L src2)));
11083
11084 ins_cost(INSN_COST * 3);
11085 format %{ "smull $dst, $src1, $src2" %}
11086
11087 ins_encode %{
11088 __ smull(as_Register($dst$$reg),
11089 as_Register($src1$$reg),
11090 as_Register($src2$$reg));
11091 %}
11092
11093 ins_pipe(imul_reg_reg);
11094 %}
11095
11096 // Long Multiply
11097
11098 instruct mulL(iRegLNoSp dst, iRegL src1, iRegL src2) %{
11099 match(Set dst (MulL src1 src2));
11100
11101 ins_cost(INSN_COST * 5);
11102 format %{ "mul $dst, $src1, $src2" %}
11103
11104 ins_encode %{
11105 __ mul(as_Register($dst$$reg),
11106 as_Register($src1$$reg),
11107 as_Register($src2$$reg));
11108 %}
11109
11110 ins_pipe(lmul_reg_reg);
11111 %}
11112
11113 instruct mulHiL_rReg(iRegLNoSp dst, iRegL src1, iRegL src2, rFlagsReg cr)
11114 %{
11115 match(Set dst (MulHiL src1 src2));
11116
11117 ins_cost(INSN_COST * 7);
11118 format %{ "smulh $dst, $src1, $src2, \t# mulhi" %}
11119
11120 ins_encode %{
11121 __ smulh(as_Register($dst$$reg),
11122 as_Register($src1$$reg),
11123 as_Register($src2$$reg));
11124 %}
11125
11126 ins_pipe(lmul_reg_reg);
11127 %}
11128
11129 // Combined Integer Multiply & Add/Sub
11130
11131 instruct maddI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegIorL2I src3) %{
11132 match(Set dst (AddI src3 (MulI src1 src2)));
11133
11134 ins_cost(INSN_COST * 3);
11135 format %{ "madd $dst, $src1, $src2, $src3" %}
11136
11137 ins_encode %{
11138 __ maddw(as_Register($dst$$reg),
11139 as_Register($src1$$reg),
11140 as_Register($src2$$reg),
11141 as_Register($src3$$reg));
11142 %}
11143
11144 ins_pipe(imac_reg_reg);
11145 %}
11146
11147 instruct msubI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegIorL2I src3) %{
11148 match(Set dst (SubI src3 (MulI src1 src2)));
11149
11150 ins_cost(INSN_COST * 3);
11151 format %{ "msub $dst, $src1, $src2, $src3" %}
11152
11153 ins_encode %{
11154 __ msubw(as_Register($dst$$reg),
11155 as_Register($src1$$reg),
11156 as_Register($src2$$reg),
11157 as_Register($src3$$reg));
11158 %}
11159
11160 ins_pipe(imac_reg_reg);
11161 %}
11162
11163 // Combined Integer Multiply & Neg
11164
11165 instruct mnegI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI0 zero) %{
11166 match(Set dst (MulI (SubI zero src1) src2));
11167
11168 ins_cost(INSN_COST * 3);
11169 format %{ "mneg $dst, $src1, $src2" %}
11170
11171 ins_encode %{
11172 __ mnegw(as_Register($dst$$reg),
11173 as_Register($src1$$reg),
11174 as_Register($src2$$reg));
11175 %}
11176
11177 ins_pipe(imac_reg_reg);
11178 %}
11179
11180 // Combined Long Multiply & Add/Sub
11181
11182 instruct maddL(iRegLNoSp dst, iRegL src1, iRegL src2, iRegL src3) %{
11183 match(Set dst (AddL src3 (MulL src1 src2)));
11184
11185 ins_cost(INSN_COST * 5);
11186 format %{ "madd $dst, $src1, $src2, $src3" %}
11187
11188 ins_encode %{
11189 __ madd(as_Register($dst$$reg),
11190 as_Register($src1$$reg),
11191 as_Register($src2$$reg),
11192 as_Register($src3$$reg));
11193 %}
11194
11195 ins_pipe(lmac_reg_reg);
11196 %}
11197
11198 instruct msubL(iRegLNoSp dst, iRegL src1, iRegL src2, iRegL src3) %{
11199 match(Set dst (SubL src3 (MulL src1 src2)));
11200
11201 ins_cost(INSN_COST * 5);
11202 format %{ "msub $dst, $src1, $src2, $src3" %}
11203
11204 ins_encode %{
11205 __ msub(as_Register($dst$$reg),
11206 as_Register($src1$$reg),
11207 as_Register($src2$$reg),
11208 as_Register($src3$$reg));
11209 %}
11210
11211 ins_pipe(lmac_reg_reg);
11212 %}
11213
11214 // Combined Long Multiply & Neg
11215
11216 instruct mnegL(iRegLNoSp dst, iRegL src1, iRegL src2, immL0 zero) %{
11217 match(Set dst (MulL (SubL zero src1) src2));
11218
11219 ins_cost(INSN_COST * 5);
11220 format %{ "mneg $dst, $src1, $src2" %}
11221
11222 ins_encode %{
11223 __ mneg(as_Register($dst$$reg),
11224 as_Register($src1$$reg),
11225 as_Register($src2$$reg));
11226 %}
11227
11228 ins_pipe(lmac_reg_reg);
11229 %}
11230
11231 // Combine Integer Signed Multiply & Add/Sub/Neg Long
11232
11233 instruct smaddL(iRegLNoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegLNoSp src3) %{
11234 match(Set dst (AddL src3 (MulL (ConvI2L src1) (ConvI2L src2))));
11235
11236 ins_cost(INSN_COST * 3);
11237 format %{ "smaddl $dst, $src1, $src2, $src3" %}
11238
11239 ins_encode %{
11240 __ smaddl(as_Register($dst$$reg),
11241 as_Register($src1$$reg),
11242 as_Register($src2$$reg),
11243 as_Register($src3$$reg));
11244 %}
11245
11246 ins_pipe(imac_reg_reg);
11247 %}
11248
11249 instruct smsubL(iRegLNoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegLNoSp src3) %{
11250 match(Set dst (SubL src3 (MulL (ConvI2L src1) (ConvI2L src2))));
11251
11252 ins_cost(INSN_COST * 3);
11253 format %{ "smsubl $dst, $src1, $src2, $src3" %}
11254
11255 ins_encode %{
11256 __ smsubl(as_Register($dst$$reg),
11257 as_Register($src1$$reg),
11258 as_Register($src2$$reg),
11259 as_Register($src3$$reg));
11260 %}
11261
11262 ins_pipe(imac_reg_reg);
11263 %}
11264
11265 instruct smnegL(iRegLNoSp dst, iRegIorL2I src1, iRegIorL2I src2, immL0 zero) %{
11266 match(Set dst (MulL (SubL zero (ConvI2L src1)) (ConvI2L src2)));
11267
11268 ins_cost(INSN_COST * 3);
11269 format %{ "smnegl $dst, $src1, $src2" %}
11270
11271 ins_encode %{
11272 __ smnegl(as_Register($dst$$reg),
11273 as_Register($src1$$reg),
11274 as_Register($src2$$reg));
11275 %}
11276
11277 ins_pipe(imac_reg_reg);
11278 %}
11279
11280 // Combined Multiply-Add Shorts into Integer (dst = src1 * src2 + src3 * src4)
11281
11282 instruct muladdS2I(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegIorL2I src3, iRegIorL2I src4) %{
11283 match(Set dst (MulAddS2I (Binary src1 src2) (Binary src3 src4)));
11284
11285 ins_cost(INSN_COST * 5);
11286 format %{ "mulw rscratch1, $src1, $src2\n\t"
11287 "maddw $dst, $src3, $src4, rscratch1" %}
11288
11289 ins_encode %{
11290 __ mulw(rscratch1, as_Register($src1$$reg), as_Register($src2$$reg));
11291 __ maddw(as_Register($dst$$reg), as_Register($src3$$reg), as_Register($src4$$reg), rscratch1); %}
11292
11293 ins_pipe(imac_reg_reg);
11294 %}
11295
11296 // Integer Divide
11297
11298 instruct divI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
11299 match(Set dst (DivI src1 src2));
11300
11301 ins_cost(INSN_COST * 19);
11302 format %{ "sdivw $dst, $src1, $src2" %}
11303
11304 ins_encode(aarch64_enc_divw(dst, src1, src2));
11305 ins_pipe(idiv_reg_reg);
11306 %}
11307
11308 // Long Divide
11309
11310 instruct divL(iRegLNoSp dst, iRegL src1, iRegL src2) %{
11311 match(Set dst (DivL src1 src2));
11312
11313 ins_cost(INSN_COST * 35);
11314 format %{ "sdiv $dst, $src1, $src2" %}
11315
11316 ins_encode(aarch64_enc_div(dst, src1, src2));
11317 ins_pipe(ldiv_reg_reg);
11318 %}
11319
11320 // Integer Remainder
11321
11322 instruct modI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
11323 match(Set dst (ModI src1 src2));
11324
11325 ins_cost(INSN_COST * 22);
11326 format %{ "sdivw rscratch1, $src1, $src2\n\t"
11327 "msubw($dst, rscratch1, $src2, $src1" %}
11328
11329 ins_encode(aarch64_enc_modw(dst, src1, src2));
11330 ins_pipe(idiv_reg_reg);
11331 %}
11332
11333 // Long Remainder
11334
11335 instruct modL(iRegLNoSp dst, iRegL src1, iRegL src2) %{
11336 match(Set dst (ModL src1 src2));
11337
11338 ins_cost(INSN_COST * 38);
11339 format %{ "sdiv rscratch1, $src1, $src2\n"
11340 "msub($dst, rscratch1, $src2, $src1" %}
11341
11342 ins_encode(aarch64_enc_mod(dst, src1, src2));
11343 ins_pipe(ldiv_reg_reg);
11344 %}
11345
11346 // Integer Shifts
11347
11348 // Shift Left Register
11349 instruct lShiftI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
11350 match(Set dst (LShiftI src1 src2));
11351
11352 ins_cost(INSN_COST * 2);
11353 format %{ "lslvw $dst, $src1, $src2" %}
11354
11355 ins_encode %{
11356 __ lslvw(as_Register($dst$$reg),
11357 as_Register($src1$$reg),
11358 as_Register($src2$$reg));
11359 %}
11360
11361 ins_pipe(ialu_reg_reg_vshift);
11362 %}
11363
11364 // Shift Left Immediate
11365 instruct lShiftI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immI src2) %{
11366 match(Set dst (LShiftI src1 src2));
11367
11368 ins_cost(INSN_COST);
11369 format %{ "lslw $dst, $src1, ($src2 & 0x1f)" %}
11370
11371 ins_encode %{
11372 __ lslw(as_Register($dst$$reg),
11373 as_Register($src1$$reg),
11374 $src2$$constant & 0x1f);
11375 %}
11376
11377 ins_pipe(ialu_reg_shift);
11378 %}
11379
11380 // Shift Right Logical Register
11381 instruct urShiftI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
11382 match(Set dst (URShiftI src1 src2));
11383
11384 ins_cost(INSN_COST * 2);
11385 format %{ "lsrvw $dst, $src1, $src2" %}
11386
11387 ins_encode %{
11388 __ lsrvw(as_Register($dst$$reg),
11389 as_Register($src1$$reg),
11390 as_Register($src2$$reg));
11391 %}
11392
11393 ins_pipe(ialu_reg_reg_vshift);
11394 %}
11395
11396 // Shift Right Logical Immediate
11397 instruct urShiftI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immI src2) %{
11398 match(Set dst (URShiftI src1 src2));
11399
11400 ins_cost(INSN_COST);
11401 format %{ "lsrw $dst, $src1, ($src2 & 0x1f)" %}
11402
11403 ins_encode %{
11404 __ lsrw(as_Register($dst$$reg),
11405 as_Register($src1$$reg),
11406 $src2$$constant & 0x1f);
11407 %}
11408
11409 ins_pipe(ialu_reg_shift);
11410 %}
11411
11412 // Shift Right Arithmetic Register
11413 instruct rShiftI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
11414 match(Set dst (RShiftI src1 src2));
11415
11416 ins_cost(INSN_COST * 2);
11417 format %{ "asrvw $dst, $src1, $src2" %}
11418
11419 ins_encode %{
11420 __ asrvw(as_Register($dst$$reg),
11421 as_Register($src1$$reg),
11422 as_Register($src2$$reg));
11423 %}
11424
11425 ins_pipe(ialu_reg_reg_vshift);
11426 %}
11427
11428 // Shift Right Arithmetic Immediate
11429 instruct rShiftI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immI src2) %{
11430 match(Set dst (RShiftI src1 src2));
11431
11432 ins_cost(INSN_COST);
11433 format %{ "asrw $dst, $src1, ($src2 & 0x1f)" %}
11434
11435 ins_encode %{
11436 __ asrw(as_Register($dst$$reg),
11437 as_Register($src1$$reg),
11438 $src2$$constant & 0x1f);
11439 %}
11440
11441 ins_pipe(ialu_reg_shift);
11442 %}
11443
11444 // Combined Int Mask and Right Shift (using UBFM)
11445 // TODO
11446
11447 // Long Shifts
11448
11449 // Shift Left Register
11450 instruct lShiftL_reg_reg(iRegLNoSp dst, iRegL src1, iRegIorL2I src2) %{
11451 match(Set dst (LShiftL src1 src2));
11452
11453 ins_cost(INSN_COST * 2);
11454 format %{ "lslv $dst, $src1, $src2" %}
11455
11456 ins_encode %{
11457 __ lslv(as_Register($dst$$reg),
11458 as_Register($src1$$reg),
11459 as_Register($src2$$reg));
11460 %}
11461
11462 ins_pipe(ialu_reg_reg_vshift);
11463 %}
11464
11465 // Shift Left Immediate
11466 instruct lShiftL_reg_imm(iRegLNoSp dst, iRegL src1, immI src2) %{
11467 match(Set dst (LShiftL src1 src2));
11468
11469 ins_cost(INSN_COST);
11470 format %{ "lsl $dst, $src1, ($src2 & 0x3f)" %}
11471
11472 ins_encode %{
11473 __ lsl(as_Register($dst$$reg),
11474 as_Register($src1$$reg),
11475 $src2$$constant & 0x3f);
11476 %}
11477
11478 ins_pipe(ialu_reg_shift);
11479 %}
11480
11481 // Shift Right Logical Register
11482 instruct urShiftL_reg_reg(iRegLNoSp dst, iRegL src1, iRegIorL2I src2) %{
11483 match(Set dst (URShiftL src1 src2));
11484
11485 ins_cost(INSN_COST * 2);
11486 format %{ "lsrv $dst, $src1, $src2" %}
11487
11488 ins_encode %{
11489 __ lsrv(as_Register($dst$$reg),
11490 as_Register($src1$$reg),
11491 as_Register($src2$$reg));
11492 %}
11493
11494 ins_pipe(ialu_reg_reg_vshift);
11495 %}
11496
11497 // Shift Right Logical Immediate
11498 instruct urShiftL_reg_imm(iRegLNoSp dst, iRegL src1, immI src2) %{
11499 match(Set dst (URShiftL src1 src2));
11500
11501 ins_cost(INSN_COST);
11502 format %{ "lsr $dst, $src1, ($src2 & 0x3f)" %}
11503
11504 ins_encode %{
11505 __ lsr(as_Register($dst$$reg),
11506 as_Register($src1$$reg),
11507 $src2$$constant & 0x3f);
11508 %}
11509
11510 ins_pipe(ialu_reg_shift);
11511 %}
11512
11513 // A special-case pattern for card table stores.
11514 instruct urShiftP_reg_imm(iRegLNoSp dst, iRegP src1, immI src2) %{
11515 match(Set dst (URShiftL (CastP2X src1) src2));
11516
11517 ins_cost(INSN_COST);
11518 format %{ "lsr $dst, p2x($src1), ($src2 & 0x3f)" %}
11519
11520 ins_encode %{
11521 __ lsr(as_Register($dst$$reg),
11522 as_Register($src1$$reg),
11523 $src2$$constant & 0x3f);
11524 %}
11525
11526 ins_pipe(ialu_reg_shift);
11527 %}
11528
11529 // Shift Right Arithmetic Register
11530 instruct rShiftL_reg_reg(iRegLNoSp dst, iRegL src1, iRegIorL2I src2) %{
11531 match(Set dst (RShiftL src1 src2));
11532
11533 ins_cost(INSN_COST * 2);
11534 format %{ "asrv $dst, $src1, $src2" %}
11535
11536 ins_encode %{
11537 __ asrv(as_Register($dst$$reg),
11538 as_Register($src1$$reg),
11539 as_Register($src2$$reg));
11540 %}
11541
11542 ins_pipe(ialu_reg_reg_vshift);
11543 %}
11544
11545 // Shift Right Arithmetic Immediate
11546 instruct rShiftL_reg_imm(iRegLNoSp dst, iRegL src1, immI src2) %{
11547 match(Set dst (RShiftL src1 src2));
11548
11549 ins_cost(INSN_COST);
11550 format %{ "asr $dst, $src1, ($src2 & 0x3f)" %}
11551
11552 ins_encode %{
11553 __ asr(as_Register($dst$$reg),
11554 as_Register($src1$$reg),
11555 $src2$$constant & 0x3f);
11556 %}
11557
11558 ins_pipe(ialu_reg_shift);
11559 %}
11560
11561 // BEGIN This section of the file is automatically generated. Do not edit --------------
11562 // This section is generated from aarch64_ad.m4
11563
11564
11565 // This pattern is automatically generated from aarch64_ad.m4
11566 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11567 instruct regL_not_reg(iRegLNoSp dst,
11568 iRegL src1, immL_M1 m1,
11569 rFlagsReg cr) %{
11570 match(Set dst (XorL src1 m1));
11571 ins_cost(INSN_COST);
11572 format %{ "eon $dst, $src1, zr" %}
11573
11574 ins_encode %{
11575 __ eon(as_Register($dst$$reg),
11576 as_Register($src1$$reg),
11577 zr,
11578 Assembler::LSL, 0);
11579 %}
11580
11581 ins_pipe(ialu_reg);
11582 %}
11583
11584 // This pattern is automatically generated from aarch64_ad.m4
11585 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11586 instruct regI_not_reg(iRegINoSp dst,
11587 iRegIorL2I src1, immI_M1 m1,
11588 rFlagsReg cr) %{
11589 match(Set dst (XorI src1 m1));
11590 ins_cost(INSN_COST);
11591 format %{ "eonw $dst, $src1, zr" %}
11592
11593 ins_encode %{
11594 __ eonw(as_Register($dst$$reg),
11595 as_Register($src1$$reg),
11596 zr,
11597 Assembler::LSL, 0);
11598 %}
11599
11600 ins_pipe(ialu_reg);
11601 %}
11602
11603 // This pattern is automatically generated from aarch64_ad.m4
11604 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11605 instruct NegI_reg_URShift_reg(iRegINoSp dst,
11606 immI0 zero, iRegIorL2I src1, immI src2) %{
11607 match(Set dst (SubI zero (URShiftI src1 src2)));
11608
11609 ins_cost(1.9 * INSN_COST);
11610 format %{ "negw $dst, $src1, LSR $src2" %}
11611
11612 ins_encode %{
11613 __ negw(as_Register($dst$$reg), as_Register($src1$$reg),
11614 Assembler::LSR, $src2$$constant & 0x1f);
11615 %}
11616
11617 ins_pipe(ialu_reg_shift);
11618 %}
11619
11620 // This pattern is automatically generated from aarch64_ad.m4
11621 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11622 instruct NegI_reg_RShift_reg(iRegINoSp dst,
11623 immI0 zero, iRegIorL2I src1, immI src2) %{
11624 match(Set dst (SubI zero (RShiftI src1 src2)));
11625
11626 ins_cost(1.9 * INSN_COST);
11627 format %{ "negw $dst, $src1, ASR $src2" %}
11628
11629 ins_encode %{
11630 __ negw(as_Register($dst$$reg), as_Register($src1$$reg),
11631 Assembler::ASR, $src2$$constant & 0x1f);
11632 %}
11633
11634 ins_pipe(ialu_reg_shift);
11635 %}
11636
11637 // This pattern is automatically generated from aarch64_ad.m4
11638 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11639 instruct NegI_reg_LShift_reg(iRegINoSp dst,
11640 immI0 zero, iRegIorL2I src1, immI src2) %{
11641 match(Set dst (SubI zero (LShiftI src1 src2)));
11642
11643 ins_cost(1.9 * INSN_COST);
11644 format %{ "negw $dst, $src1, LSL $src2" %}
11645
11646 ins_encode %{
11647 __ negw(as_Register($dst$$reg), as_Register($src1$$reg),
11648 Assembler::LSL, $src2$$constant & 0x1f);
11649 %}
11650
11651 ins_pipe(ialu_reg_shift);
11652 %}
11653
11654 // This pattern is automatically generated from aarch64_ad.m4
11655 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11656 instruct NegL_reg_URShift_reg(iRegLNoSp dst,
11657 immL0 zero, iRegL src1, immI src2) %{
11658 match(Set dst (SubL zero (URShiftL src1 src2)));
11659
11660 ins_cost(1.9 * INSN_COST);
11661 format %{ "neg $dst, $src1, LSR $src2" %}
11662
11663 ins_encode %{
11664 __ neg(as_Register($dst$$reg), as_Register($src1$$reg),
11665 Assembler::LSR, $src2$$constant & 0x3f);
11666 %}
11667
11668 ins_pipe(ialu_reg_shift);
11669 %}
11670
11671 // This pattern is automatically generated from aarch64_ad.m4
11672 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11673 instruct NegL_reg_RShift_reg(iRegLNoSp dst,
11674 immL0 zero, iRegL src1, immI src2) %{
11675 match(Set dst (SubL zero (RShiftL src1 src2)));
11676
11677 ins_cost(1.9 * INSN_COST);
11678 format %{ "neg $dst, $src1, ASR $src2" %}
11679
11680 ins_encode %{
11681 __ neg(as_Register($dst$$reg), as_Register($src1$$reg),
11682 Assembler::ASR, $src2$$constant & 0x3f);
11683 %}
11684
11685 ins_pipe(ialu_reg_shift);
11686 %}
11687
11688 // This pattern is automatically generated from aarch64_ad.m4
11689 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11690 instruct NegL_reg_LShift_reg(iRegLNoSp dst,
11691 immL0 zero, iRegL src1, immI src2) %{
11692 match(Set dst (SubL zero (LShiftL src1 src2)));
11693
11694 ins_cost(1.9 * INSN_COST);
11695 format %{ "neg $dst, $src1, LSL $src2" %}
11696
11697 ins_encode %{
11698 __ neg(as_Register($dst$$reg), as_Register($src1$$reg),
11699 Assembler::LSL, $src2$$constant & 0x3f);
11700 %}
11701
11702 ins_pipe(ialu_reg_shift);
11703 %}
11704
11705 // This pattern is automatically generated from aarch64_ad.m4
11706 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11707 instruct AndI_reg_not_reg(iRegINoSp dst,
11708 iRegIorL2I src1, iRegIorL2I src2, immI_M1 m1) %{
11709 match(Set dst (AndI src1 (XorI src2 m1)));
11710 ins_cost(INSN_COST);
11711 format %{ "bicw $dst, $src1, $src2" %}
11712
11713 ins_encode %{
11714 __ bicw(as_Register($dst$$reg),
11715 as_Register($src1$$reg),
11716 as_Register($src2$$reg),
11717 Assembler::LSL, 0);
11718 %}
11719
11720 ins_pipe(ialu_reg_reg);
11721 %}
11722
11723 // This pattern is automatically generated from aarch64_ad.m4
11724 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11725 instruct AndL_reg_not_reg(iRegLNoSp dst,
11726 iRegL src1, iRegL src2, immL_M1 m1) %{
11727 match(Set dst (AndL src1 (XorL src2 m1)));
11728 ins_cost(INSN_COST);
11729 format %{ "bic $dst, $src1, $src2" %}
11730
11731 ins_encode %{
11732 __ bic(as_Register($dst$$reg),
11733 as_Register($src1$$reg),
11734 as_Register($src2$$reg),
11735 Assembler::LSL, 0);
11736 %}
11737
11738 ins_pipe(ialu_reg_reg);
11739 %}
11740
11741 // This pattern is automatically generated from aarch64_ad.m4
11742 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11743 instruct OrI_reg_not_reg(iRegINoSp dst,
11744 iRegIorL2I src1, iRegIorL2I src2, immI_M1 m1) %{
11745 match(Set dst (OrI src1 (XorI src2 m1)));
11746 ins_cost(INSN_COST);
11747 format %{ "ornw $dst, $src1, $src2" %}
11748
11749 ins_encode %{
11750 __ ornw(as_Register($dst$$reg),
11751 as_Register($src1$$reg),
11752 as_Register($src2$$reg),
11753 Assembler::LSL, 0);
11754 %}
11755
11756 ins_pipe(ialu_reg_reg);
11757 %}
11758
11759 // This pattern is automatically generated from aarch64_ad.m4
11760 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11761 instruct OrL_reg_not_reg(iRegLNoSp dst,
11762 iRegL src1, iRegL src2, immL_M1 m1) %{
11763 match(Set dst (OrL src1 (XorL src2 m1)));
11764 ins_cost(INSN_COST);
11765 format %{ "orn $dst, $src1, $src2" %}
11766
11767 ins_encode %{
11768 __ orn(as_Register($dst$$reg),
11769 as_Register($src1$$reg),
11770 as_Register($src2$$reg),
11771 Assembler::LSL, 0);
11772 %}
11773
11774 ins_pipe(ialu_reg_reg);
11775 %}
11776
11777 // This pattern is automatically generated from aarch64_ad.m4
11778 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11779 instruct XorI_reg_not_reg(iRegINoSp dst,
11780 iRegIorL2I src1, iRegIorL2I src2, immI_M1 m1) %{
11781 match(Set dst (XorI m1 (XorI src2 src1)));
11782 ins_cost(INSN_COST);
11783 format %{ "eonw $dst, $src1, $src2" %}
11784
11785 ins_encode %{
11786 __ eonw(as_Register($dst$$reg),
11787 as_Register($src1$$reg),
11788 as_Register($src2$$reg),
11789 Assembler::LSL, 0);
11790 %}
11791
11792 ins_pipe(ialu_reg_reg);
11793 %}
11794
11795 // This pattern is automatically generated from aarch64_ad.m4
11796 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11797 instruct XorL_reg_not_reg(iRegLNoSp dst,
11798 iRegL src1, iRegL src2, immL_M1 m1) %{
11799 match(Set dst (XorL m1 (XorL src2 src1)));
11800 ins_cost(INSN_COST);
11801 format %{ "eon $dst, $src1, $src2" %}
11802
11803 ins_encode %{
11804 __ eon(as_Register($dst$$reg),
11805 as_Register($src1$$reg),
11806 as_Register($src2$$reg),
11807 Assembler::LSL, 0);
11808 %}
11809
11810 ins_pipe(ialu_reg_reg);
11811 %}
11812
11813 // This pattern is automatically generated from aarch64_ad.m4
11814 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11815 // val & (-1 ^ (val >>> shift)) ==> bicw
11816 instruct AndI_reg_URShift_not_reg(iRegINoSp dst,
11817 iRegIorL2I src1, iRegIorL2I src2,
11818 immI src3, immI_M1 src4) %{
11819 match(Set dst (AndI src1 (XorI(URShiftI src2 src3) src4)));
11820 ins_cost(1.9 * INSN_COST);
11821 format %{ "bicw $dst, $src1, $src2, LSR $src3" %}
11822
11823 ins_encode %{
11824 __ bicw(as_Register($dst$$reg),
11825 as_Register($src1$$reg),
11826 as_Register($src2$$reg),
11827 Assembler::LSR,
11828 $src3$$constant & 0x1f);
11829 %}
11830
11831 ins_pipe(ialu_reg_reg_shift);
11832 %}
11833
11834 // This pattern is automatically generated from aarch64_ad.m4
11835 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11836 // val & (-1 ^ (val >>> shift)) ==> bic
11837 instruct AndL_reg_URShift_not_reg(iRegLNoSp dst,
11838 iRegL src1, iRegL src2,
11839 immI src3, immL_M1 src4) %{
11840 match(Set dst (AndL src1 (XorL(URShiftL src2 src3) src4)));
11841 ins_cost(1.9 * INSN_COST);
11842 format %{ "bic $dst, $src1, $src2, LSR $src3" %}
11843
11844 ins_encode %{
11845 __ bic(as_Register($dst$$reg),
11846 as_Register($src1$$reg),
11847 as_Register($src2$$reg),
11848 Assembler::LSR,
11849 $src3$$constant & 0x3f);
11850 %}
11851
11852 ins_pipe(ialu_reg_reg_shift);
11853 %}
11854
11855 // This pattern is automatically generated from aarch64_ad.m4
11856 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11857 // val & (-1 ^ (val >> shift)) ==> bicw
11858 instruct AndI_reg_RShift_not_reg(iRegINoSp dst,
11859 iRegIorL2I src1, iRegIorL2I src2,
11860 immI src3, immI_M1 src4) %{
11861 match(Set dst (AndI src1 (XorI(RShiftI src2 src3) src4)));
11862 ins_cost(1.9 * INSN_COST);
11863 format %{ "bicw $dst, $src1, $src2, ASR $src3" %}
11864
11865 ins_encode %{
11866 __ bicw(as_Register($dst$$reg),
11867 as_Register($src1$$reg),
11868 as_Register($src2$$reg),
11869 Assembler::ASR,
11870 $src3$$constant & 0x1f);
11871 %}
11872
11873 ins_pipe(ialu_reg_reg_shift);
11874 %}
11875
11876 // This pattern is automatically generated from aarch64_ad.m4
11877 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11878 // val & (-1 ^ (val >> shift)) ==> bic
11879 instruct AndL_reg_RShift_not_reg(iRegLNoSp dst,
11880 iRegL src1, iRegL src2,
11881 immI src3, immL_M1 src4) %{
11882 match(Set dst (AndL src1 (XorL(RShiftL src2 src3) src4)));
11883 ins_cost(1.9 * INSN_COST);
11884 format %{ "bic $dst, $src1, $src2, ASR $src3" %}
11885
11886 ins_encode %{
11887 __ bic(as_Register($dst$$reg),
11888 as_Register($src1$$reg),
11889 as_Register($src2$$reg),
11890 Assembler::ASR,
11891 $src3$$constant & 0x3f);
11892 %}
11893
11894 ins_pipe(ialu_reg_reg_shift);
11895 %}
11896
11897 // This pattern is automatically generated from aarch64_ad.m4
11898 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11899 // val & (-1 ^ (val ror shift)) ==> bicw
11900 instruct AndI_reg_RotateRight_not_reg(iRegINoSp dst,
11901 iRegIorL2I src1, iRegIorL2I src2,
11902 immI src3, immI_M1 src4) %{
11903 match(Set dst (AndI src1 (XorI(RotateRight src2 src3) src4)));
11904 ins_cost(1.9 * INSN_COST);
11905 format %{ "bicw $dst, $src1, $src2, ROR $src3" %}
11906
11907 ins_encode %{
11908 __ bicw(as_Register($dst$$reg),
11909 as_Register($src1$$reg),
11910 as_Register($src2$$reg),
11911 Assembler::ROR,
11912 $src3$$constant & 0x1f);
11913 %}
11914
11915 ins_pipe(ialu_reg_reg_shift);
11916 %}
11917
11918 // This pattern is automatically generated from aarch64_ad.m4
11919 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11920 // val & (-1 ^ (val ror shift)) ==> bic
11921 instruct AndL_reg_RotateRight_not_reg(iRegLNoSp dst,
11922 iRegL src1, iRegL src2,
11923 immI src3, immL_M1 src4) %{
11924 match(Set dst (AndL src1 (XorL(RotateRight src2 src3) src4)));
11925 ins_cost(1.9 * INSN_COST);
11926 format %{ "bic $dst, $src1, $src2, ROR $src3" %}
11927
11928 ins_encode %{
11929 __ bic(as_Register($dst$$reg),
11930 as_Register($src1$$reg),
11931 as_Register($src2$$reg),
11932 Assembler::ROR,
11933 $src3$$constant & 0x3f);
11934 %}
11935
11936 ins_pipe(ialu_reg_reg_shift);
11937 %}
11938
11939 // This pattern is automatically generated from aarch64_ad.m4
11940 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11941 // val & (-1 ^ (val << shift)) ==> bicw
11942 instruct AndI_reg_LShift_not_reg(iRegINoSp dst,
11943 iRegIorL2I src1, iRegIorL2I src2,
11944 immI src3, immI_M1 src4) %{
11945 match(Set dst (AndI src1 (XorI(LShiftI src2 src3) src4)));
11946 ins_cost(1.9 * INSN_COST);
11947 format %{ "bicw $dst, $src1, $src2, LSL $src3" %}
11948
11949 ins_encode %{
11950 __ bicw(as_Register($dst$$reg),
11951 as_Register($src1$$reg),
11952 as_Register($src2$$reg),
11953 Assembler::LSL,
11954 $src3$$constant & 0x1f);
11955 %}
11956
11957 ins_pipe(ialu_reg_reg_shift);
11958 %}
11959
11960 // This pattern is automatically generated from aarch64_ad.m4
11961 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11962 // val & (-1 ^ (val << shift)) ==> bic
11963 instruct AndL_reg_LShift_not_reg(iRegLNoSp dst,
11964 iRegL src1, iRegL src2,
11965 immI src3, immL_M1 src4) %{
11966 match(Set dst (AndL src1 (XorL(LShiftL src2 src3) src4)));
11967 ins_cost(1.9 * INSN_COST);
11968 format %{ "bic $dst, $src1, $src2, LSL $src3" %}
11969
11970 ins_encode %{
11971 __ bic(as_Register($dst$$reg),
11972 as_Register($src1$$reg),
11973 as_Register($src2$$reg),
11974 Assembler::LSL,
11975 $src3$$constant & 0x3f);
11976 %}
11977
11978 ins_pipe(ialu_reg_reg_shift);
11979 %}
11980
11981 // This pattern is automatically generated from aarch64_ad.m4
11982 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11983 // val ^ (-1 ^ (val >>> shift)) ==> eonw
11984 instruct XorI_reg_URShift_not_reg(iRegINoSp dst,
11985 iRegIorL2I src1, iRegIorL2I src2,
11986 immI src3, immI_M1 src4) %{
11987 match(Set dst (XorI src4 (XorI(URShiftI src2 src3) src1)));
11988 ins_cost(1.9 * INSN_COST);
11989 format %{ "eonw $dst, $src1, $src2, LSR $src3" %}
11990
11991 ins_encode %{
11992 __ eonw(as_Register($dst$$reg),
11993 as_Register($src1$$reg),
11994 as_Register($src2$$reg),
11995 Assembler::LSR,
11996 $src3$$constant & 0x1f);
11997 %}
11998
11999 ins_pipe(ialu_reg_reg_shift);
12000 %}
12001
12002 // This pattern is automatically generated from aarch64_ad.m4
12003 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12004 // val ^ (-1 ^ (val >>> shift)) ==> eon
12005 instruct XorL_reg_URShift_not_reg(iRegLNoSp dst,
12006 iRegL src1, iRegL src2,
12007 immI src3, immL_M1 src4) %{
12008 match(Set dst (XorL src4 (XorL(URShiftL src2 src3) src1)));
12009 ins_cost(1.9 * INSN_COST);
12010 format %{ "eon $dst, $src1, $src2, LSR $src3" %}
12011
12012 ins_encode %{
12013 __ eon(as_Register($dst$$reg),
12014 as_Register($src1$$reg),
12015 as_Register($src2$$reg),
12016 Assembler::LSR,
12017 $src3$$constant & 0x3f);
12018 %}
12019
12020 ins_pipe(ialu_reg_reg_shift);
12021 %}
12022
12023 // This pattern is automatically generated from aarch64_ad.m4
12024 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12025 // val ^ (-1 ^ (val >> shift)) ==> eonw
12026 instruct XorI_reg_RShift_not_reg(iRegINoSp dst,
12027 iRegIorL2I src1, iRegIorL2I src2,
12028 immI src3, immI_M1 src4) %{
12029 match(Set dst (XorI src4 (XorI(RShiftI src2 src3) src1)));
12030 ins_cost(1.9 * INSN_COST);
12031 format %{ "eonw $dst, $src1, $src2, ASR $src3" %}
12032
12033 ins_encode %{
12034 __ eonw(as_Register($dst$$reg),
12035 as_Register($src1$$reg),
12036 as_Register($src2$$reg),
12037 Assembler::ASR,
12038 $src3$$constant & 0x1f);
12039 %}
12040
12041 ins_pipe(ialu_reg_reg_shift);
12042 %}
12043
12044 // This pattern is automatically generated from aarch64_ad.m4
12045 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12046 // val ^ (-1 ^ (val >> shift)) ==> eon
12047 instruct XorL_reg_RShift_not_reg(iRegLNoSp dst,
12048 iRegL src1, iRegL src2,
12049 immI src3, immL_M1 src4) %{
12050 match(Set dst (XorL src4 (XorL(RShiftL src2 src3) src1)));
12051 ins_cost(1.9 * INSN_COST);
12052 format %{ "eon $dst, $src1, $src2, ASR $src3" %}
12053
12054 ins_encode %{
12055 __ eon(as_Register($dst$$reg),
12056 as_Register($src1$$reg),
12057 as_Register($src2$$reg),
12058 Assembler::ASR,
12059 $src3$$constant & 0x3f);
12060 %}
12061
12062 ins_pipe(ialu_reg_reg_shift);
12063 %}
12064
12065 // This pattern is automatically generated from aarch64_ad.m4
12066 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12067 // val ^ (-1 ^ (val ror shift)) ==> eonw
12068 instruct XorI_reg_RotateRight_not_reg(iRegINoSp dst,
12069 iRegIorL2I src1, iRegIorL2I src2,
12070 immI src3, immI_M1 src4) %{
12071 match(Set dst (XorI src4 (XorI(RotateRight src2 src3) src1)));
12072 ins_cost(1.9 * INSN_COST);
12073 format %{ "eonw $dst, $src1, $src2, ROR $src3" %}
12074
12075 ins_encode %{
12076 __ eonw(as_Register($dst$$reg),
12077 as_Register($src1$$reg),
12078 as_Register($src2$$reg),
12079 Assembler::ROR,
12080 $src3$$constant & 0x1f);
12081 %}
12082
12083 ins_pipe(ialu_reg_reg_shift);
12084 %}
12085
12086 // This pattern is automatically generated from aarch64_ad.m4
12087 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12088 // val ^ (-1 ^ (val ror shift)) ==> eon
12089 instruct XorL_reg_RotateRight_not_reg(iRegLNoSp dst,
12090 iRegL src1, iRegL src2,
12091 immI src3, immL_M1 src4) %{
12092 match(Set dst (XorL src4 (XorL(RotateRight src2 src3) src1)));
12093 ins_cost(1.9 * INSN_COST);
12094 format %{ "eon $dst, $src1, $src2, ROR $src3" %}
12095
12096 ins_encode %{
12097 __ eon(as_Register($dst$$reg),
12098 as_Register($src1$$reg),
12099 as_Register($src2$$reg),
12100 Assembler::ROR,
12101 $src3$$constant & 0x3f);
12102 %}
12103
12104 ins_pipe(ialu_reg_reg_shift);
12105 %}
12106
12107 // This pattern is automatically generated from aarch64_ad.m4
12108 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12109 // val ^ (-1 ^ (val << shift)) ==> eonw
12110 instruct XorI_reg_LShift_not_reg(iRegINoSp dst,
12111 iRegIorL2I src1, iRegIorL2I src2,
12112 immI src3, immI_M1 src4) %{
12113 match(Set dst (XorI src4 (XorI(LShiftI src2 src3) src1)));
12114 ins_cost(1.9 * INSN_COST);
12115 format %{ "eonw $dst, $src1, $src2, LSL $src3" %}
12116
12117 ins_encode %{
12118 __ eonw(as_Register($dst$$reg),
12119 as_Register($src1$$reg),
12120 as_Register($src2$$reg),
12121 Assembler::LSL,
12122 $src3$$constant & 0x1f);
12123 %}
12124
12125 ins_pipe(ialu_reg_reg_shift);
12126 %}
12127
12128 // This pattern is automatically generated from aarch64_ad.m4
12129 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12130 // val ^ (-1 ^ (val << shift)) ==> eon
12131 instruct XorL_reg_LShift_not_reg(iRegLNoSp dst,
12132 iRegL src1, iRegL src2,
12133 immI src3, immL_M1 src4) %{
12134 match(Set dst (XorL src4 (XorL(LShiftL src2 src3) src1)));
12135 ins_cost(1.9 * INSN_COST);
12136 format %{ "eon $dst, $src1, $src2, LSL $src3" %}
12137
12138 ins_encode %{
12139 __ eon(as_Register($dst$$reg),
12140 as_Register($src1$$reg),
12141 as_Register($src2$$reg),
12142 Assembler::LSL,
12143 $src3$$constant & 0x3f);
12144 %}
12145
12146 ins_pipe(ialu_reg_reg_shift);
12147 %}
12148
12149 // This pattern is automatically generated from aarch64_ad.m4
12150 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12151 // val | (-1 ^ (val >>> shift)) ==> ornw
12152 instruct OrI_reg_URShift_not_reg(iRegINoSp dst,
12153 iRegIorL2I src1, iRegIorL2I src2,
12154 immI src3, immI_M1 src4) %{
12155 match(Set dst (OrI src1 (XorI(URShiftI src2 src3) src4)));
12156 ins_cost(1.9 * INSN_COST);
12157 format %{ "ornw $dst, $src1, $src2, LSR $src3" %}
12158
12159 ins_encode %{
12160 __ ornw(as_Register($dst$$reg),
12161 as_Register($src1$$reg),
12162 as_Register($src2$$reg),
12163 Assembler::LSR,
12164 $src3$$constant & 0x1f);
12165 %}
12166
12167 ins_pipe(ialu_reg_reg_shift);
12168 %}
12169
12170 // This pattern is automatically generated from aarch64_ad.m4
12171 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12172 // val | (-1 ^ (val >>> shift)) ==> orn
12173 instruct OrL_reg_URShift_not_reg(iRegLNoSp dst,
12174 iRegL src1, iRegL src2,
12175 immI src3, immL_M1 src4) %{
12176 match(Set dst (OrL src1 (XorL(URShiftL src2 src3) src4)));
12177 ins_cost(1.9 * INSN_COST);
12178 format %{ "orn $dst, $src1, $src2, LSR $src3" %}
12179
12180 ins_encode %{
12181 __ orn(as_Register($dst$$reg),
12182 as_Register($src1$$reg),
12183 as_Register($src2$$reg),
12184 Assembler::LSR,
12185 $src3$$constant & 0x3f);
12186 %}
12187
12188 ins_pipe(ialu_reg_reg_shift);
12189 %}
12190
12191 // This pattern is automatically generated from aarch64_ad.m4
12192 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12193 // val | (-1 ^ (val >> shift)) ==> ornw
12194 instruct OrI_reg_RShift_not_reg(iRegINoSp dst,
12195 iRegIorL2I src1, iRegIorL2I src2,
12196 immI src3, immI_M1 src4) %{
12197 match(Set dst (OrI src1 (XorI(RShiftI src2 src3) src4)));
12198 ins_cost(1.9 * INSN_COST);
12199 format %{ "ornw $dst, $src1, $src2, ASR $src3" %}
12200
12201 ins_encode %{
12202 __ ornw(as_Register($dst$$reg),
12203 as_Register($src1$$reg),
12204 as_Register($src2$$reg),
12205 Assembler::ASR,
12206 $src3$$constant & 0x1f);
12207 %}
12208
12209 ins_pipe(ialu_reg_reg_shift);
12210 %}
12211
12212 // This pattern is automatically generated from aarch64_ad.m4
12213 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12214 // val | (-1 ^ (val >> shift)) ==> orn
12215 instruct OrL_reg_RShift_not_reg(iRegLNoSp dst,
12216 iRegL src1, iRegL src2,
12217 immI src3, immL_M1 src4) %{
12218 match(Set dst (OrL src1 (XorL(RShiftL src2 src3) src4)));
12219 ins_cost(1.9 * INSN_COST);
12220 format %{ "orn $dst, $src1, $src2, ASR $src3" %}
12221
12222 ins_encode %{
12223 __ orn(as_Register($dst$$reg),
12224 as_Register($src1$$reg),
12225 as_Register($src2$$reg),
12226 Assembler::ASR,
12227 $src3$$constant & 0x3f);
12228 %}
12229
12230 ins_pipe(ialu_reg_reg_shift);
12231 %}
12232
12233 // This pattern is automatically generated from aarch64_ad.m4
12234 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12235 // val | (-1 ^ (val ror shift)) ==> ornw
12236 instruct OrI_reg_RotateRight_not_reg(iRegINoSp dst,
12237 iRegIorL2I src1, iRegIorL2I src2,
12238 immI src3, immI_M1 src4) %{
12239 match(Set dst (OrI src1 (XorI(RotateRight src2 src3) src4)));
12240 ins_cost(1.9 * INSN_COST);
12241 format %{ "ornw $dst, $src1, $src2, ROR $src3" %}
12242
12243 ins_encode %{
12244 __ ornw(as_Register($dst$$reg),
12245 as_Register($src1$$reg),
12246 as_Register($src2$$reg),
12247 Assembler::ROR,
12248 $src3$$constant & 0x1f);
12249 %}
12250
12251 ins_pipe(ialu_reg_reg_shift);
12252 %}
12253
12254 // This pattern is automatically generated from aarch64_ad.m4
12255 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12256 // val | (-1 ^ (val ror shift)) ==> orn
12257 instruct OrL_reg_RotateRight_not_reg(iRegLNoSp dst,
12258 iRegL src1, iRegL src2,
12259 immI src3, immL_M1 src4) %{
12260 match(Set dst (OrL src1 (XorL(RotateRight src2 src3) src4)));
12261 ins_cost(1.9 * INSN_COST);
12262 format %{ "orn $dst, $src1, $src2, ROR $src3" %}
12263
12264 ins_encode %{
12265 __ orn(as_Register($dst$$reg),
12266 as_Register($src1$$reg),
12267 as_Register($src2$$reg),
12268 Assembler::ROR,
12269 $src3$$constant & 0x3f);
12270 %}
12271
12272 ins_pipe(ialu_reg_reg_shift);
12273 %}
12274
12275 // This pattern is automatically generated from aarch64_ad.m4
12276 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12277 // val | (-1 ^ (val << shift)) ==> ornw
12278 instruct OrI_reg_LShift_not_reg(iRegINoSp dst,
12279 iRegIorL2I src1, iRegIorL2I src2,
12280 immI src3, immI_M1 src4) %{
12281 match(Set dst (OrI src1 (XorI(LShiftI src2 src3) src4)));
12282 ins_cost(1.9 * INSN_COST);
12283 format %{ "ornw $dst, $src1, $src2, LSL $src3" %}
12284
12285 ins_encode %{
12286 __ ornw(as_Register($dst$$reg),
12287 as_Register($src1$$reg),
12288 as_Register($src2$$reg),
12289 Assembler::LSL,
12290 $src3$$constant & 0x1f);
12291 %}
12292
12293 ins_pipe(ialu_reg_reg_shift);
12294 %}
12295
12296 // This pattern is automatically generated from aarch64_ad.m4
12297 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12298 // val | (-1 ^ (val << shift)) ==> orn
12299 instruct OrL_reg_LShift_not_reg(iRegLNoSp dst,
12300 iRegL src1, iRegL src2,
12301 immI src3, immL_M1 src4) %{
12302 match(Set dst (OrL src1 (XorL(LShiftL src2 src3) src4)));
12303 ins_cost(1.9 * INSN_COST);
12304 format %{ "orn $dst, $src1, $src2, LSL $src3" %}
12305
12306 ins_encode %{
12307 __ orn(as_Register($dst$$reg),
12308 as_Register($src1$$reg),
12309 as_Register($src2$$reg),
12310 Assembler::LSL,
12311 $src3$$constant & 0x3f);
12312 %}
12313
12314 ins_pipe(ialu_reg_reg_shift);
12315 %}
12316
12317 // This pattern is automatically generated from aarch64_ad.m4
12318 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12319 instruct AndI_reg_URShift_reg(iRegINoSp dst,
12320 iRegIorL2I src1, iRegIorL2I src2,
12321 immI src3) %{
12322 match(Set dst (AndI src1 (URShiftI src2 src3)));
12323
12324 ins_cost(1.9 * INSN_COST);
12325 format %{ "andw $dst, $src1, $src2, LSR $src3" %}
12326
12327 ins_encode %{
12328 __ andw(as_Register($dst$$reg),
12329 as_Register($src1$$reg),
12330 as_Register($src2$$reg),
12331 Assembler::LSR,
12332 $src3$$constant & 0x1f);
12333 %}
12334
12335 ins_pipe(ialu_reg_reg_shift);
12336 %}
12337
12338 // This pattern is automatically generated from aarch64_ad.m4
12339 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12340 instruct AndL_reg_URShift_reg(iRegLNoSp dst,
12341 iRegL src1, iRegL src2,
12342 immI src3) %{
12343 match(Set dst (AndL src1 (URShiftL src2 src3)));
12344
12345 ins_cost(1.9 * INSN_COST);
12346 format %{ "andr $dst, $src1, $src2, LSR $src3" %}
12347
12348 ins_encode %{
12349 __ andr(as_Register($dst$$reg),
12350 as_Register($src1$$reg),
12351 as_Register($src2$$reg),
12352 Assembler::LSR,
12353 $src3$$constant & 0x3f);
12354 %}
12355
12356 ins_pipe(ialu_reg_reg_shift);
12357 %}
12358
12359 // This pattern is automatically generated from aarch64_ad.m4
12360 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12361 instruct AndI_reg_RShift_reg(iRegINoSp dst,
12362 iRegIorL2I src1, iRegIorL2I src2,
12363 immI src3) %{
12364 match(Set dst (AndI src1 (RShiftI src2 src3)));
12365
12366 ins_cost(1.9 * INSN_COST);
12367 format %{ "andw $dst, $src1, $src2, ASR $src3" %}
12368
12369 ins_encode %{
12370 __ andw(as_Register($dst$$reg),
12371 as_Register($src1$$reg),
12372 as_Register($src2$$reg),
12373 Assembler::ASR,
12374 $src3$$constant & 0x1f);
12375 %}
12376
12377 ins_pipe(ialu_reg_reg_shift);
12378 %}
12379
12380 // This pattern is automatically generated from aarch64_ad.m4
12381 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12382 instruct AndL_reg_RShift_reg(iRegLNoSp dst,
12383 iRegL src1, iRegL src2,
12384 immI src3) %{
12385 match(Set dst (AndL src1 (RShiftL src2 src3)));
12386
12387 ins_cost(1.9 * INSN_COST);
12388 format %{ "andr $dst, $src1, $src2, ASR $src3" %}
12389
12390 ins_encode %{
12391 __ andr(as_Register($dst$$reg),
12392 as_Register($src1$$reg),
12393 as_Register($src2$$reg),
12394 Assembler::ASR,
12395 $src3$$constant & 0x3f);
12396 %}
12397
12398 ins_pipe(ialu_reg_reg_shift);
12399 %}
12400
12401 // This pattern is automatically generated from aarch64_ad.m4
12402 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12403 instruct AndI_reg_LShift_reg(iRegINoSp dst,
12404 iRegIorL2I src1, iRegIorL2I src2,
12405 immI src3) %{
12406 match(Set dst (AndI src1 (LShiftI src2 src3)));
12407
12408 ins_cost(1.9 * INSN_COST);
12409 format %{ "andw $dst, $src1, $src2, LSL $src3" %}
12410
12411 ins_encode %{
12412 __ andw(as_Register($dst$$reg),
12413 as_Register($src1$$reg),
12414 as_Register($src2$$reg),
12415 Assembler::LSL,
12416 $src3$$constant & 0x1f);
12417 %}
12418
12419 ins_pipe(ialu_reg_reg_shift);
12420 %}
12421
12422 // This pattern is automatically generated from aarch64_ad.m4
12423 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12424 instruct AndL_reg_LShift_reg(iRegLNoSp dst,
12425 iRegL src1, iRegL src2,
12426 immI src3) %{
12427 match(Set dst (AndL src1 (LShiftL src2 src3)));
12428
12429 ins_cost(1.9 * INSN_COST);
12430 format %{ "andr $dst, $src1, $src2, LSL $src3" %}
12431
12432 ins_encode %{
12433 __ andr(as_Register($dst$$reg),
12434 as_Register($src1$$reg),
12435 as_Register($src2$$reg),
12436 Assembler::LSL,
12437 $src3$$constant & 0x3f);
12438 %}
12439
12440 ins_pipe(ialu_reg_reg_shift);
12441 %}
12442
12443 // This pattern is automatically generated from aarch64_ad.m4
12444 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12445 instruct AndI_reg_RotateRight_reg(iRegINoSp dst,
12446 iRegIorL2I src1, iRegIorL2I src2,
12447 immI src3) %{
12448 match(Set dst (AndI src1 (RotateRight src2 src3)));
12449
12450 ins_cost(1.9 * INSN_COST);
12451 format %{ "andw $dst, $src1, $src2, ROR $src3" %}
12452
12453 ins_encode %{
12454 __ andw(as_Register($dst$$reg),
12455 as_Register($src1$$reg),
12456 as_Register($src2$$reg),
12457 Assembler::ROR,
12458 $src3$$constant & 0x1f);
12459 %}
12460
12461 ins_pipe(ialu_reg_reg_shift);
12462 %}
12463
12464 // This pattern is automatically generated from aarch64_ad.m4
12465 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12466 instruct AndL_reg_RotateRight_reg(iRegLNoSp dst,
12467 iRegL src1, iRegL src2,
12468 immI src3) %{
12469 match(Set dst (AndL src1 (RotateRight src2 src3)));
12470
12471 ins_cost(1.9 * INSN_COST);
12472 format %{ "andr $dst, $src1, $src2, ROR $src3" %}
12473
12474 ins_encode %{
12475 __ andr(as_Register($dst$$reg),
12476 as_Register($src1$$reg),
12477 as_Register($src2$$reg),
12478 Assembler::ROR,
12479 $src3$$constant & 0x3f);
12480 %}
12481
12482 ins_pipe(ialu_reg_reg_shift);
12483 %}
12484
12485 // This pattern is automatically generated from aarch64_ad.m4
12486 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12487 instruct XorI_reg_URShift_reg(iRegINoSp dst,
12488 iRegIorL2I src1, iRegIorL2I src2,
12489 immI src3) %{
12490 match(Set dst (XorI src1 (URShiftI src2 src3)));
12491
12492 ins_cost(1.9 * INSN_COST);
12493 format %{ "eorw $dst, $src1, $src2, LSR $src3" %}
12494
12495 ins_encode %{
12496 __ eorw(as_Register($dst$$reg),
12497 as_Register($src1$$reg),
12498 as_Register($src2$$reg),
12499 Assembler::LSR,
12500 $src3$$constant & 0x1f);
12501 %}
12502
12503 ins_pipe(ialu_reg_reg_shift);
12504 %}
12505
12506 // This pattern is automatically generated from aarch64_ad.m4
12507 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12508 instruct XorL_reg_URShift_reg(iRegLNoSp dst,
12509 iRegL src1, iRegL src2,
12510 immI src3) %{
12511 match(Set dst (XorL src1 (URShiftL src2 src3)));
12512
12513 ins_cost(1.9 * INSN_COST);
12514 format %{ "eor $dst, $src1, $src2, LSR $src3" %}
12515
12516 ins_encode %{
12517 __ eor(as_Register($dst$$reg),
12518 as_Register($src1$$reg),
12519 as_Register($src2$$reg),
12520 Assembler::LSR,
12521 $src3$$constant & 0x3f);
12522 %}
12523
12524 ins_pipe(ialu_reg_reg_shift);
12525 %}
12526
12527 // This pattern is automatically generated from aarch64_ad.m4
12528 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12529 instruct XorI_reg_RShift_reg(iRegINoSp dst,
12530 iRegIorL2I src1, iRegIorL2I src2,
12531 immI src3) %{
12532 match(Set dst (XorI src1 (RShiftI src2 src3)));
12533
12534 ins_cost(1.9 * INSN_COST);
12535 format %{ "eorw $dst, $src1, $src2, ASR $src3" %}
12536
12537 ins_encode %{
12538 __ eorw(as_Register($dst$$reg),
12539 as_Register($src1$$reg),
12540 as_Register($src2$$reg),
12541 Assembler::ASR,
12542 $src3$$constant & 0x1f);
12543 %}
12544
12545 ins_pipe(ialu_reg_reg_shift);
12546 %}
12547
12548 // This pattern is automatically generated from aarch64_ad.m4
12549 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12550 instruct XorL_reg_RShift_reg(iRegLNoSp dst,
12551 iRegL src1, iRegL src2,
12552 immI src3) %{
12553 match(Set dst (XorL src1 (RShiftL src2 src3)));
12554
12555 ins_cost(1.9 * INSN_COST);
12556 format %{ "eor $dst, $src1, $src2, ASR $src3" %}
12557
12558 ins_encode %{
12559 __ eor(as_Register($dst$$reg),
12560 as_Register($src1$$reg),
12561 as_Register($src2$$reg),
12562 Assembler::ASR,
12563 $src3$$constant & 0x3f);
12564 %}
12565
12566 ins_pipe(ialu_reg_reg_shift);
12567 %}
12568
12569 // This pattern is automatically generated from aarch64_ad.m4
12570 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12571 instruct XorI_reg_LShift_reg(iRegINoSp dst,
12572 iRegIorL2I src1, iRegIorL2I src2,
12573 immI src3) %{
12574 match(Set dst (XorI src1 (LShiftI src2 src3)));
12575
12576 ins_cost(1.9 * INSN_COST);
12577 format %{ "eorw $dst, $src1, $src2, LSL $src3" %}
12578
12579 ins_encode %{
12580 __ eorw(as_Register($dst$$reg),
12581 as_Register($src1$$reg),
12582 as_Register($src2$$reg),
12583 Assembler::LSL,
12584 $src3$$constant & 0x1f);
12585 %}
12586
12587 ins_pipe(ialu_reg_reg_shift);
12588 %}
12589
12590 // This pattern is automatically generated from aarch64_ad.m4
12591 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12592 instruct XorL_reg_LShift_reg(iRegLNoSp dst,
12593 iRegL src1, iRegL src2,
12594 immI src3) %{
12595 match(Set dst (XorL src1 (LShiftL src2 src3)));
12596
12597 ins_cost(1.9 * INSN_COST);
12598 format %{ "eor $dst, $src1, $src2, LSL $src3" %}
12599
12600 ins_encode %{
12601 __ eor(as_Register($dst$$reg),
12602 as_Register($src1$$reg),
12603 as_Register($src2$$reg),
12604 Assembler::LSL,
12605 $src3$$constant & 0x3f);
12606 %}
12607
12608 ins_pipe(ialu_reg_reg_shift);
12609 %}
12610
12611 // This pattern is automatically generated from aarch64_ad.m4
12612 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12613 instruct XorI_reg_RotateRight_reg(iRegINoSp dst,
12614 iRegIorL2I src1, iRegIorL2I src2,
12615 immI src3) %{
12616 match(Set dst (XorI src1 (RotateRight src2 src3)));
12617
12618 ins_cost(1.9 * INSN_COST);
12619 format %{ "eorw $dst, $src1, $src2, ROR $src3" %}
12620
12621 ins_encode %{
12622 __ eorw(as_Register($dst$$reg),
12623 as_Register($src1$$reg),
12624 as_Register($src2$$reg),
12625 Assembler::ROR,
12626 $src3$$constant & 0x1f);
12627 %}
12628
12629 ins_pipe(ialu_reg_reg_shift);
12630 %}
12631
12632 // This pattern is automatically generated from aarch64_ad.m4
12633 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12634 instruct XorL_reg_RotateRight_reg(iRegLNoSp dst,
12635 iRegL src1, iRegL src2,
12636 immI src3) %{
12637 match(Set dst (XorL src1 (RotateRight src2 src3)));
12638
12639 ins_cost(1.9 * INSN_COST);
12640 format %{ "eor $dst, $src1, $src2, ROR $src3" %}
12641
12642 ins_encode %{
12643 __ eor(as_Register($dst$$reg),
12644 as_Register($src1$$reg),
12645 as_Register($src2$$reg),
12646 Assembler::ROR,
12647 $src3$$constant & 0x3f);
12648 %}
12649
12650 ins_pipe(ialu_reg_reg_shift);
12651 %}
12652
12653 // This pattern is automatically generated from aarch64_ad.m4
12654 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12655 instruct OrI_reg_URShift_reg(iRegINoSp dst,
12656 iRegIorL2I src1, iRegIorL2I src2,
12657 immI src3) %{
12658 match(Set dst (OrI src1 (URShiftI src2 src3)));
12659
12660 ins_cost(1.9 * INSN_COST);
12661 format %{ "orrw $dst, $src1, $src2, LSR $src3" %}
12662
12663 ins_encode %{
12664 __ orrw(as_Register($dst$$reg),
12665 as_Register($src1$$reg),
12666 as_Register($src2$$reg),
12667 Assembler::LSR,
12668 $src3$$constant & 0x1f);
12669 %}
12670
12671 ins_pipe(ialu_reg_reg_shift);
12672 %}
12673
12674 // This pattern is automatically generated from aarch64_ad.m4
12675 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12676 instruct OrL_reg_URShift_reg(iRegLNoSp dst,
12677 iRegL src1, iRegL src2,
12678 immI src3) %{
12679 match(Set dst (OrL src1 (URShiftL src2 src3)));
12680
12681 ins_cost(1.9 * INSN_COST);
12682 format %{ "orr $dst, $src1, $src2, LSR $src3" %}
12683
12684 ins_encode %{
12685 __ orr(as_Register($dst$$reg),
12686 as_Register($src1$$reg),
12687 as_Register($src2$$reg),
12688 Assembler::LSR,
12689 $src3$$constant & 0x3f);
12690 %}
12691
12692 ins_pipe(ialu_reg_reg_shift);
12693 %}
12694
12695 // This pattern is automatically generated from aarch64_ad.m4
12696 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12697 instruct OrI_reg_RShift_reg(iRegINoSp dst,
12698 iRegIorL2I src1, iRegIorL2I src2,
12699 immI src3) %{
12700 match(Set dst (OrI src1 (RShiftI src2 src3)));
12701
12702 ins_cost(1.9 * INSN_COST);
12703 format %{ "orrw $dst, $src1, $src2, ASR $src3" %}
12704
12705 ins_encode %{
12706 __ orrw(as_Register($dst$$reg),
12707 as_Register($src1$$reg),
12708 as_Register($src2$$reg),
12709 Assembler::ASR,
12710 $src3$$constant & 0x1f);
12711 %}
12712
12713 ins_pipe(ialu_reg_reg_shift);
12714 %}
12715
12716 // This pattern is automatically generated from aarch64_ad.m4
12717 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12718 instruct OrL_reg_RShift_reg(iRegLNoSp dst,
12719 iRegL src1, iRegL src2,
12720 immI src3) %{
12721 match(Set dst (OrL src1 (RShiftL src2 src3)));
12722
12723 ins_cost(1.9 * INSN_COST);
12724 format %{ "orr $dst, $src1, $src2, ASR $src3" %}
12725
12726 ins_encode %{
12727 __ orr(as_Register($dst$$reg),
12728 as_Register($src1$$reg),
12729 as_Register($src2$$reg),
12730 Assembler::ASR,
12731 $src3$$constant & 0x3f);
12732 %}
12733
12734 ins_pipe(ialu_reg_reg_shift);
12735 %}
12736
12737 // This pattern is automatically generated from aarch64_ad.m4
12738 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12739 instruct OrI_reg_LShift_reg(iRegINoSp dst,
12740 iRegIorL2I src1, iRegIorL2I src2,
12741 immI src3) %{
12742 match(Set dst (OrI src1 (LShiftI src2 src3)));
12743
12744 ins_cost(1.9 * INSN_COST);
12745 format %{ "orrw $dst, $src1, $src2, LSL $src3" %}
12746
12747 ins_encode %{
12748 __ orrw(as_Register($dst$$reg),
12749 as_Register($src1$$reg),
12750 as_Register($src2$$reg),
12751 Assembler::LSL,
12752 $src3$$constant & 0x1f);
12753 %}
12754
12755 ins_pipe(ialu_reg_reg_shift);
12756 %}
12757
12758 // This pattern is automatically generated from aarch64_ad.m4
12759 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12760 instruct OrL_reg_LShift_reg(iRegLNoSp dst,
12761 iRegL src1, iRegL src2,
12762 immI src3) %{
12763 match(Set dst (OrL src1 (LShiftL src2 src3)));
12764
12765 ins_cost(1.9 * INSN_COST);
12766 format %{ "orr $dst, $src1, $src2, LSL $src3" %}
12767
12768 ins_encode %{
12769 __ orr(as_Register($dst$$reg),
12770 as_Register($src1$$reg),
12771 as_Register($src2$$reg),
12772 Assembler::LSL,
12773 $src3$$constant & 0x3f);
12774 %}
12775
12776 ins_pipe(ialu_reg_reg_shift);
12777 %}
12778
12779 // This pattern is automatically generated from aarch64_ad.m4
12780 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12781 instruct OrI_reg_RotateRight_reg(iRegINoSp dst,
12782 iRegIorL2I src1, iRegIorL2I src2,
12783 immI src3) %{
12784 match(Set dst (OrI src1 (RotateRight src2 src3)));
12785
12786 ins_cost(1.9 * INSN_COST);
12787 format %{ "orrw $dst, $src1, $src2, ROR $src3" %}
12788
12789 ins_encode %{
12790 __ orrw(as_Register($dst$$reg),
12791 as_Register($src1$$reg),
12792 as_Register($src2$$reg),
12793 Assembler::ROR,
12794 $src3$$constant & 0x1f);
12795 %}
12796
12797 ins_pipe(ialu_reg_reg_shift);
12798 %}
12799
12800 // This pattern is automatically generated from aarch64_ad.m4
12801 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12802 instruct OrL_reg_RotateRight_reg(iRegLNoSp dst,
12803 iRegL src1, iRegL src2,
12804 immI src3) %{
12805 match(Set dst (OrL src1 (RotateRight src2 src3)));
12806
12807 ins_cost(1.9 * INSN_COST);
12808 format %{ "orr $dst, $src1, $src2, ROR $src3" %}
12809
12810 ins_encode %{
12811 __ orr(as_Register($dst$$reg),
12812 as_Register($src1$$reg),
12813 as_Register($src2$$reg),
12814 Assembler::ROR,
12815 $src3$$constant & 0x3f);
12816 %}
12817
12818 ins_pipe(ialu_reg_reg_shift);
12819 %}
12820
12821 // This pattern is automatically generated from aarch64_ad.m4
12822 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12823 instruct AddI_reg_URShift_reg(iRegINoSp dst,
12824 iRegIorL2I src1, iRegIorL2I src2,
12825 immI src3) %{
12826 match(Set dst (AddI src1 (URShiftI src2 src3)));
12827
12828 ins_cost(1.9 * INSN_COST);
12829 format %{ "addw $dst, $src1, $src2, LSR $src3" %}
12830
12831 ins_encode %{
12832 __ addw(as_Register($dst$$reg),
12833 as_Register($src1$$reg),
12834 as_Register($src2$$reg),
12835 Assembler::LSR,
12836 $src3$$constant & 0x1f);
12837 %}
12838
12839 ins_pipe(ialu_reg_reg_shift);
12840 %}
12841
12842 // This pattern is automatically generated from aarch64_ad.m4
12843 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12844 instruct AddL_reg_URShift_reg(iRegLNoSp dst,
12845 iRegL src1, iRegL src2,
12846 immI src3) %{
12847 match(Set dst (AddL src1 (URShiftL src2 src3)));
12848
12849 ins_cost(1.9 * INSN_COST);
12850 format %{ "add $dst, $src1, $src2, LSR $src3" %}
12851
12852 ins_encode %{
12853 __ add(as_Register($dst$$reg),
12854 as_Register($src1$$reg),
12855 as_Register($src2$$reg),
12856 Assembler::LSR,
12857 $src3$$constant & 0x3f);
12858 %}
12859
12860 ins_pipe(ialu_reg_reg_shift);
12861 %}
12862
12863 // This pattern is automatically generated from aarch64_ad.m4
12864 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12865 instruct AddI_reg_RShift_reg(iRegINoSp dst,
12866 iRegIorL2I src1, iRegIorL2I src2,
12867 immI src3) %{
12868 match(Set dst (AddI src1 (RShiftI src2 src3)));
12869
12870 ins_cost(1.9 * INSN_COST);
12871 format %{ "addw $dst, $src1, $src2, ASR $src3" %}
12872
12873 ins_encode %{
12874 __ addw(as_Register($dst$$reg),
12875 as_Register($src1$$reg),
12876 as_Register($src2$$reg),
12877 Assembler::ASR,
12878 $src3$$constant & 0x1f);
12879 %}
12880
12881 ins_pipe(ialu_reg_reg_shift);
12882 %}
12883
12884 // This pattern is automatically generated from aarch64_ad.m4
12885 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12886 instruct AddL_reg_RShift_reg(iRegLNoSp dst,
12887 iRegL src1, iRegL src2,
12888 immI src3) %{
12889 match(Set dst (AddL src1 (RShiftL src2 src3)));
12890
12891 ins_cost(1.9 * INSN_COST);
12892 format %{ "add $dst, $src1, $src2, ASR $src3" %}
12893
12894 ins_encode %{
12895 __ add(as_Register($dst$$reg),
12896 as_Register($src1$$reg),
12897 as_Register($src2$$reg),
12898 Assembler::ASR,
12899 $src3$$constant & 0x3f);
12900 %}
12901
12902 ins_pipe(ialu_reg_reg_shift);
12903 %}
12904
12905 // This pattern is automatically generated from aarch64_ad.m4
12906 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12907 instruct AddI_reg_LShift_reg(iRegINoSp dst,
12908 iRegIorL2I src1, iRegIorL2I src2,
12909 immI src3) %{
12910 match(Set dst (AddI src1 (LShiftI src2 src3)));
12911
12912 ins_cost(1.9 * INSN_COST);
12913 format %{ "addw $dst, $src1, $src2, LSL $src3" %}
12914
12915 ins_encode %{
12916 __ addw(as_Register($dst$$reg),
12917 as_Register($src1$$reg),
12918 as_Register($src2$$reg),
12919 Assembler::LSL,
12920 $src3$$constant & 0x1f);
12921 %}
12922
12923 ins_pipe(ialu_reg_reg_shift);
12924 %}
12925
12926 // This pattern is automatically generated from aarch64_ad.m4
12927 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12928 instruct AddL_reg_LShift_reg(iRegLNoSp dst,
12929 iRegL src1, iRegL src2,
12930 immI src3) %{
12931 match(Set dst (AddL src1 (LShiftL src2 src3)));
12932
12933 ins_cost(1.9 * INSN_COST);
12934 format %{ "add $dst, $src1, $src2, LSL $src3" %}
12935
12936 ins_encode %{
12937 __ add(as_Register($dst$$reg),
12938 as_Register($src1$$reg),
12939 as_Register($src2$$reg),
12940 Assembler::LSL,
12941 $src3$$constant & 0x3f);
12942 %}
12943
12944 ins_pipe(ialu_reg_reg_shift);
12945 %}
12946
12947 // This pattern is automatically generated from aarch64_ad.m4
12948 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12949 instruct SubI_reg_URShift_reg(iRegINoSp dst,
12950 iRegIorL2I src1, iRegIorL2I src2,
12951 immI src3) %{
12952 match(Set dst (SubI src1 (URShiftI src2 src3)));
12953
12954 ins_cost(1.9 * INSN_COST);
12955 format %{ "subw $dst, $src1, $src2, LSR $src3" %}
12956
12957 ins_encode %{
12958 __ subw(as_Register($dst$$reg),
12959 as_Register($src1$$reg),
12960 as_Register($src2$$reg),
12961 Assembler::LSR,
12962 $src3$$constant & 0x1f);
12963 %}
12964
12965 ins_pipe(ialu_reg_reg_shift);
12966 %}
12967
12968 // This pattern is automatically generated from aarch64_ad.m4
12969 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12970 instruct SubL_reg_URShift_reg(iRegLNoSp dst,
12971 iRegL src1, iRegL src2,
12972 immI src3) %{
12973 match(Set dst (SubL src1 (URShiftL src2 src3)));
12974
12975 ins_cost(1.9 * INSN_COST);
12976 format %{ "sub $dst, $src1, $src2, LSR $src3" %}
12977
12978 ins_encode %{
12979 __ sub(as_Register($dst$$reg),
12980 as_Register($src1$$reg),
12981 as_Register($src2$$reg),
12982 Assembler::LSR,
12983 $src3$$constant & 0x3f);
12984 %}
12985
12986 ins_pipe(ialu_reg_reg_shift);
12987 %}
12988
12989 // This pattern is automatically generated from aarch64_ad.m4
12990 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12991 instruct SubI_reg_RShift_reg(iRegINoSp dst,
12992 iRegIorL2I src1, iRegIorL2I src2,
12993 immI src3) %{
12994 match(Set dst (SubI src1 (RShiftI src2 src3)));
12995
12996 ins_cost(1.9 * INSN_COST);
12997 format %{ "subw $dst, $src1, $src2, ASR $src3" %}
12998
12999 ins_encode %{
13000 __ subw(as_Register($dst$$reg),
13001 as_Register($src1$$reg),
13002 as_Register($src2$$reg),
13003 Assembler::ASR,
13004 $src3$$constant & 0x1f);
13005 %}
13006
13007 ins_pipe(ialu_reg_reg_shift);
13008 %}
13009
13010 // This pattern is automatically generated from aarch64_ad.m4
13011 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13012 instruct SubL_reg_RShift_reg(iRegLNoSp dst,
13013 iRegL src1, iRegL src2,
13014 immI src3) %{
13015 match(Set dst (SubL src1 (RShiftL src2 src3)));
13016
13017 ins_cost(1.9 * INSN_COST);
13018 format %{ "sub $dst, $src1, $src2, ASR $src3" %}
13019
13020 ins_encode %{
13021 __ sub(as_Register($dst$$reg),
13022 as_Register($src1$$reg),
13023 as_Register($src2$$reg),
13024 Assembler::ASR,
13025 $src3$$constant & 0x3f);
13026 %}
13027
13028 ins_pipe(ialu_reg_reg_shift);
13029 %}
13030
13031 // This pattern is automatically generated from aarch64_ad.m4
13032 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13033 instruct SubI_reg_LShift_reg(iRegINoSp dst,
13034 iRegIorL2I src1, iRegIorL2I src2,
13035 immI src3) %{
13036 match(Set dst (SubI src1 (LShiftI src2 src3)));
13037
13038 ins_cost(1.9 * INSN_COST);
13039 format %{ "subw $dst, $src1, $src2, LSL $src3" %}
13040
13041 ins_encode %{
13042 __ subw(as_Register($dst$$reg),
13043 as_Register($src1$$reg),
13044 as_Register($src2$$reg),
13045 Assembler::LSL,
13046 $src3$$constant & 0x1f);
13047 %}
13048
13049 ins_pipe(ialu_reg_reg_shift);
13050 %}
13051
13052 // This pattern is automatically generated from aarch64_ad.m4
13053 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13054 instruct SubL_reg_LShift_reg(iRegLNoSp dst,
13055 iRegL src1, iRegL src2,
13056 immI src3) %{
13057 match(Set dst (SubL src1 (LShiftL src2 src3)));
13058
13059 ins_cost(1.9 * INSN_COST);
13060 format %{ "sub $dst, $src1, $src2, LSL $src3" %}
13061
13062 ins_encode %{
13063 __ sub(as_Register($dst$$reg),
13064 as_Register($src1$$reg),
13065 as_Register($src2$$reg),
13066 Assembler::LSL,
13067 $src3$$constant & 0x3f);
13068 %}
13069
13070 ins_pipe(ialu_reg_reg_shift);
13071 %}
13072
13073 // This pattern is automatically generated from aarch64_ad.m4
13074 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13075
13076 // Shift Left followed by Shift Right.
13077 // This idiom is used by the compiler for the i2b bytecode etc.
13078 instruct sbfmL(iRegLNoSp dst, iRegL src, immI lshift_count, immI rshift_count)
13079 %{
13080 match(Set dst (RShiftL (LShiftL src lshift_count) rshift_count));
13081 ins_cost(INSN_COST * 2);
13082 format %{ "sbfm $dst, $src, $rshift_count - $lshift_count, #63 - $lshift_count" %}
13083 ins_encode %{
13084 int lshift = $lshift_count$$constant & 63;
13085 int rshift = $rshift_count$$constant & 63;
13086 int s = 63 - lshift;
13087 int r = (rshift - lshift) & 63;
13088 __ sbfm(as_Register($dst$$reg),
13089 as_Register($src$$reg),
13090 r, s);
13091 %}
13092
13093 ins_pipe(ialu_reg_shift);
13094 %}
13095
13096 // This pattern is automatically generated from aarch64_ad.m4
13097 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13098
13099 // Shift Left followed by Shift Right.
13100 // This idiom is used by the compiler for the i2b bytecode etc.
13101 instruct sbfmwI(iRegINoSp dst, iRegIorL2I src, immI lshift_count, immI rshift_count)
13102 %{
13103 match(Set dst (RShiftI (LShiftI src lshift_count) rshift_count));
13104 ins_cost(INSN_COST * 2);
13105 format %{ "sbfmw $dst, $src, $rshift_count - $lshift_count, #31 - $lshift_count" %}
13106 ins_encode %{
13107 int lshift = $lshift_count$$constant & 31;
13108 int rshift = $rshift_count$$constant & 31;
13109 int s = 31 - lshift;
13110 int r = (rshift - lshift) & 31;
13111 __ sbfmw(as_Register($dst$$reg),
13112 as_Register($src$$reg),
13113 r, s);
13114 %}
13115
13116 ins_pipe(ialu_reg_shift);
13117 %}
13118
13119 // This pattern is automatically generated from aarch64_ad.m4
13120 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13121
13122 // Shift Left followed by Shift Right.
13123 // This idiom is used by the compiler for the i2b bytecode etc.
13124 instruct ubfmL(iRegLNoSp dst, iRegL src, immI lshift_count, immI rshift_count)
13125 %{
13126 match(Set dst (URShiftL (LShiftL src lshift_count) rshift_count));
13127 ins_cost(INSN_COST * 2);
13128 format %{ "ubfm $dst, $src, $rshift_count - $lshift_count, #63 - $lshift_count" %}
13129 ins_encode %{
13130 int lshift = $lshift_count$$constant & 63;
13131 int rshift = $rshift_count$$constant & 63;
13132 int s = 63 - lshift;
13133 int r = (rshift - lshift) & 63;
13134 __ ubfm(as_Register($dst$$reg),
13135 as_Register($src$$reg),
13136 r, s);
13137 %}
13138
13139 ins_pipe(ialu_reg_shift);
13140 %}
13141
13142 // This pattern is automatically generated from aarch64_ad.m4
13143 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13144
13145 // Shift Left followed by Shift Right.
13146 // This idiom is used by the compiler for the i2b bytecode etc.
13147 instruct ubfmwI(iRegINoSp dst, iRegIorL2I src, immI lshift_count, immI rshift_count)
13148 %{
13149 match(Set dst (URShiftI (LShiftI src lshift_count) rshift_count));
13150 ins_cost(INSN_COST * 2);
13151 format %{ "ubfmw $dst, $src, $rshift_count - $lshift_count, #31 - $lshift_count" %}
13152 ins_encode %{
13153 int lshift = $lshift_count$$constant & 31;
13154 int rshift = $rshift_count$$constant & 31;
13155 int s = 31 - lshift;
13156 int r = (rshift - lshift) & 31;
13157 __ ubfmw(as_Register($dst$$reg),
13158 as_Register($src$$reg),
13159 r, s);
13160 %}
13161
13162 ins_pipe(ialu_reg_shift);
13163 %}
13164
13165 // Bitfield extract with shift & mask
13166
13167 // This pattern is automatically generated from aarch64_ad.m4
13168 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13169 instruct ubfxwI(iRegINoSp dst, iRegIorL2I src, immI rshift, immI_bitmask mask)
13170 %{
13171 match(Set dst (AndI (URShiftI src rshift) mask));
13172 // Make sure we are not going to exceed what ubfxw can do.
13173 predicate((exact_log2(n->in(2)->get_int() + 1) + (n->in(1)->in(2)->get_int() & 31)) <= (31 + 1));
13174
13175 ins_cost(INSN_COST);
13176 format %{ "ubfxw $dst, $src, $rshift, $mask" %}
13177 ins_encode %{
13178 int rshift = $rshift$$constant & 31;
13179 intptr_t mask = $mask$$constant;
13180 int width = exact_log2(mask+1);
13181 __ ubfxw(as_Register($dst$$reg),
13182 as_Register($src$$reg), rshift, width);
13183 %}
13184 ins_pipe(ialu_reg_shift);
13185 %}
13186
13187 // This pattern is automatically generated from aarch64_ad.m4
13188 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13189 instruct ubfxL(iRegLNoSp dst, iRegL src, immI rshift, immL_bitmask mask)
13190 %{
13191 match(Set dst (AndL (URShiftL src rshift) mask));
13192 // Make sure we are not going to exceed what ubfx can do.
13193 predicate((exact_log2_long(n->in(2)->get_long() + 1) + (n->in(1)->in(2)->get_int() & 63)) <= (63 + 1));
13194
13195 ins_cost(INSN_COST);
13196 format %{ "ubfx $dst, $src, $rshift, $mask" %}
13197 ins_encode %{
13198 int rshift = $rshift$$constant & 63;
13199 intptr_t mask = $mask$$constant;
13200 int width = exact_log2_long(mask+1);
13201 __ ubfx(as_Register($dst$$reg),
13202 as_Register($src$$reg), rshift, width);
13203 %}
13204 ins_pipe(ialu_reg_shift);
13205 %}
13206
13207
13208 // This pattern is automatically generated from aarch64_ad.m4
13209 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13210
13211 // We can use ubfx when extending an And with a mask when we know mask
13212 // is positive. We know that because immI_bitmask guarantees it.
13213 instruct ubfxIConvI2L(iRegLNoSp dst, iRegIorL2I src, immI rshift, immI_bitmask mask)
13214 %{
13215 match(Set dst (ConvI2L (AndI (URShiftI src rshift) mask)));
13216 // Make sure we are not going to exceed what ubfxw can do.
13217 predicate((exact_log2(n->in(1)->in(2)->get_int() + 1) + (n->in(1)->in(1)->in(2)->get_int() & 31)) <= (31 + 1));
13218
13219 ins_cost(INSN_COST * 2);
13220 format %{ "ubfx $dst, $src, $rshift, $mask" %}
13221 ins_encode %{
13222 int rshift = $rshift$$constant & 31;
13223 intptr_t mask = $mask$$constant;
13224 int width = exact_log2(mask+1);
13225 __ ubfx(as_Register($dst$$reg),
13226 as_Register($src$$reg), rshift, width);
13227 %}
13228 ins_pipe(ialu_reg_shift);
13229 %}
13230
13231
13232 // This pattern is automatically generated from aarch64_ad.m4
13233 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13234
13235 // We can use ubfiz when masking by a positive number and then left shifting the result.
13236 // We know that the mask is positive because immI_bitmask guarantees it.
13237 instruct ubfizwI(iRegINoSp dst, iRegIorL2I src, immI lshift, immI_bitmask mask)
13238 %{
13239 match(Set dst (LShiftI (AndI src mask) lshift));
13240 predicate((exact_log2(n->in(1)->in(2)->get_int() + 1) + (n->in(2)->get_int() & 31)) <= (31 + 1));
13241
13242 ins_cost(INSN_COST);
13243 format %{ "ubfizw $dst, $src, $lshift, $mask" %}
13244 ins_encode %{
13245 int lshift = $lshift$$constant & 31;
13246 intptr_t mask = $mask$$constant;
13247 int width = exact_log2(mask+1);
13248 __ ubfizw(as_Register($dst$$reg),
13249 as_Register($src$$reg), lshift, width);
13250 %}
13251 ins_pipe(ialu_reg_shift);
13252 %}
13253
13254 // This pattern is automatically generated from aarch64_ad.m4
13255 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13256
13257 // We can use ubfiz when masking by a positive number and then left shifting the result.
13258 // We know that the mask is positive because immL_bitmask guarantees it.
13259 instruct ubfizL(iRegLNoSp dst, iRegL src, immI lshift, immL_bitmask mask)
13260 %{
13261 match(Set dst (LShiftL (AndL src mask) lshift));
13262 predicate((exact_log2_long(n->in(1)->in(2)->get_long() + 1) + (n->in(2)->get_int() & 63)) <= (63 + 1));
13263
13264 ins_cost(INSN_COST);
13265 format %{ "ubfiz $dst, $src, $lshift, $mask" %}
13266 ins_encode %{
13267 int lshift = $lshift$$constant & 63;
13268 intptr_t mask = $mask$$constant;
13269 int width = exact_log2_long(mask+1);
13270 __ ubfiz(as_Register($dst$$reg),
13271 as_Register($src$$reg), lshift, width);
13272 %}
13273 ins_pipe(ialu_reg_shift);
13274 %}
13275
13276 // This pattern is automatically generated from aarch64_ad.m4
13277 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13278
13279 // We can use ubfiz when masking by a positive number and then left shifting the result.
13280 // We know that the mask is positive because immI_bitmask guarantees it.
13281 instruct ubfizwIConvI2L(iRegLNoSp dst, iRegIorL2I src, immI lshift, immI_bitmask mask)
13282 %{
13283 match(Set dst (ConvI2L (LShiftI (AndI src mask) lshift)));
13284 predicate((exact_log2(n->in(1)->in(1)->in(2)->get_int() + 1) + (n->in(1)->in(2)->get_int() & 31)) <= 31);
13285
13286 ins_cost(INSN_COST);
13287 format %{ "ubfizw $dst, $src, $lshift, $mask" %}
13288 ins_encode %{
13289 int lshift = $lshift$$constant & 31;
13290 intptr_t mask = $mask$$constant;
13291 int width = exact_log2(mask+1);
13292 __ ubfizw(as_Register($dst$$reg),
13293 as_Register($src$$reg), lshift, width);
13294 %}
13295 ins_pipe(ialu_reg_shift);
13296 %}
13297
13298 // This pattern is automatically generated from aarch64_ad.m4
13299 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13300
13301 // We can use ubfiz when masking by a positive number and then left shifting the result.
13302 // We know that the mask is positive because immL_bitmask guarantees it.
13303 instruct ubfizLConvL2I(iRegINoSp dst, iRegL src, immI lshift, immL_positive_bitmaskI mask)
13304 %{
13305 match(Set dst (ConvL2I (LShiftL (AndL src mask) lshift)));
13306 predicate((exact_log2_long(n->in(1)->in(1)->in(2)->get_long() + 1) + (n->in(1)->in(2)->get_int() & 63)) <= 31);
13307
13308 ins_cost(INSN_COST);
13309 format %{ "ubfiz $dst, $src, $lshift, $mask" %}
13310 ins_encode %{
13311 int lshift = $lshift$$constant & 63;
13312 intptr_t mask = $mask$$constant;
13313 int width = exact_log2_long(mask+1);
13314 __ ubfiz(as_Register($dst$$reg),
13315 as_Register($src$$reg), lshift, width);
13316 %}
13317 ins_pipe(ialu_reg_shift);
13318 %}
13319
13320
13321 // This pattern is automatically generated from aarch64_ad.m4
13322 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13323
13324 // If there is a convert I to L block between and AndI and a LShiftL, we can also match ubfiz
13325 instruct ubfizIConvI2L(iRegLNoSp dst, iRegIorL2I src, immI lshift, immI_bitmask mask)
13326 %{
13327 match(Set dst (LShiftL (ConvI2L (AndI src mask)) lshift));
13328 predicate((exact_log2(n->in(1)->in(1)->in(2)->get_int() + 1) + (n->in(2)->get_int() & 63)) <= (63 + 1));
13329
13330 ins_cost(INSN_COST);
13331 format %{ "ubfiz $dst, $src, $lshift, $mask" %}
13332 ins_encode %{
13333 int lshift = $lshift$$constant & 63;
13334 intptr_t mask = $mask$$constant;
13335 int width = exact_log2(mask+1);
13336 __ ubfiz(as_Register($dst$$reg),
13337 as_Register($src$$reg), lshift, width);
13338 %}
13339 ins_pipe(ialu_reg_shift);
13340 %}
13341
13342 // This pattern is automatically generated from aarch64_ad.m4
13343 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13344
13345 // If there is a convert L to I block between and AndL and a LShiftI, we can also match ubfiz
13346 instruct ubfizLConvL2Ix(iRegINoSp dst, iRegL src, immI lshift, immL_positive_bitmaskI mask)
13347 %{
13348 match(Set dst (LShiftI (ConvL2I (AndL src mask)) lshift));
13349 predicate((exact_log2_long(n->in(1)->in(1)->in(2)->get_long() + 1) + (n->in(2)->get_int() & 31)) <= 31);
13350
13351 ins_cost(INSN_COST);
13352 format %{ "ubfiz $dst, $src, $lshift, $mask" %}
13353 ins_encode %{
13354 int lshift = $lshift$$constant & 31;
13355 intptr_t mask = $mask$$constant;
13356 int width = exact_log2(mask+1);
13357 __ ubfiz(as_Register($dst$$reg),
13358 as_Register($src$$reg), lshift, width);
13359 %}
13360 ins_pipe(ialu_reg_shift);
13361 %}
13362
13363 // This pattern is automatically generated from aarch64_ad.m4
13364 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13365
13366 // Can skip int2long conversions after AND with small bitmask
13367 instruct ubfizIConvI2LAndI(iRegLNoSp dst, iRegI src, immI_bitmask msk)
13368 %{
13369 match(Set dst (ConvI2L (AndI src msk)));
13370 ins_cost(INSN_COST);
13371 format %{ "ubfiz $dst, $src, 0, exact_log2($msk + 1) " %}
13372 ins_encode %{
13373 __ ubfiz(as_Register($dst$$reg), as_Register($src$$reg), 0, exact_log2($msk$$constant + 1));
13374 %}
13375 ins_pipe(ialu_reg_shift);
13376 %}
13377
13378
13379 // Rotations
13380 // This pattern is automatically generated from aarch64_ad.m4
13381 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13382 instruct extrOrL(iRegLNoSp dst, iRegL src1, iRegL src2, immI lshift, immI rshift, rFlagsReg cr)
13383 %{
13384 match(Set dst (OrL (LShiftL src1 lshift) (URShiftL src2 rshift)));
13385 predicate(0 == (((n->in(1)->in(2)->get_int() & 63) + (n->in(2)->in(2)->get_int() & 63)) & 63));
13386
13387 ins_cost(INSN_COST);
13388 format %{ "extr $dst, $src1, $src2, #$rshift" %}
13389
13390 ins_encode %{
13391 __ extr(as_Register($dst$$reg), as_Register($src1$$reg), as_Register($src2$$reg),
13392 $rshift$$constant & 63);
13393 %}
13394 ins_pipe(ialu_reg_reg_extr);
13395 %}
13396
13397
13398 // This pattern is automatically generated from aarch64_ad.m4
13399 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13400 instruct extrOrI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI lshift, immI rshift, rFlagsReg cr)
13401 %{
13402 match(Set dst (OrI (LShiftI src1 lshift) (URShiftI src2 rshift)));
13403 predicate(0 == (((n->in(1)->in(2)->get_int() & 31) + (n->in(2)->in(2)->get_int() & 31)) & 31));
13404
13405 ins_cost(INSN_COST);
13406 format %{ "extr $dst, $src1, $src2, #$rshift" %}
13407
13408 ins_encode %{
13409 __ extrw(as_Register($dst$$reg), as_Register($src1$$reg), as_Register($src2$$reg),
13410 $rshift$$constant & 31);
13411 %}
13412 ins_pipe(ialu_reg_reg_extr);
13413 %}
13414
13415
13416 // This pattern is automatically generated from aarch64_ad.m4
13417 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13418 instruct extrAddL(iRegLNoSp dst, iRegL src1, iRegL src2, immI lshift, immI rshift, rFlagsReg cr)
13419 %{
13420 match(Set dst (AddL (LShiftL src1 lshift) (URShiftL src2 rshift)));
13421 predicate(0 == (((n->in(1)->in(2)->get_int() & 63) + (n->in(2)->in(2)->get_int() & 63)) & 63));
13422
13423 ins_cost(INSN_COST);
13424 format %{ "extr $dst, $src1, $src2, #$rshift" %}
13425
13426 ins_encode %{
13427 __ extr(as_Register($dst$$reg), as_Register($src1$$reg), as_Register($src2$$reg),
13428 $rshift$$constant & 63);
13429 %}
13430 ins_pipe(ialu_reg_reg_extr);
13431 %}
13432
13433
13434 // This pattern is automatically generated from aarch64_ad.m4
13435 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13436 instruct extrAddI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI lshift, immI rshift, rFlagsReg cr)
13437 %{
13438 match(Set dst (AddI (LShiftI src1 lshift) (URShiftI src2 rshift)));
13439 predicate(0 == (((n->in(1)->in(2)->get_int() & 31) + (n->in(2)->in(2)->get_int() & 31)) & 31));
13440
13441 ins_cost(INSN_COST);
13442 format %{ "extr $dst, $src1, $src2, #$rshift" %}
13443
13444 ins_encode %{
13445 __ extrw(as_Register($dst$$reg), as_Register($src1$$reg), as_Register($src2$$reg),
13446 $rshift$$constant & 31);
13447 %}
13448 ins_pipe(ialu_reg_reg_extr);
13449 %}
13450
13451
13452 // This pattern is automatically generated from aarch64_ad.m4
13453 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13454 instruct rorI_imm(iRegINoSp dst, iRegI src, immI shift)
13455 %{
13456 match(Set dst (RotateRight src shift));
13457
13458 ins_cost(INSN_COST);
13459 format %{ "ror $dst, $src, $shift" %}
13460
13461 ins_encode %{
13462 __ extrw(as_Register($dst$$reg), as_Register($src$$reg), as_Register($src$$reg),
13463 $shift$$constant & 0x1f);
13464 %}
13465 ins_pipe(ialu_reg_reg_vshift);
13466 %}
13467
13468 // This pattern is automatically generated from aarch64_ad.m4
13469 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13470 instruct rorL_imm(iRegLNoSp dst, iRegL src, immI shift)
13471 %{
13472 match(Set dst (RotateRight src shift));
13473
13474 ins_cost(INSN_COST);
13475 format %{ "ror $dst, $src, $shift" %}
13476
13477 ins_encode %{
13478 __ extr(as_Register($dst$$reg), as_Register($src$$reg), as_Register($src$$reg),
13479 $shift$$constant & 0x3f);
13480 %}
13481 ins_pipe(ialu_reg_reg_vshift);
13482 %}
13483
13484 // This pattern is automatically generated from aarch64_ad.m4
13485 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13486 instruct rorI_reg(iRegINoSp dst, iRegI src, iRegI shift)
13487 %{
13488 match(Set dst (RotateRight src shift));
13489
13490 ins_cost(INSN_COST);
13491 format %{ "ror $dst, $src, $shift" %}
13492
13493 ins_encode %{
13494 __ rorvw(as_Register($dst$$reg), as_Register($src$$reg), as_Register($shift$$reg));
13495 %}
13496 ins_pipe(ialu_reg_reg_vshift);
13497 %}
13498
13499 // This pattern is automatically generated from aarch64_ad.m4
13500 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13501 instruct rorL_reg(iRegLNoSp dst, iRegL src, iRegI shift)
13502 %{
13503 match(Set dst (RotateRight src shift));
13504
13505 ins_cost(INSN_COST);
13506 format %{ "ror $dst, $src, $shift" %}
13507
13508 ins_encode %{
13509 __ rorv(as_Register($dst$$reg), as_Register($src$$reg), as_Register($shift$$reg));
13510 %}
13511 ins_pipe(ialu_reg_reg_vshift);
13512 %}
13513
13514 // This pattern is automatically generated from aarch64_ad.m4
13515 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13516 instruct rolI_reg(iRegINoSp dst, iRegI src, iRegI shift)
13517 %{
13518 match(Set dst (RotateLeft src shift));
13519
13520 ins_cost(INSN_COST);
13521 format %{ "rol $dst, $src, $shift" %}
13522
13523 ins_encode %{
13524 __ subw(rscratch1, zr, as_Register($shift$$reg));
13525 __ rorvw(as_Register($dst$$reg), as_Register($src$$reg), rscratch1);
13526 %}
13527 ins_pipe(ialu_reg_reg_vshift);
13528 %}
13529
13530 // This pattern is automatically generated from aarch64_ad.m4
13531 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13532 instruct rolL_reg(iRegLNoSp dst, iRegL src, iRegI shift)
13533 %{
13534 match(Set dst (RotateLeft src shift));
13535
13536 ins_cost(INSN_COST);
13537 format %{ "rol $dst, $src, $shift" %}
13538
13539 ins_encode %{
13540 __ subw(rscratch1, zr, as_Register($shift$$reg));
13541 __ rorv(as_Register($dst$$reg), as_Register($src$$reg), rscratch1);
13542 %}
13543 ins_pipe(ialu_reg_reg_vshift);
13544 %}
13545
13546
13547 // Add/subtract (extended)
13548
13549 // This pattern is automatically generated from aarch64_ad.m4
13550 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13551 instruct AddExtI(iRegLNoSp dst, iRegL src1, iRegIorL2I src2, rFlagsReg cr)
13552 %{
13553 match(Set dst (AddL src1 (ConvI2L src2)));
13554 ins_cost(INSN_COST);
13555 format %{ "add $dst, $src1, $src2, sxtw" %}
13556
13557 ins_encode %{
13558 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13559 as_Register($src2$$reg), ext::sxtw);
13560 %}
13561 ins_pipe(ialu_reg_reg);
13562 %}
13563
13564 // This pattern is automatically generated from aarch64_ad.m4
13565 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13566 instruct SubExtI(iRegLNoSp dst, iRegL src1, iRegIorL2I src2, rFlagsReg cr)
13567 %{
13568 match(Set dst (SubL src1 (ConvI2L src2)));
13569 ins_cost(INSN_COST);
13570 format %{ "sub $dst, $src1, $src2, sxtw" %}
13571
13572 ins_encode %{
13573 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13574 as_Register($src2$$reg), ext::sxtw);
13575 %}
13576 ins_pipe(ialu_reg_reg);
13577 %}
13578
13579 // This pattern is automatically generated from aarch64_ad.m4
13580 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13581 instruct AddExtI_sxth(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_16 lshift, immI_16 rshift, rFlagsReg cr)
13582 %{
13583 match(Set dst (AddI src1 (RShiftI (LShiftI src2 lshift) rshift)));
13584 ins_cost(INSN_COST);
13585 format %{ "add $dst, $src1, $src2, sxth" %}
13586
13587 ins_encode %{
13588 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13589 as_Register($src2$$reg), ext::sxth);
13590 %}
13591 ins_pipe(ialu_reg_reg);
13592 %}
13593
13594 // This pattern is automatically generated from aarch64_ad.m4
13595 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13596 instruct AddExtI_sxtb(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_24 lshift, immI_24 rshift, rFlagsReg cr)
13597 %{
13598 match(Set dst (AddI src1 (RShiftI (LShiftI src2 lshift) rshift)));
13599 ins_cost(INSN_COST);
13600 format %{ "add $dst, $src1, $src2, sxtb" %}
13601
13602 ins_encode %{
13603 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13604 as_Register($src2$$reg), ext::sxtb);
13605 %}
13606 ins_pipe(ialu_reg_reg);
13607 %}
13608
13609 // This pattern is automatically generated from aarch64_ad.m4
13610 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13611 instruct AddExtI_uxtb(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_24 lshift, immI_24 rshift, rFlagsReg cr)
13612 %{
13613 match(Set dst (AddI src1 (URShiftI (LShiftI src2 lshift) rshift)));
13614 ins_cost(INSN_COST);
13615 format %{ "add $dst, $src1, $src2, uxtb" %}
13616
13617 ins_encode %{
13618 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13619 as_Register($src2$$reg), ext::uxtb);
13620 %}
13621 ins_pipe(ialu_reg_reg);
13622 %}
13623
13624 // This pattern is automatically generated from aarch64_ad.m4
13625 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13626 instruct AddExtL_sxth(iRegLNoSp dst, iRegL src1, iRegL src2, immI_48 lshift, immI_48 rshift, rFlagsReg cr)
13627 %{
13628 match(Set dst (AddL src1 (RShiftL (LShiftL src2 lshift) rshift)));
13629 ins_cost(INSN_COST);
13630 format %{ "add $dst, $src1, $src2, sxth" %}
13631
13632 ins_encode %{
13633 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13634 as_Register($src2$$reg), ext::sxth);
13635 %}
13636 ins_pipe(ialu_reg_reg);
13637 %}
13638
13639 // This pattern is automatically generated from aarch64_ad.m4
13640 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13641 instruct AddExtL_sxtw(iRegLNoSp dst, iRegL src1, iRegL src2, immI_32 lshift, immI_32 rshift, rFlagsReg cr)
13642 %{
13643 match(Set dst (AddL src1 (RShiftL (LShiftL src2 lshift) rshift)));
13644 ins_cost(INSN_COST);
13645 format %{ "add $dst, $src1, $src2, sxtw" %}
13646
13647 ins_encode %{
13648 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13649 as_Register($src2$$reg), ext::sxtw);
13650 %}
13651 ins_pipe(ialu_reg_reg);
13652 %}
13653
13654 // This pattern is automatically generated from aarch64_ad.m4
13655 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13656 instruct AddExtL_sxtb(iRegLNoSp dst, iRegL src1, iRegL src2, immI_56 lshift, immI_56 rshift, rFlagsReg cr)
13657 %{
13658 match(Set dst (AddL src1 (RShiftL (LShiftL src2 lshift) rshift)));
13659 ins_cost(INSN_COST);
13660 format %{ "add $dst, $src1, $src2, sxtb" %}
13661
13662 ins_encode %{
13663 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13664 as_Register($src2$$reg), ext::sxtb);
13665 %}
13666 ins_pipe(ialu_reg_reg);
13667 %}
13668
13669 // This pattern is automatically generated from aarch64_ad.m4
13670 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13671 instruct AddExtL_uxtb(iRegLNoSp dst, iRegL src1, iRegL src2, immI_56 lshift, immI_56 rshift, rFlagsReg cr)
13672 %{
13673 match(Set dst (AddL src1 (URShiftL (LShiftL src2 lshift) rshift)));
13674 ins_cost(INSN_COST);
13675 format %{ "add $dst, $src1, $src2, uxtb" %}
13676
13677 ins_encode %{
13678 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13679 as_Register($src2$$reg), ext::uxtb);
13680 %}
13681 ins_pipe(ialu_reg_reg);
13682 %}
13683
13684 // This pattern is automatically generated from aarch64_ad.m4
13685 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13686 instruct AddExtI_uxtb_and(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_255 mask, rFlagsReg cr)
13687 %{
13688 match(Set dst (AddI src1 (AndI src2 mask)));
13689 ins_cost(INSN_COST);
13690 format %{ "addw $dst, $src1, $src2, uxtb" %}
13691
13692 ins_encode %{
13693 __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
13694 as_Register($src2$$reg), ext::uxtb);
13695 %}
13696 ins_pipe(ialu_reg_reg);
13697 %}
13698
13699 // This pattern is automatically generated from aarch64_ad.m4
13700 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13701 instruct AddExtI_uxth_and(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_65535 mask, rFlagsReg cr)
13702 %{
13703 match(Set dst (AddI src1 (AndI src2 mask)));
13704 ins_cost(INSN_COST);
13705 format %{ "addw $dst, $src1, $src2, uxth" %}
13706
13707 ins_encode %{
13708 __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
13709 as_Register($src2$$reg), ext::uxth);
13710 %}
13711 ins_pipe(ialu_reg_reg);
13712 %}
13713
13714 // This pattern is automatically generated from aarch64_ad.m4
13715 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13716 instruct AddExtL_uxtb_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_255 mask, rFlagsReg cr)
13717 %{
13718 match(Set dst (AddL src1 (AndL src2 mask)));
13719 ins_cost(INSN_COST);
13720 format %{ "add $dst, $src1, $src2, uxtb" %}
13721
13722 ins_encode %{
13723 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13724 as_Register($src2$$reg), ext::uxtb);
13725 %}
13726 ins_pipe(ialu_reg_reg);
13727 %}
13728
13729 // This pattern is automatically generated from aarch64_ad.m4
13730 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13731 instruct AddExtL_uxth_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_65535 mask, rFlagsReg cr)
13732 %{
13733 match(Set dst (AddL src1 (AndL src2 mask)));
13734 ins_cost(INSN_COST);
13735 format %{ "add $dst, $src1, $src2, uxth" %}
13736
13737 ins_encode %{
13738 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13739 as_Register($src2$$reg), ext::uxth);
13740 %}
13741 ins_pipe(ialu_reg_reg);
13742 %}
13743
13744 // This pattern is automatically generated from aarch64_ad.m4
13745 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13746 instruct AddExtL_uxtw_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_4294967295 mask, rFlagsReg cr)
13747 %{
13748 match(Set dst (AddL src1 (AndL src2 mask)));
13749 ins_cost(INSN_COST);
13750 format %{ "add $dst, $src1, $src2, uxtw" %}
13751
13752 ins_encode %{
13753 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13754 as_Register($src2$$reg), ext::uxtw);
13755 %}
13756 ins_pipe(ialu_reg_reg);
13757 %}
13758
13759 // This pattern is automatically generated from aarch64_ad.m4
13760 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13761 instruct SubExtI_uxtb_and(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_255 mask, rFlagsReg cr)
13762 %{
13763 match(Set dst (SubI src1 (AndI src2 mask)));
13764 ins_cost(INSN_COST);
13765 format %{ "subw $dst, $src1, $src2, uxtb" %}
13766
13767 ins_encode %{
13768 __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
13769 as_Register($src2$$reg), ext::uxtb);
13770 %}
13771 ins_pipe(ialu_reg_reg);
13772 %}
13773
13774 // This pattern is automatically generated from aarch64_ad.m4
13775 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13776 instruct SubExtI_uxth_and(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_65535 mask, rFlagsReg cr)
13777 %{
13778 match(Set dst (SubI src1 (AndI src2 mask)));
13779 ins_cost(INSN_COST);
13780 format %{ "subw $dst, $src1, $src2, uxth" %}
13781
13782 ins_encode %{
13783 __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
13784 as_Register($src2$$reg), ext::uxth);
13785 %}
13786 ins_pipe(ialu_reg_reg);
13787 %}
13788
13789 // This pattern is automatically generated from aarch64_ad.m4
13790 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13791 instruct SubExtL_uxtb_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_255 mask, rFlagsReg cr)
13792 %{
13793 match(Set dst (SubL src1 (AndL src2 mask)));
13794 ins_cost(INSN_COST);
13795 format %{ "sub $dst, $src1, $src2, uxtb" %}
13796
13797 ins_encode %{
13798 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13799 as_Register($src2$$reg), ext::uxtb);
13800 %}
13801 ins_pipe(ialu_reg_reg);
13802 %}
13803
13804 // This pattern is automatically generated from aarch64_ad.m4
13805 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13806 instruct SubExtL_uxth_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_65535 mask, rFlagsReg cr)
13807 %{
13808 match(Set dst (SubL src1 (AndL src2 mask)));
13809 ins_cost(INSN_COST);
13810 format %{ "sub $dst, $src1, $src2, uxth" %}
13811
13812 ins_encode %{
13813 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13814 as_Register($src2$$reg), ext::uxth);
13815 %}
13816 ins_pipe(ialu_reg_reg);
13817 %}
13818
13819 // This pattern is automatically generated from aarch64_ad.m4
13820 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13821 instruct SubExtL_uxtw_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_4294967295 mask, rFlagsReg cr)
13822 %{
13823 match(Set dst (SubL src1 (AndL src2 mask)));
13824 ins_cost(INSN_COST);
13825 format %{ "sub $dst, $src1, $src2, uxtw" %}
13826
13827 ins_encode %{
13828 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13829 as_Register($src2$$reg), ext::uxtw);
13830 %}
13831 ins_pipe(ialu_reg_reg);
13832 %}
13833
13834
13835 // This pattern is automatically generated from aarch64_ad.m4
13836 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13837 instruct AddExtL_sxtb_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_56 lshift1, immI_56 rshift1, rFlagsReg cr)
13838 %{
13839 match(Set dst (AddL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
13840 ins_cost(1.9 * INSN_COST);
13841 format %{ "add $dst, $src1, $src2, sxtb #lshift2" %}
13842
13843 ins_encode %{
13844 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13845 as_Register($src2$$reg), ext::sxtb, ($lshift2$$constant));
13846 %}
13847 ins_pipe(ialu_reg_reg_shift);
13848 %}
13849
13850 // This pattern is automatically generated from aarch64_ad.m4
13851 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13852 instruct AddExtL_sxth_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_48 lshift1, immI_48 rshift1, rFlagsReg cr)
13853 %{
13854 match(Set dst (AddL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
13855 ins_cost(1.9 * INSN_COST);
13856 format %{ "add $dst, $src1, $src2, sxth #lshift2" %}
13857
13858 ins_encode %{
13859 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13860 as_Register($src2$$reg), ext::sxth, ($lshift2$$constant));
13861 %}
13862 ins_pipe(ialu_reg_reg_shift);
13863 %}
13864
13865 // This pattern is automatically generated from aarch64_ad.m4
13866 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13867 instruct AddExtL_sxtw_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_32 lshift1, immI_32 rshift1, rFlagsReg cr)
13868 %{
13869 match(Set dst (AddL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
13870 ins_cost(1.9 * INSN_COST);
13871 format %{ "add $dst, $src1, $src2, sxtw #lshift2" %}
13872
13873 ins_encode %{
13874 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13875 as_Register($src2$$reg), ext::sxtw, ($lshift2$$constant));
13876 %}
13877 ins_pipe(ialu_reg_reg_shift);
13878 %}
13879
13880 // This pattern is automatically generated from aarch64_ad.m4
13881 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13882 instruct SubExtL_sxtb_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_56 lshift1, immI_56 rshift1, rFlagsReg cr)
13883 %{
13884 match(Set dst (SubL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
13885 ins_cost(1.9 * INSN_COST);
13886 format %{ "sub $dst, $src1, $src2, sxtb #lshift2" %}
13887
13888 ins_encode %{
13889 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13890 as_Register($src2$$reg), ext::sxtb, ($lshift2$$constant));
13891 %}
13892 ins_pipe(ialu_reg_reg_shift);
13893 %}
13894
13895 // This pattern is automatically generated from aarch64_ad.m4
13896 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13897 instruct SubExtL_sxth_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_48 lshift1, immI_48 rshift1, rFlagsReg cr)
13898 %{
13899 match(Set dst (SubL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
13900 ins_cost(1.9 * INSN_COST);
13901 format %{ "sub $dst, $src1, $src2, sxth #lshift2" %}
13902
13903 ins_encode %{
13904 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13905 as_Register($src2$$reg), ext::sxth, ($lshift2$$constant));
13906 %}
13907 ins_pipe(ialu_reg_reg_shift);
13908 %}
13909
13910 // This pattern is automatically generated from aarch64_ad.m4
13911 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13912 instruct SubExtL_sxtw_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_32 lshift1, immI_32 rshift1, rFlagsReg cr)
13913 %{
13914 match(Set dst (SubL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
13915 ins_cost(1.9 * INSN_COST);
13916 format %{ "sub $dst, $src1, $src2, sxtw #lshift2" %}
13917
13918 ins_encode %{
13919 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13920 as_Register($src2$$reg), ext::sxtw, ($lshift2$$constant));
13921 %}
13922 ins_pipe(ialu_reg_reg_shift);
13923 %}
13924
13925 // This pattern is automatically generated from aarch64_ad.m4
13926 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13927 instruct AddExtI_sxtb_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immIExt lshift2, immI_24 lshift1, immI_24 rshift1, rFlagsReg cr)
13928 %{
13929 match(Set dst (AddI src1 (LShiftI (RShiftI (LShiftI src2 lshift1) rshift1) lshift2)));
13930 ins_cost(1.9 * INSN_COST);
13931 format %{ "addw $dst, $src1, $src2, sxtb #lshift2" %}
13932
13933 ins_encode %{
13934 __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
13935 as_Register($src2$$reg), ext::sxtb, ($lshift2$$constant));
13936 %}
13937 ins_pipe(ialu_reg_reg_shift);
13938 %}
13939
13940 // This pattern is automatically generated from aarch64_ad.m4
13941 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13942 instruct AddExtI_sxth_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immIExt lshift2, immI_16 lshift1, immI_16 rshift1, rFlagsReg cr)
13943 %{
13944 match(Set dst (AddI src1 (LShiftI (RShiftI (LShiftI src2 lshift1) rshift1) lshift2)));
13945 ins_cost(1.9 * INSN_COST);
13946 format %{ "addw $dst, $src1, $src2, sxth #lshift2" %}
13947
13948 ins_encode %{
13949 __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
13950 as_Register($src2$$reg), ext::sxth, ($lshift2$$constant));
13951 %}
13952 ins_pipe(ialu_reg_reg_shift);
13953 %}
13954
13955 // This pattern is automatically generated from aarch64_ad.m4
13956 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13957 instruct SubExtI_sxtb_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immIExt lshift2, immI_24 lshift1, immI_24 rshift1, rFlagsReg cr)
13958 %{
13959 match(Set dst (SubI src1 (LShiftI (RShiftI (LShiftI src2 lshift1) rshift1) lshift2)));
13960 ins_cost(1.9 * INSN_COST);
13961 format %{ "subw $dst, $src1, $src2, sxtb #lshift2" %}
13962
13963 ins_encode %{
13964 __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
13965 as_Register($src2$$reg), ext::sxtb, ($lshift2$$constant));
13966 %}
13967 ins_pipe(ialu_reg_reg_shift);
13968 %}
13969
13970 // This pattern is automatically generated from aarch64_ad.m4
13971 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13972 instruct SubExtI_sxth_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immIExt lshift2, immI_16 lshift1, immI_16 rshift1, rFlagsReg cr)
13973 %{
13974 match(Set dst (SubI src1 (LShiftI (RShiftI (LShiftI src2 lshift1) rshift1) lshift2)));
13975 ins_cost(1.9 * INSN_COST);
13976 format %{ "subw $dst, $src1, $src2, sxth #lshift2" %}
13977
13978 ins_encode %{
13979 __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
13980 as_Register($src2$$reg), ext::sxth, ($lshift2$$constant));
13981 %}
13982 ins_pipe(ialu_reg_reg_shift);
13983 %}
13984
13985 // This pattern is automatically generated from aarch64_ad.m4
13986 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13987 instruct AddExtI_shift(iRegLNoSp dst, iRegL src1, iRegIorL2I src2, immIExt lshift, rFlagsReg cr)
13988 %{
13989 match(Set dst (AddL src1 (LShiftL (ConvI2L src2) lshift)));
13990 ins_cost(1.9 * INSN_COST);
13991 format %{ "add $dst, $src1, $src2, sxtw #lshift" %}
13992
13993 ins_encode %{
13994 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13995 as_Register($src2$$reg), ext::sxtw, ($lshift$$constant));
13996 %}
13997 ins_pipe(ialu_reg_reg_shift);
13998 %}
13999
14000 // This pattern is automatically generated from aarch64_ad.m4
14001 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
14002 instruct SubExtI_shift(iRegLNoSp dst, iRegL src1, iRegIorL2I src2, immIExt lshift, rFlagsReg cr)
14003 %{
14004 match(Set dst (SubL src1 (LShiftL (ConvI2L src2) lshift)));
14005 ins_cost(1.9 * INSN_COST);
14006 format %{ "sub $dst, $src1, $src2, sxtw #lshift" %}
14007
14008 ins_encode %{
14009 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
14010 as_Register($src2$$reg), ext::sxtw, ($lshift$$constant));
14011 %}
14012 ins_pipe(ialu_reg_reg_shift);
14013 %}
14014
14015 // This pattern is automatically generated from aarch64_ad.m4
14016 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
14017 instruct AddExtL_uxtb_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_255 mask, immIExt lshift, rFlagsReg cr)
14018 %{
14019 match(Set dst (AddL src1 (LShiftL (AndL src2 mask) lshift)));
14020 ins_cost(1.9 * INSN_COST);
14021 format %{ "add $dst, $src1, $src2, uxtb #lshift" %}
14022
14023 ins_encode %{
14024 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
14025 as_Register($src2$$reg), ext::uxtb, ($lshift$$constant));
14026 %}
14027 ins_pipe(ialu_reg_reg_shift);
14028 %}
14029
14030 // This pattern is automatically generated from aarch64_ad.m4
14031 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
14032 instruct AddExtL_uxth_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_65535 mask, immIExt lshift, rFlagsReg cr)
14033 %{
14034 match(Set dst (AddL src1 (LShiftL (AndL src2 mask) lshift)));
14035 ins_cost(1.9 * INSN_COST);
14036 format %{ "add $dst, $src1, $src2, uxth #lshift" %}
14037
14038 ins_encode %{
14039 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
14040 as_Register($src2$$reg), ext::uxth, ($lshift$$constant));
14041 %}
14042 ins_pipe(ialu_reg_reg_shift);
14043 %}
14044
14045 // This pattern is automatically generated from aarch64_ad.m4
14046 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
14047 instruct AddExtL_uxtw_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_4294967295 mask, immIExt lshift, rFlagsReg cr)
14048 %{
14049 match(Set dst (AddL src1 (LShiftL (AndL src2 mask) lshift)));
14050 ins_cost(1.9 * INSN_COST);
14051 format %{ "add $dst, $src1, $src2, uxtw #lshift" %}
14052
14053 ins_encode %{
14054 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
14055 as_Register($src2$$reg), ext::uxtw, ($lshift$$constant));
14056 %}
14057 ins_pipe(ialu_reg_reg_shift);
14058 %}
14059
14060 // This pattern is automatically generated from aarch64_ad.m4
14061 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
14062 instruct SubExtL_uxtb_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_255 mask, immIExt lshift, rFlagsReg cr)
14063 %{
14064 match(Set dst (SubL src1 (LShiftL (AndL src2 mask) lshift)));
14065 ins_cost(1.9 * INSN_COST);
14066 format %{ "sub $dst, $src1, $src2, uxtb #lshift" %}
14067
14068 ins_encode %{
14069 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
14070 as_Register($src2$$reg), ext::uxtb, ($lshift$$constant));
14071 %}
14072 ins_pipe(ialu_reg_reg_shift);
14073 %}
14074
14075 // This pattern is automatically generated from aarch64_ad.m4
14076 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
14077 instruct SubExtL_uxth_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_65535 mask, immIExt lshift, rFlagsReg cr)
14078 %{
14079 match(Set dst (SubL src1 (LShiftL (AndL src2 mask) lshift)));
14080 ins_cost(1.9 * INSN_COST);
14081 format %{ "sub $dst, $src1, $src2, uxth #lshift" %}
14082
14083 ins_encode %{
14084 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
14085 as_Register($src2$$reg), ext::uxth, ($lshift$$constant));
14086 %}
14087 ins_pipe(ialu_reg_reg_shift);
14088 %}
14089
14090 // This pattern is automatically generated from aarch64_ad.m4
14091 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
14092 instruct SubExtL_uxtw_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_4294967295 mask, immIExt lshift, rFlagsReg cr)
14093 %{
14094 match(Set dst (SubL src1 (LShiftL (AndL src2 mask) lshift)));
14095 ins_cost(1.9 * INSN_COST);
14096 format %{ "sub $dst, $src1, $src2, uxtw #lshift" %}
14097
14098 ins_encode %{
14099 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
14100 as_Register($src2$$reg), ext::uxtw, ($lshift$$constant));
14101 %}
14102 ins_pipe(ialu_reg_reg_shift);
14103 %}
14104
14105 // This pattern is automatically generated from aarch64_ad.m4
14106 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
14107 instruct AddExtI_uxtb_and_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_255 mask, immIExt lshift, rFlagsReg cr)
14108 %{
14109 match(Set dst (AddI src1 (LShiftI (AndI src2 mask) lshift)));
14110 ins_cost(1.9 * INSN_COST);
14111 format %{ "addw $dst, $src1, $src2, uxtb #lshift" %}
14112
14113 ins_encode %{
14114 __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
14115 as_Register($src2$$reg), ext::uxtb, ($lshift$$constant));
14116 %}
14117 ins_pipe(ialu_reg_reg_shift);
14118 %}
14119
14120 // This pattern is automatically generated from aarch64_ad.m4
14121 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
14122 instruct AddExtI_uxth_and_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_65535 mask, immIExt lshift, rFlagsReg cr)
14123 %{
14124 match(Set dst (AddI src1 (LShiftI (AndI src2 mask) lshift)));
14125 ins_cost(1.9 * INSN_COST);
14126 format %{ "addw $dst, $src1, $src2, uxth #lshift" %}
14127
14128 ins_encode %{
14129 __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
14130 as_Register($src2$$reg), ext::uxth, ($lshift$$constant));
14131 %}
14132 ins_pipe(ialu_reg_reg_shift);
14133 %}
14134
14135 // This pattern is automatically generated from aarch64_ad.m4
14136 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
14137 instruct SubExtI_uxtb_and_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_255 mask, immIExt lshift, rFlagsReg cr)
14138 %{
14139 match(Set dst (SubI src1 (LShiftI (AndI src2 mask) lshift)));
14140 ins_cost(1.9 * INSN_COST);
14141 format %{ "subw $dst, $src1, $src2, uxtb #lshift" %}
14142
14143 ins_encode %{
14144 __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
14145 as_Register($src2$$reg), ext::uxtb, ($lshift$$constant));
14146 %}
14147 ins_pipe(ialu_reg_reg_shift);
14148 %}
14149
14150 // This pattern is automatically generated from aarch64_ad.m4
14151 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
14152 instruct SubExtI_uxth_and_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_65535 mask, immIExt lshift, rFlagsReg cr)
14153 %{
14154 match(Set dst (SubI src1 (LShiftI (AndI src2 mask) lshift)));
14155 ins_cost(1.9 * INSN_COST);
14156 format %{ "subw $dst, $src1, $src2, uxth #lshift" %}
14157
14158 ins_encode %{
14159 __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
14160 as_Register($src2$$reg), ext::uxth, ($lshift$$constant));
14161 %}
14162 ins_pipe(ialu_reg_reg_shift);
14163 %}
14164
14165
14166
14167 // END This section of the file is automatically generated. Do not edit --------------
14168
14169
14170 // ============================================================================
14171 // Floating Point Arithmetic Instructions
14172
14173 instruct addF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
14174 match(Set dst (AddF src1 src2));
14175
14176 ins_cost(INSN_COST * 5);
14177 format %{ "fadds $dst, $src1, $src2" %}
14178
14179 ins_encode %{
14180 __ fadds(as_FloatRegister($dst$$reg),
14181 as_FloatRegister($src1$$reg),
14182 as_FloatRegister($src2$$reg));
14183 %}
14184
14185 ins_pipe(fp_dop_reg_reg_s);
14186 %}
14187
14188 instruct addD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
14189 match(Set dst (AddD src1 src2));
14190
14191 ins_cost(INSN_COST * 5);
14192 format %{ "faddd $dst, $src1, $src2" %}
14193
14194 ins_encode %{
14195 __ faddd(as_FloatRegister($dst$$reg),
14196 as_FloatRegister($src1$$reg),
14197 as_FloatRegister($src2$$reg));
14198 %}
14199
14200 ins_pipe(fp_dop_reg_reg_d);
14201 %}
14202
14203 instruct subF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
14204 match(Set dst (SubF src1 src2));
14205
14206 ins_cost(INSN_COST * 5);
14207 format %{ "fsubs $dst, $src1, $src2" %}
14208
14209 ins_encode %{
14210 __ fsubs(as_FloatRegister($dst$$reg),
14211 as_FloatRegister($src1$$reg),
14212 as_FloatRegister($src2$$reg));
14213 %}
14214
14215 ins_pipe(fp_dop_reg_reg_s);
14216 %}
14217
14218 instruct subD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
14219 match(Set dst (SubD src1 src2));
14220
14221 ins_cost(INSN_COST * 5);
14222 format %{ "fsubd $dst, $src1, $src2" %}
14223
14224 ins_encode %{
14225 __ fsubd(as_FloatRegister($dst$$reg),
14226 as_FloatRegister($src1$$reg),
14227 as_FloatRegister($src2$$reg));
14228 %}
14229
14230 ins_pipe(fp_dop_reg_reg_d);
14231 %}
14232
14233 instruct mulF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
14234 match(Set dst (MulF src1 src2));
14235
14236 ins_cost(INSN_COST * 6);
14237 format %{ "fmuls $dst, $src1, $src2" %}
14238
14239 ins_encode %{
14240 __ fmuls(as_FloatRegister($dst$$reg),
14241 as_FloatRegister($src1$$reg),
14242 as_FloatRegister($src2$$reg));
14243 %}
14244
14245 ins_pipe(fp_dop_reg_reg_s);
14246 %}
14247
14248 instruct mulD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
14249 match(Set dst (MulD src1 src2));
14250
14251 ins_cost(INSN_COST * 6);
14252 format %{ "fmuld $dst, $src1, $src2" %}
14253
14254 ins_encode %{
14255 __ fmuld(as_FloatRegister($dst$$reg),
14256 as_FloatRegister($src1$$reg),
14257 as_FloatRegister($src2$$reg));
14258 %}
14259
14260 ins_pipe(fp_dop_reg_reg_d);
14261 %}
14262
14263 // src1 * src2 + src3
14264 instruct maddF_reg_reg(vRegF dst, vRegF src1, vRegF src2, vRegF src3) %{
14265 predicate(UseFMA);
14266 match(Set dst (FmaF src3 (Binary src1 src2)));
14267
14268 format %{ "fmadds $dst, $src1, $src2, $src3" %}
14269
14270 ins_encode %{
14271 __ fmadds(as_FloatRegister($dst$$reg),
14272 as_FloatRegister($src1$$reg),
14273 as_FloatRegister($src2$$reg),
14274 as_FloatRegister($src3$$reg));
14275 %}
14276
14277 ins_pipe(pipe_class_default);
14278 %}
14279
14280 // src1 * src2 + src3
14281 instruct maddD_reg_reg(vRegD dst, vRegD src1, vRegD src2, vRegD src3) %{
14282 predicate(UseFMA);
14283 match(Set dst (FmaD src3 (Binary src1 src2)));
14284
14285 format %{ "fmaddd $dst, $src1, $src2, $src3" %}
14286
14287 ins_encode %{
14288 __ fmaddd(as_FloatRegister($dst$$reg),
14289 as_FloatRegister($src1$$reg),
14290 as_FloatRegister($src2$$reg),
14291 as_FloatRegister($src3$$reg));
14292 %}
14293
14294 ins_pipe(pipe_class_default);
14295 %}
14296
14297 // -src1 * src2 + src3
14298 instruct msubF_reg_reg(vRegF dst, vRegF src1, vRegF src2, vRegF src3) %{
14299 predicate(UseFMA);
14300 match(Set dst (FmaF src3 (Binary (NegF src1) src2)));
14301 match(Set dst (FmaF src3 (Binary src1 (NegF src2))));
14302
14303 format %{ "fmsubs $dst, $src1, $src2, $src3" %}
14304
14305 ins_encode %{
14306 __ fmsubs(as_FloatRegister($dst$$reg),
14307 as_FloatRegister($src1$$reg),
14308 as_FloatRegister($src2$$reg),
14309 as_FloatRegister($src3$$reg));
14310 %}
14311
14312 ins_pipe(pipe_class_default);
14313 %}
14314
14315 // -src1 * src2 + src3
14316 instruct msubD_reg_reg(vRegD dst, vRegD src1, vRegD src2, vRegD src3) %{
14317 predicate(UseFMA);
14318 match(Set dst (FmaD src3 (Binary (NegD src1) src2)));
14319 match(Set dst (FmaD src3 (Binary src1 (NegD src2))));
14320
14321 format %{ "fmsubd $dst, $src1, $src2, $src3" %}
14322
14323 ins_encode %{
14324 __ fmsubd(as_FloatRegister($dst$$reg),
14325 as_FloatRegister($src1$$reg),
14326 as_FloatRegister($src2$$reg),
14327 as_FloatRegister($src3$$reg));
14328 %}
14329
14330 ins_pipe(pipe_class_default);
14331 %}
14332
14333 // -src1 * src2 - src3
14334 instruct mnaddF_reg_reg(vRegF dst, vRegF src1, vRegF src2, vRegF src3) %{
14335 predicate(UseFMA);
14336 match(Set dst (FmaF (NegF src3) (Binary (NegF src1) src2)));
14337 match(Set dst (FmaF (NegF src3) (Binary src1 (NegF src2))));
14338
14339 format %{ "fnmadds $dst, $src1, $src2, $src3" %}
14340
14341 ins_encode %{
14342 __ fnmadds(as_FloatRegister($dst$$reg),
14343 as_FloatRegister($src1$$reg),
14344 as_FloatRegister($src2$$reg),
14345 as_FloatRegister($src3$$reg));
14346 %}
14347
14348 ins_pipe(pipe_class_default);
14349 %}
14350
14351 // -src1 * src2 - src3
14352 instruct mnaddD_reg_reg(vRegD dst, vRegD src1, vRegD src2, vRegD src3) %{
14353 predicate(UseFMA);
14354 match(Set dst (FmaD (NegD src3) (Binary (NegD src1) src2)));
14355 match(Set dst (FmaD (NegD src3) (Binary src1 (NegD src2))));
14356
14357 format %{ "fnmaddd $dst, $src1, $src2, $src3" %}
14358
14359 ins_encode %{
14360 __ fnmaddd(as_FloatRegister($dst$$reg),
14361 as_FloatRegister($src1$$reg),
14362 as_FloatRegister($src2$$reg),
14363 as_FloatRegister($src3$$reg));
14364 %}
14365
14366 ins_pipe(pipe_class_default);
14367 %}
14368
14369 // src1 * src2 - src3
14370 instruct mnsubF_reg_reg(vRegF dst, vRegF src1, vRegF src2, vRegF src3, immF0 zero) %{
14371 predicate(UseFMA);
14372 match(Set dst (FmaF (NegF src3) (Binary src1 src2)));
14373
14374 format %{ "fnmsubs $dst, $src1, $src2, $src3" %}
14375
14376 ins_encode %{
14377 __ fnmsubs(as_FloatRegister($dst$$reg),
14378 as_FloatRegister($src1$$reg),
14379 as_FloatRegister($src2$$reg),
14380 as_FloatRegister($src3$$reg));
14381 %}
14382
14383 ins_pipe(pipe_class_default);
14384 %}
14385
14386 // src1 * src2 - src3
14387 instruct mnsubD_reg_reg(vRegD dst, vRegD src1, vRegD src2, vRegD src3, immD0 zero) %{
14388 predicate(UseFMA);
14389 match(Set dst (FmaD (NegD src3) (Binary src1 src2)));
14390
14391 format %{ "fnmsubd $dst, $src1, $src2, $src3" %}
14392
14393 ins_encode %{
14394 // n.b. insn name should be fnmsubd
14395 __ fnmsub(as_FloatRegister($dst$$reg),
14396 as_FloatRegister($src1$$reg),
14397 as_FloatRegister($src2$$reg),
14398 as_FloatRegister($src3$$reg));
14399 %}
14400
14401 ins_pipe(pipe_class_default);
14402 %}
14403
14404
14405 // Math.max(FF)F
14406 instruct maxF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
14407 match(Set dst (MaxF src1 src2));
14408
14409 format %{ "fmaxs $dst, $src1, $src2" %}
14410 ins_encode %{
14411 __ fmaxs(as_FloatRegister($dst$$reg),
14412 as_FloatRegister($src1$$reg),
14413 as_FloatRegister($src2$$reg));
14414 %}
14415
14416 ins_pipe(fp_dop_reg_reg_s);
14417 %}
14418
14419 // Math.min(FF)F
14420 instruct minF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
14421 match(Set dst (MinF src1 src2));
14422
14423 format %{ "fmins $dst, $src1, $src2" %}
14424 ins_encode %{
14425 __ fmins(as_FloatRegister($dst$$reg),
14426 as_FloatRegister($src1$$reg),
14427 as_FloatRegister($src2$$reg));
14428 %}
14429
14430 ins_pipe(fp_dop_reg_reg_s);
14431 %}
14432
14433 // Math.max(DD)D
14434 instruct maxD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
14435 match(Set dst (MaxD src1 src2));
14436
14437 format %{ "fmaxd $dst, $src1, $src2" %}
14438 ins_encode %{
14439 __ fmaxd(as_FloatRegister($dst$$reg),
14440 as_FloatRegister($src1$$reg),
14441 as_FloatRegister($src2$$reg));
14442 %}
14443
14444 ins_pipe(fp_dop_reg_reg_d);
14445 %}
14446
14447 // Math.min(DD)D
14448 instruct minD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
14449 match(Set dst (MinD src1 src2));
14450
14451 format %{ "fmind $dst, $src1, $src2" %}
14452 ins_encode %{
14453 __ fmind(as_FloatRegister($dst$$reg),
14454 as_FloatRegister($src1$$reg),
14455 as_FloatRegister($src2$$reg));
14456 %}
14457
14458 ins_pipe(fp_dop_reg_reg_d);
14459 %}
14460
14461
14462 instruct divF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
14463 match(Set dst (DivF src1 src2));
14464
14465 ins_cost(INSN_COST * 18);
14466 format %{ "fdivs $dst, $src1, $src2" %}
14467
14468 ins_encode %{
14469 __ fdivs(as_FloatRegister($dst$$reg),
14470 as_FloatRegister($src1$$reg),
14471 as_FloatRegister($src2$$reg));
14472 %}
14473
14474 ins_pipe(fp_div_s);
14475 %}
14476
14477 instruct divD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
14478 match(Set dst (DivD src1 src2));
14479
14480 ins_cost(INSN_COST * 32);
14481 format %{ "fdivd $dst, $src1, $src2" %}
14482
14483 ins_encode %{
14484 __ fdivd(as_FloatRegister($dst$$reg),
14485 as_FloatRegister($src1$$reg),
14486 as_FloatRegister($src2$$reg));
14487 %}
14488
14489 ins_pipe(fp_div_d);
14490 %}
14491
14492 instruct negF_reg_reg(vRegF dst, vRegF src) %{
14493 match(Set dst (NegF src));
14494
14495 ins_cost(INSN_COST * 3);
14496 format %{ "fneg $dst, $src" %}
14497
14498 ins_encode %{
14499 __ fnegs(as_FloatRegister($dst$$reg),
14500 as_FloatRegister($src$$reg));
14501 %}
14502
14503 ins_pipe(fp_uop_s);
14504 %}
14505
14506 instruct negD_reg_reg(vRegD dst, vRegD src) %{
14507 match(Set dst (NegD src));
14508
14509 ins_cost(INSN_COST * 3);
14510 format %{ "fnegd $dst, $src" %}
14511
14512 ins_encode %{
14513 __ fnegd(as_FloatRegister($dst$$reg),
14514 as_FloatRegister($src$$reg));
14515 %}
14516
14517 ins_pipe(fp_uop_d);
14518 %}
14519
14520 instruct absI_reg(iRegINoSp dst, iRegIorL2I src, rFlagsReg cr)
14521 %{
14522 match(Set dst (AbsI src));
14523
14524 effect(KILL cr);
14525 ins_cost(INSN_COST * 2);
14526 format %{ "cmpw $src, zr\n\t"
14527 "cnegw $dst, $src, Assembler::LT\t# int abs"
14528 %}
14529
14530 ins_encode %{
14531 __ cmpw(as_Register($src$$reg), zr);
14532 __ cnegw(as_Register($dst$$reg), as_Register($src$$reg), Assembler::LT);
14533 %}
14534 ins_pipe(pipe_class_default);
14535 %}
14536
14537 instruct absL_reg(iRegLNoSp dst, iRegL src, rFlagsReg cr)
14538 %{
14539 match(Set dst (AbsL src));
14540
14541 effect(KILL cr);
14542 ins_cost(INSN_COST * 2);
14543 format %{ "cmp $src, zr\n\t"
14544 "cneg $dst, $src, Assembler::LT\t# long abs"
14545 %}
14546
14547 ins_encode %{
14548 __ cmp(as_Register($src$$reg), zr);
14549 __ cneg(as_Register($dst$$reg), as_Register($src$$reg), Assembler::LT);
14550 %}
14551 ins_pipe(pipe_class_default);
14552 %}
14553
14554 instruct absF_reg(vRegF dst, vRegF src) %{
14555 match(Set dst (AbsF src));
14556
14557 ins_cost(INSN_COST * 3);
14558 format %{ "fabss $dst, $src" %}
14559 ins_encode %{
14560 __ fabss(as_FloatRegister($dst$$reg),
14561 as_FloatRegister($src$$reg));
14562 %}
14563
14564 ins_pipe(fp_uop_s);
14565 %}
14566
14567 instruct absD_reg(vRegD dst, vRegD src) %{
14568 match(Set dst (AbsD src));
14569
14570 ins_cost(INSN_COST * 3);
14571 format %{ "fabsd $dst, $src" %}
14572 ins_encode %{
14573 __ fabsd(as_FloatRegister($dst$$reg),
14574 as_FloatRegister($src$$reg));
14575 %}
14576
14577 ins_pipe(fp_uop_d);
14578 %}
14579
14580 instruct absdF_reg(vRegF dst, vRegF src1, vRegF src2) %{
14581 match(Set dst (AbsF (SubF src1 src2)));
14582
14583 ins_cost(INSN_COST * 3);
14584 format %{ "fabds $dst, $src1, $src2" %}
14585 ins_encode %{
14586 __ fabds(as_FloatRegister($dst$$reg),
14587 as_FloatRegister($src1$$reg),
14588 as_FloatRegister($src2$$reg));
14589 %}
14590
14591 ins_pipe(fp_uop_s);
14592 %}
14593
14594 instruct absdD_reg(vRegD dst, vRegD src1, vRegD src2) %{
14595 match(Set dst (AbsD (SubD src1 src2)));
14596
14597 ins_cost(INSN_COST * 3);
14598 format %{ "fabdd $dst, $src1, $src2" %}
14599 ins_encode %{
14600 __ fabdd(as_FloatRegister($dst$$reg),
14601 as_FloatRegister($src1$$reg),
14602 as_FloatRegister($src2$$reg));
14603 %}
14604
14605 ins_pipe(fp_uop_d);
14606 %}
14607
14608 instruct sqrtD_reg(vRegD dst, vRegD src) %{
14609 match(Set dst (SqrtD src));
14610
14611 ins_cost(INSN_COST * 50);
14612 format %{ "fsqrtd $dst, $src" %}
14613 ins_encode %{
14614 __ fsqrtd(as_FloatRegister($dst$$reg),
14615 as_FloatRegister($src$$reg));
14616 %}
14617
14618 ins_pipe(fp_div_s);
14619 %}
14620
14621 instruct sqrtF_reg(vRegF dst, vRegF src) %{
14622 match(Set dst (SqrtF src));
14623
14624 ins_cost(INSN_COST * 50);
14625 format %{ "fsqrts $dst, $src" %}
14626 ins_encode %{
14627 __ fsqrts(as_FloatRegister($dst$$reg),
14628 as_FloatRegister($src$$reg));
14629 %}
14630
14631 ins_pipe(fp_div_d);
14632 %}
14633
14634 // Math.rint, floor, ceil
14635 instruct roundD_reg(vRegD dst, vRegD src, immI rmode) %{
14636 match(Set dst (RoundDoubleMode src rmode));
14637 format %{ "frint $dst, $src, $rmode" %}
14638 ins_encode %{
14639 switch ($rmode$$constant) {
14640 case RoundDoubleModeNode::rmode_rint:
14641 __ frintnd(as_FloatRegister($dst$$reg),
14642 as_FloatRegister($src$$reg));
14643 break;
14644 case RoundDoubleModeNode::rmode_floor:
14645 __ frintmd(as_FloatRegister($dst$$reg),
14646 as_FloatRegister($src$$reg));
14647 break;
14648 case RoundDoubleModeNode::rmode_ceil:
14649 __ frintpd(as_FloatRegister($dst$$reg),
14650 as_FloatRegister($src$$reg));
14651 break;
14652 }
14653 %}
14654 ins_pipe(fp_uop_d);
14655 %}
14656
14657 instruct copySignD_reg(vRegD dst, vRegD src1, vRegD src2, vRegD zero) %{
14658 match(Set dst (CopySignD src1 (Binary src2 zero)));
14659 effect(TEMP_DEF dst, USE src1, USE src2, USE zero);
14660 format %{ "CopySignD $dst $src1 $src2" %}
14661 ins_encode %{
14662 FloatRegister dst = as_FloatRegister($dst$$reg),
14663 src1 = as_FloatRegister($src1$$reg),
14664 src2 = as_FloatRegister($src2$$reg),
14665 zero = as_FloatRegister($zero$$reg);
14666 __ fnegd(dst, zero);
14667 __ bsl(dst, __ T8B, src2, src1);
14668 %}
14669 ins_pipe(fp_uop_d);
14670 %}
14671
14672 instruct copySignF_reg(vRegF dst, vRegF src1, vRegF src2) %{
14673 match(Set dst (CopySignF src1 src2));
14674 effect(TEMP_DEF dst, USE src1, USE src2);
14675 format %{ "CopySignF $dst $src1 $src2" %}
14676 ins_encode %{
14677 FloatRegister dst = as_FloatRegister($dst$$reg),
14678 src1 = as_FloatRegister($src1$$reg),
14679 src2 = as_FloatRegister($src2$$reg);
14680 __ movi(dst, __ T2S, 0x80, 24);
14681 __ bsl(dst, __ T8B, src2, src1);
14682 %}
14683 ins_pipe(fp_uop_d);
14684 %}
14685
14686 instruct signumD_reg(vRegD dst, vRegD src, vRegD zero, vRegD one) %{
14687 match(Set dst (SignumD src (Binary zero one)));
14688 effect(TEMP_DEF dst, USE src, USE zero, USE one);
14689 format %{ "signumD $dst, $src" %}
14690 ins_encode %{
14691 FloatRegister src = as_FloatRegister($src$$reg),
14692 dst = as_FloatRegister($dst$$reg),
14693 zero = as_FloatRegister($zero$$reg),
14694 one = as_FloatRegister($one$$reg);
14695 __ facgtd(dst, src, zero); // dst=0 for +-0.0 and NaN. 0xFFF..F otherwise
14696 __ ushrd(dst, dst, 1); // dst=0 for +-0.0 and NaN. 0x7FF..F otherwise
14697 // Bit selection instruction gets bit from "one" for each enabled bit in
14698 // "dst", otherwise gets a bit from "src". For "src" that contains +-0.0 or
14699 // NaN the whole "src" will be copied because "dst" is zero. For all other
14700 // "src" values dst is 0x7FF..F, which means only the sign bit is copied
14701 // from "src", and all other bits are copied from 1.0.
14702 __ bsl(dst, __ T8B, one, src);
14703 %}
14704 ins_pipe(fp_uop_d);
14705 %}
14706
14707 instruct signumF_reg(vRegF dst, vRegF src, vRegF zero, vRegF one) %{
14708 match(Set dst (SignumF src (Binary zero one)));
14709 effect(TEMP_DEF dst, USE src, USE zero, USE one);
14710 format %{ "signumF $dst, $src" %}
14711 ins_encode %{
14712 FloatRegister src = as_FloatRegister($src$$reg),
14713 dst = as_FloatRegister($dst$$reg),
14714 zero = as_FloatRegister($zero$$reg),
14715 one = as_FloatRegister($one$$reg);
14716 __ facgts(dst, src, zero); // dst=0 for +-0.0 and NaN. 0xFFF..F otherwise
14717 __ ushr(dst, __ T2S, dst, 1); // dst=0 for +-0.0 and NaN. 0x7FF..F otherwise
14718 // Bit selection instruction gets bit from "one" for each enabled bit in
14719 // "dst", otherwise gets a bit from "src". For "src" that contains +-0.0 or
14720 // NaN the whole "src" will be copied because "dst" is zero. For all other
14721 // "src" values dst is 0x7FF..F, which means only the sign bit is copied
14722 // from "src", and all other bits are copied from 1.0.
14723 __ bsl(dst, __ T8B, one, src);
14724 %}
14725 ins_pipe(fp_uop_d);
14726 %}
14727
14728 instruct onspinwait() %{
14729 match(OnSpinWait);
14730 ins_cost(INSN_COST);
14731
14732 format %{ "onspinwait" %}
14733
14734 ins_encode %{
14735 __ spin_wait();
14736 %}
14737 ins_pipe(pipe_class_empty);
14738 %}
14739
14740 // ============================================================================
14741 // Logical Instructions
14742
14743 // Integer Logical Instructions
14744
14745 // And Instructions
14746
14747
14748 instruct andI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, rFlagsReg cr) %{
14749 match(Set dst (AndI src1 src2));
14750
14751 format %{ "andw $dst, $src1, $src2\t# int" %}
14752
14753 ins_cost(INSN_COST);
14754 ins_encode %{
14755 __ andw(as_Register($dst$$reg),
14756 as_Register($src1$$reg),
14757 as_Register($src2$$reg));
14758 %}
14759
14760 ins_pipe(ialu_reg_reg);
14761 %}
14762
14763 instruct andI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immILog src2, rFlagsReg cr) %{
14764 match(Set dst (AndI src1 src2));
14765
14766 format %{ "andsw $dst, $src1, $src2\t# int" %}
14767
14768 ins_cost(INSN_COST);
14769 ins_encode %{
14770 __ andw(as_Register($dst$$reg),
14771 as_Register($src1$$reg),
14772 (uint64_t)($src2$$constant));
14773 %}
14774
14775 ins_pipe(ialu_reg_imm);
14776 %}
14777
14778 // Or Instructions
14779
14780 instruct orI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
14781 match(Set dst (OrI src1 src2));
14782
14783 format %{ "orrw $dst, $src1, $src2\t# int" %}
14784
14785 ins_cost(INSN_COST);
14786 ins_encode %{
14787 __ orrw(as_Register($dst$$reg),
14788 as_Register($src1$$reg),
14789 as_Register($src2$$reg));
14790 %}
14791
14792 ins_pipe(ialu_reg_reg);
14793 %}
14794
14795 instruct orI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immILog src2) %{
14796 match(Set dst (OrI src1 src2));
14797
14798 format %{ "orrw $dst, $src1, $src2\t# int" %}
14799
14800 ins_cost(INSN_COST);
14801 ins_encode %{
14802 __ orrw(as_Register($dst$$reg),
14803 as_Register($src1$$reg),
14804 (uint64_t)($src2$$constant));
14805 %}
14806
14807 ins_pipe(ialu_reg_imm);
14808 %}
14809
14810 // Xor Instructions
14811
14812 instruct xorI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
14813 match(Set dst (XorI src1 src2));
14814
14815 format %{ "eorw $dst, $src1, $src2\t# int" %}
14816
14817 ins_cost(INSN_COST);
14818 ins_encode %{
14819 __ eorw(as_Register($dst$$reg),
14820 as_Register($src1$$reg),
14821 as_Register($src2$$reg));
14822 %}
14823
14824 ins_pipe(ialu_reg_reg);
14825 %}
14826
14827 instruct xorI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immILog src2) %{
14828 match(Set dst (XorI src1 src2));
14829
14830 format %{ "eorw $dst, $src1, $src2\t# int" %}
14831
14832 ins_cost(INSN_COST);
14833 ins_encode %{
14834 __ eorw(as_Register($dst$$reg),
14835 as_Register($src1$$reg),
14836 (uint64_t)($src2$$constant));
14837 %}
14838
14839 ins_pipe(ialu_reg_imm);
14840 %}
14841
14842 // Long Logical Instructions
14843 // TODO
14844
14845 instruct andL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2, rFlagsReg cr) %{
14846 match(Set dst (AndL src1 src2));
14847
14848 format %{ "and $dst, $src1, $src2\t# int" %}
14849
14850 ins_cost(INSN_COST);
14851 ins_encode %{
14852 __ andr(as_Register($dst$$reg),
14853 as_Register($src1$$reg),
14854 as_Register($src2$$reg));
14855 %}
14856
14857 ins_pipe(ialu_reg_reg);
14858 %}
14859
14860 instruct andL_reg_imm(iRegLNoSp dst, iRegL src1, immLLog src2, rFlagsReg cr) %{
14861 match(Set dst (AndL src1 src2));
14862
14863 format %{ "and $dst, $src1, $src2\t# int" %}
14864
14865 ins_cost(INSN_COST);
14866 ins_encode %{
14867 __ andr(as_Register($dst$$reg),
14868 as_Register($src1$$reg),
14869 (uint64_t)($src2$$constant));
14870 %}
14871
14872 ins_pipe(ialu_reg_imm);
14873 %}
14874
14875 // Or Instructions
14876
14877 instruct orL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
14878 match(Set dst (OrL src1 src2));
14879
14880 format %{ "orr $dst, $src1, $src2\t# int" %}
14881
14882 ins_cost(INSN_COST);
14883 ins_encode %{
14884 __ orr(as_Register($dst$$reg),
14885 as_Register($src1$$reg),
14886 as_Register($src2$$reg));
14887 %}
14888
14889 ins_pipe(ialu_reg_reg);
14890 %}
14891
14892 instruct orL_reg_imm(iRegLNoSp dst, iRegL src1, immLLog src2) %{
14893 match(Set dst (OrL src1 src2));
14894
14895 format %{ "orr $dst, $src1, $src2\t# int" %}
14896
14897 ins_cost(INSN_COST);
14898 ins_encode %{
14899 __ orr(as_Register($dst$$reg),
14900 as_Register($src1$$reg),
14901 (uint64_t)($src2$$constant));
14902 %}
14903
14904 ins_pipe(ialu_reg_imm);
14905 %}
14906
14907 // Xor Instructions
14908
14909 instruct xorL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
14910 match(Set dst (XorL src1 src2));
14911
14912 format %{ "eor $dst, $src1, $src2\t# int" %}
14913
14914 ins_cost(INSN_COST);
14915 ins_encode %{
14916 __ eor(as_Register($dst$$reg),
14917 as_Register($src1$$reg),
14918 as_Register($src2$$reg));
14919 %}
14920
14921 ins_pipe(ialu_reg_reg);
14922 %}
14923
14924 instruct xorL_reg_imm(iRegLNoSp dst, iRegL src1, immLLog src2) %{
14925 match(Set dst (XorL src1 src2));
14926
14927 ins_cost(INSN_COST);
14928 format %{ "eor $dst, $src1, $src2\t# int" %}
14929
14930 ins_encode %{
14931 __ eor(as_Register($dst$$reg),
14932 as_Register($src1$$reg),
14933 (uint64_t)($src2$$constant));
14934 %}
14935
14936 ins_pipe(ialu_reg_imm);
14937 %}
14938
14939 instruct convI2L_reg_reg(iRegLNoSp dst, iRegIorL2I src)
14940 %{
14941 match(Set dst (ConvI2L src));
14942
14943 ins_cost(INSN_COST);
14944 format %{ "sxtw $dst, $src\t# i2l" %}
14945 ins_encode %{
14946 __ sbfm($dst$$Register, $src$$Register, 0, 31);
14947 %}
14948 ins_pipe(ialu_reg_shift);
14949 %}
14950
14951 // this pattern occurs in bigmath arithmetic
14952 instruct convUI2L_reg_reg(iRegLNoSp dst, iRegIorL2I src, immL_32bits mask)
14953 %{
14954 match(Set dst (AndL (ConvI2L src) mask));
14955
14956 ins_cost(INSN_COST);
14957 format %{ "ubfm $dst, $src, 0, 31\t# ui2l" %}
14958 ins_encode %{
14959 __ ubfm($dst$$Register, $src$$Register, 0, 31);
14960 %}
14961
14962 ins_pipe(ialu_reg_shift);
14963 %}
14964
14965 instruct convL2I_reg(iRegINoSp dst, iRegL src) %{
14966 match(Set dst (ConvL2I src));
14967
14968 ins_cost(INSN_COST);
14969 format %{ "movw $dst, $src \t// l2i" %}
14970
14971 ins_encode %{
14972 __ movw(as_Register($dst$$reg), as_Register($src$$reg));
14973 %}
14974
14975 ins_pipe(ialu_reg);
14976 %}
14977
14978 instruct convI2B(iRegINoSp dst, iRegIorL2I src, rFlagsReg cr)
14979 %{
14980 match(Set dst (Conv2B src));
14981 effect(KILL cr);
14982
14983 format %{
14984 "cmpw $src, zr\n\t"
14985 "cset $dst, ne"
14986 %}
14987
14988 ins_encode %{
14989 __ cmpw(as_Register($src$$reg), zr);
14990 __ cset(as_Register($dst$$reg), Assembler::NE);
14991 %}
14992
14993 ins_pipe(ialu_reg);
14994 %}
14995
14996 instruct convP2B(iRegINoSp dst, iRegP src, rFlagsReg cr)
14997 %{
14998 match(Set dst (Conv2B src));
14999 effect(KILL cr);
15000
15001 format %{
15002 "cmp $src, zr\n\t"
15003 "cset $dst, ne"
15004 %}
15005
15006 ins_encode %{
15007 __ cmp(as_Register($src$$reg), zr);
15008 __ cset(as_Register($dst$$reg), Assembler::NE);
15009 %}
15010
15011 ins_pipe(ialu_reg);
15012 %}
15013
15014 instruct convD2F_reg(vRegF dst, vRegD src) %{
15015 match(Set dst (ConvD2F src));
15016
15017 ins_cost(INSN_COST * 5);
15018 format %{ "fcvtd $dst, $src \t// d2f" %}
15019
15020 ins_encode %{
15021 __ fcvtd(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg));
15022 %}
15023
15024 ins_pipe(fp_d2f);
15025 %}
15026
15027 instruct convF2D_reg(vRegD dst, vRegF src) %{
15028 match(Set dst (ConvF2D src));
15029
15030 ins_cost(INSN_COST * 5);
15031 format %{ "fcvts $dst, $src \t// f2d" %}
15032
15033 ins_encode %{
15034 __ fcvts(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg));
15035 %}
15036
15037 ins_pipe(fp_f2d);
15038 %}
15039
15040 instruct convF2I_reg_reg(iRegINoSp dst, vRegF src) %{
15041 match(Set dst (ConvF2I src));
15042
15043 ins_cost(INSN_COST * 5);
15044 format %{ "fcvtzsw $dst, $src \t// f2i" %}
15045
15046 ins_encode %{
15047 __ fcvtzsw(as_Register($dst$$reg), as_FloatRegister($src$$reg));
15048 %}
15049
15050 ins_pipe(fp_f2i);
15051 %}
15052
15053 instruct convF2L_reg_reg(iRegLNoSp dst, vRegF src) %{
15054 match(Set dst (ConvF2L src));
15055
15056 ins_cost(INSN_COST * 5);
15057 format %{ "fcvtzs $dst, $src \t// f2l" %}
15058
15059 ins_encode %{
15060 __ fcvtzs(as_Register($dst$$reg), as_FloatRegister($src$$reg));
15061 %}
15062
15063 ins_pipe(fp_f2l);
15064 %}
15065
15066 instruct convI2F_reg_reg(vRegF dst, iRegIorL2I src) %{
15067 match(Set dst (ConvI2F src));
15068
15069 ins_cost(INSN_COST * 5);
15070 format %{ "scvtfws $dst, $src \t// i2f" %}
15071
15072 ins_encode %{
15073 __ scvtfws(as_FloatRegister($dst$$reg), as_Register($src$$reg));
15074 %}
15075
15076 ins_pipe(fp_i2f);
15077 %}
15078
15079 instruct convL2F_reg_reg(vRegF dst, iRegL src) %{
15080 match(Set dst (ConvL2F src));
15081
15082 ins_cost(INSN_COST * 5);
15083 format %{ "scvtfs $dst, $src \t// l2f" %}
15084
15085 ins_encode %{
15086 __ scvtfs(as_FloatRegister($dst$$reg), as_Register($src$$reg));
15087 %}
15088
15089 ins_pipe(fp_l2f);
15090 %}
15091
15092 instruct convD2I_reg_reg(iRegINoSp dst, vRegD src) %{
15093 match(Set dst (ConvD2I src));
15094
15095 ins_cost(INSN_COST * 5);
15096 format %{ "fcvtzdw $dst, $src \t// d2i" %}
15097
15098 ins_encode %{
15099 __ fcvtzdw(as_Register($dst$$reg), as_FloatRegister($src$$reg));
15100 %}
15101
15102 ins_pipe(fp_d2i);
15103 %}
15104
15105 instruct convD2L_reg_reg(iRegLNoSp dst, vRegD src) %{
15106 match(Set dst (ConvD2L src));
15107
15108 ins_cost(INSN_COST * 5);
15109 format %{ "fcvtzd $dst, $src \t// d2l" %}
15110
15111 ins_encode %{
15112 __ fcvtzd(as_Register($dst$$reg), as_FloatRegister($src$$reg));
15113 %}
15114
15115 ins_pipe(fp_d2l);
15116 %}
15117
15118 instruct convI2D_reg_reg(vRegD dst, iRegIorL2I src) %{
15119 match(Set dst (ConvI2D src));
15120
15121 ins_cost(INSN_COST * 5);
15122 format %{ "scvtfwd $dst, $src \t// i2d" %}
15123
15124 ins_encode %{
15125 __ scvtfwd(as_FloatRegister($dst$$reg), as_Register($src$$reg));
15126 %}
15127
15128 ins_pipe(fp_i2d);
15129 %}
15130
15131 instruct convL2D_reg_reg(vRegD dst, iRegL src) %{
15132 match(Set dst (ConvL2D src));
15133
15134 ins_cost(INSN_COST * 5);
15135 format %{ "scvtfd $dst, $src \t// l2d" %}
15136
15137 ins_encode %{
15138 __ scvtfd(as_FloatRegister($dst$$reg), as_Register($src$$reg));
15139 %}
15140
15141 ins_pipe(fp_l2d);
15142 %}
15143
15144 instruct round_double_reg(iRegLNoSp dst, vRegD src, vRegD ftmp, rFlagsReg cr)
15145 %{
15146 match(Set dst (RoundD src));
15147 effect(TEMP_DEF dst, TEMP ftmp, KILL cr);
15148 format %{ "java_round_double $dst,$src"%}
15149 ins_encode %{
15150 __ java_round_double($dst$$Register, as_FloatRegister($src$$reg),
15151 as_FloatRegister($ftmp$$reg));
15152 %}
15153 ins_pipe(pipe_slow);
15154 %}
15155
15156 instruct round_float_reg(iRegINoSp dst, vRegF src, vRegF ftmp, rFlagsReg cr)
15157 %{
15158 match(Set dst (RoundF src));
15159 effect(TEMP_DEF dst, TEMP ftmp, KILL cr);
15160 format %{ "java_round_float $dst,$src"%}
15161 ins_encode %{
15162 __ java_round_float($dst$$Register, as_FloatRegister($src$$reg),
15163 as_FloatRegister($ftmp$$reg));
15164 %}
15165 ins_pipe(pipe_slow);
15166 %}
15167
15168 // stack <-> reg and reg <-> reg shuffles with no conversion
15169
15170 instruct MoveF2I_stack_reg(iRegINoSp dst, stackSlotF src) %{
15171
15172 match(Set dst (MoveF2I src));
15173
15174 effect(DEF dst, USE src);
15175
15176 ins_cost(4 * INSN_COST);
15177
15178 format %{ "ldrw $dst, $src\t# MoveF2I_stack_reg" %}
15179
15180 ins_encode %{
15181 __ ldrw($dst$$Register, Address(sp, $src$$disp));
15182 %}
15183
15184 ins_pipe(iload_reg_reg);
15185
15186 %}
15187
15188 instruct MoveI2F_stack_reg(vRegF dst, stackSlotI src) %{
15189
15190 match(Set dst (MoveI2F src));
15191
15192 effect(DEF dst, USE src);
15193
15194 ins_cost(4 * INSN_COST);
15195
15196 format %{ "ldrs $dst, $src\t# MoveI2F_stack_reg" %}
15197
15198 ins_encode %{
15199 __ ldrs(as_FloatRegister($dst$$reg), Address(sp, $src$$disp));
15200 %}
15201
15202 ins_pipe(pipe_class_memory);
15203
15204 %}
15205
15206 instruct MoveD2L_stack_reg(iRegLNoSp dst, stackSlotD src) %{
15207
15208 match(Set dst (MoveD2L src));
15209
15210 effect(DEF dst, USE src);
15211
15212 ins_cost(4 * INSN_COST);
15213
15214 format %{ "ldr $dst, $src\t# MoveD2L_stack_reg" %}
15215
15216 ins_encode %{
15217 __ ldr($dst$$Register, Address(sp, $src$$disp));
15218 %}
15219
15220 ins_pipe(iload_reg_reg);
15221
15222 %}
15223
15224 instruct MoveL2D_stack_reg(vRegD dst, stackSlotL src) %{
15225
15226 match(Set dst (MoveL2D src));
15227
15228 effect(DEF dst, USE src);
15229
15230 ins_cost(4 * INSN_COST);
15231
15232 format %{ "ldrd $dst, $src\t# MoveL2D_stack_reg" %}
15233
15234 ins_encode %{
15235 __ ldrd(as_FloatRegister($dst$$reg), Address(sp, $src$$disp));
15236 %}
15237
15238 ins_pipe(pipe_class_memory);
15239
15240 %}
15241
15242 instruct MoveF2I_reg_stack(stackSlotI dst, vRegF src) %{
15243
15244 match(Set dst (MoveF2I src));
15245
15246 effect(DEF dst, USE src);
15247
15248 ins_cost(INSN_COST);
15249
15250 format %{ "strs $src, $dst\t# MoveF2I_reg_stack" %}
15251
15252 ins_encode %{
15253 __ strs(as_FloatRegister($src$$reg), Address(sp, $dst$$disp));
15254 %}
15255
15256 ins_pipe(pipe_class_memory);
15257
15258 %}
15259
15260 instruct MoveI2F_reg_stack(stackSlotF dst, iRegI src) %{
15261
15262 match(Set dst (MoveI2F src));
15263
15264 effect(DEF dst, USE src);
15265
15266 ins_cost(INSN_COST);
15267
15268 format %{ "strw $src, $dst\t# MoveI2F_reg_stack" %}
15269
15270 ins_encode %{
15271 __ strw($src$$Register, Address(sp, $dst$$disp));
15272 %}
15273
15274 ins_pipe(istore_reg_reg);
15275
15276 %}
15277
15278 instruct MoveD2L_reg_stack(stackSlotL dst, vRegD src) %{
15279
15280 match(Set dst (MoveD2L src));
15281
15282 effect(DEF dst, USE src);
15283
15284 ins_cost(INSN_COST);
15285
15286 format %{ "strd $dst, $src\t# MoveD2L_reg_stack" %}
15287
15288 ins_encode %{
15289 __ strd(as_FloatRegister($src$$reg), Address(sp, $dst$$disp));
15290 %}
15291
15292 ins_pipe(pipe_class_memory);
15293
15294 %}
15295
15296 instruct MoveL2D_reg_stack(stackSlotD dst, iRegL src) %{
15297
15298 match(Set dst (MoveL2D src));
15299
15300 effect(DEF dst, USE src);
15301
15302 ins_cost(INSN_COST);
15303
15304 format %{ "str $src, $dst\t# MoveL2D_reg_stack" %}
15305
15306 ins_encode %{
15307 __ str($src$$Register, Address(sp, $dst$$disp));
15308 %}
15309
15310 ins_pipe(istore_reg_reg);
15311
15312 %}
15313
15314 instruct MoveF2I_reg_reg(iRegINoSp dst, vRegF src) %{
15315
15316 match(Set dst (MoveF2I src));
15317
15318 effect(DEF dst, USE src);
15319
15320 ins_cost(INSN_COST);
15321
15322 format %{ "fmovs $dst, $src\t# MoveF2I_reg_reg" %}
15323
15324 ins_encode %{
15325 __ fmovs($dst$$Register, as_FloatRegister($src$$reg));
15326 %}
15327
15328 ins_pipe(fp_f2i);
15329
15330 %}
15331
15332 instruct MoveI2F_reg_reg(vRegF dst, iRegI src) %{
15333
15334 match(Set dst (MoveI2F src));
15335
15336 effect(DEF dst, USE src);
15337
15338 ins_cost(INSN_COST);
15339
15340 format %{ "fmovs $dst, $src\t# MoveI2F_reg_reg" %}
15341
15342 ins_encode %{
15343 __ fmovs(as_FloatRegister($dst$$reg), $src$$Register);
15344 %}
15345
15346 ins_pipe(fp_i2f);
15347
15348 %}
15349
15350 instruct MoveD2L_reg_reg(iRegLNoSp dst, vRegD src) %{
15351
15352 match(Set dst (MoveD2L src));
15353
15354 effect(DEF dst, USE src);
15355
15356 ins_cost(INSN_COST);
15357
15358 format %{ "fmovd $dst, $src\t# MoveD2L_reg_reg" %}
15359
15360 ins_encode %{
15361 __ fmovd($dst$$Register, as_FloatRegister($src$$reg));
15362 %}
15363
15364 ins_pipe(fp_d2l);
15365
15366 %}
15367
15368 instruct MoveL2D_reg_reg(vRegD dst, iRegL src) %{
15369
15370 match(Set dst (MoveL2D src));
15371
15372 effect(DEF dst, USE src);
15373
15374 ins_cost(INSN_COST);
15375
15376 format %{ "fmovd $dst, $src\t# MoveL2D_reg_reg" %}
15377
15378 ins_encode %{
15379 __ fmovd(as_FloatRegister($dst$$reg), $src$$Register);
15380 %}
15381
15382 ins_pipe(fp_l2d);
15383
15384 %}
15385
15386 // ============================================================================
15387 // clearing of an array
15388
15389 instruct clearArray_reg_reg(iRegL_R11 cnt, iRegP_R10 base, Universe dummy, rFlagsReg cr)
15390 %{
15391 match(Set dummy (ClearArray cnt base));
15392 effect(USE_KILL cnt, USE_KILL base, KILL cr);
15393
15394 ins_cost(4 * INSN_COST);
15395 format %{ "ClearArray $cnt, $base" %}
15396
15397 ins_encode %{
15398 address tpc = __ zero_words($base$$Register, $cnt$$Register);
15399 if (tpc == NULL) {
15400 ciEnv::current()->record_failure("CodeCache is full");
15401 return;
15402 }
15403 %}
15404
15405 ins_pipe(pipe_class_memory);
15406 %}
15407
15408 instruct clearArray_imm_reg(immL cnt, iRegP_R10 base, iRegL_R11 temp, Universe dummy, rFlagsReg cr)
15409 %{
15410 predicate((uint64_t)n->in(2)->get_long()
15411 < (uint64_t)(BlockZeroingLowLimit >> LogBytesPerWord));
15412 match(Set dummy (ClearArray cnt base));
15413 effect(TEMP temp, USE_KILL base, KILL cr);
15414
15415 ins_cost(4 * INSN_COST);
15416 format %{ "ClearArray $cnt, $base" %}
15417
15418 ins_encode %{
15419 __ zero_words($base$$Register, (uint64_t)$cnt$$constant);
15420 %}
15421
15422 ins_pipe(pipe_class_memory);
15423 %}
15424
15425 // ============================================================================
15426 // Overflow Math Instructions
15427
15428 instruct overflowAddI_reg_reg(rFlagsReg cr, iRegIorL2I op1, iRegIorL2I op2)
15429 %{
15430 match(Set cr (OverflowAddI op1 op2));
15431
15432 format %{ "cmnw $op1, $op2\t# overflow check int" %}
15433 ins_cost(INSN_COST);
15434 ins_encode %{
15435 __ cmnw($op1$$Register, $op2$$Register);
15436 %}
15437
15438 ins_pipe(icmp_reg_reg);
15439 %}
15440
15441 instruct overflowAddI_reg_imm(rFlagsReg cr, iRegIorL2I op1, immIAddSub op2)
15442 %{
15443 match(Set cr (OverflowAddI op1 op2));
15444
15445 format %{ "cmnw $op1, $op2\t# overflow check int" %}
15446 ins_cost(INSN_COST);
15447 ins_encode %{
15448 __ cmnw($op1$$Register, $op2$$constant);
15449 %}
15450
15451 ins_pipe(icmp_reg_imm);
15452 %}
15453
15454 instruct overflowAddL_reg_reg(rFlagsReg cr, iRegL op1, iRegL op2)
15455 %{
15456 match(Set cr (OverflowAddL op1 op2));
15457
15458 format %{ "cmn $op1, $op2\t# overflow check long" %}
15459 ins_cost(INSN_COST);
15460 ins_encode %{
15461 __ cmn($op1$$Register, $op2$$Register);
15462 %}
15463
15464 ins_pipe(icmp_reg_reg);
15465 %}
15466
15467 instruct overflowAddL_reg_imm(rFlagsReg cr, iRegL op1, immLAddSub op2)
15468 %{
15469 match(Set cr (OverflowAddL op1 op2));
15470
15471 format %{ "cmn $op1, $op2\t# overflow check long" %}
15472 ins_cost(INSN_COST);
15473 ins_encode %{
15474 __ cmn($op1$$Register, $op2$$constant);
15475 %}
15476
15477 ins_pipe(icmp_reg_imm);
15478 %}
15479
15480 instruct overflowSubI_reg_reg(rFlagsReg cr, iRegIorL2I op1, iRegIorL2I op2)
15481 %{
15482 match(Set cr (OverflowSubI op1 op2));
15483
15484 format %{ "cmpw $op1, $op2\t# overflow check int" %}
15485 ins_cost(INSN_COST);
15486 ins_encode %{
15487 __ cmpw($op1$$Register, $op2$$Register);
15488 %}
15489
15490 ins_pipe(icmp_reg_reg);
15491 %}
15492
15493 instruct overflowSubI_reg_imm(rFlagsReg cr, iRegIorL2I op1, immIAddSub op2)
15494 %{
15495 match(Set cr (OverflowSubI op1 op2));
15496
15497 format %{ "cmpw $op1, $op2\t# overflow check int" %}
15498 ins_cost(INSN_COST);
15499 ins_encode %{
15500 __ cmpw($op1$$Register, $op2$$constant);
15501 %}
15502
15503 ins_pipe(icmp_reg_imm);
15504 %}
15505
15506 instruct overflowSubL_reg_reg(rFlagsReg cr, iRegL op1, iRegL op2)
15507 %{
15508 match(Set cr (OverflowSubL op1 op2));
15509
15510 format %{ "cmp $op1, $op2\t# overflow check long" %}
15511 ins_cost(INSN_COST);
15512 ins_encode %{
15513 __ cmp($op1$$Register, $op2$$Register);
15514 %}
15515
15516 ins_pipe(icmp_reg_reg);
15517 %}
15518
15519 instruct overflowSubL_reg_imm(rFlagsReg cr, iRegL op1, immLAddSub op2)
15520 %{
15521 match(Set cr (OverflowSubL op1 op2));
15522
15523 format %{ "cmp $op1, $op2\t# overflow check long" %}
15524 ins_cost(INSN_COST);
15525 ins_encode %{
15526 __ subs(zr, $op1$$Register, $op2$$constant);
15527 %}
15528
15529 ins_pipe(icmp_reg_imm);
15530 %}
15531
15532 instruct overflowNegI_reg(rFlagsReg cr, immI0 zero, iRegIorL2I op1)
15533 %{
15534 match(Set cr (OverflowSubI zero op1));
15535
15536 format %{ "cmpw zr, $op1\t# overflow check int" %}
15537 ins_cost(INSN_COST);
15538 ins_encode %{
15539 __ cmpw(zr, $op1$$Register);
15540 %}
15541
15542 ins_pipe(icmp_reg_imm);
15543 %}
15544
15545 instruct overflowNegL_reg(rFlagsReg cr, immI0 zero, iRegL op1)
15546 %{
15547 match(Set cr (OverflowSubL zero op1));
15548
15549 format %{ "cmp zr, $op1\t# overflow check long" %}
15550 ins_cost(INSN_COST);
15551 ins_encode %{
15552 __ cmp(zr, $op1$$Register);
15553 %}
15554
15555 ins_pipe(icmp_reg_imm);
15556 %}
15557
15558 instruct overflowMulI_reg(rFlagsReg cr, iRegIorL2I op1, iRegIorL2I op2)
15559 %{
15560 match(Set cr (OverflowMulI op1 op2));
15561
15562 format %{ "smull rscratch1, $op1, $op2\t# overflow check int\n\t"
15563 "cmp rscratch1, rscratch1, sxtw\n\t"
15564 "movw rscratch1, #0x80000000\n\t"
15565 "cselw rscratch1, rscratch1, zr, NE\n\t"
15566 "cmpw rscratch1, #1" %}
15567 ins_cost(5 * INSN_COST);
15568 ins_encode %{
15569 __ smull(rscratch1, $op1$$Register, $op2$$Register);
15570 __ subs(zr, rscratch1, rscratch1, ext::sxtw); // NE => overflow
15571 __ movw(rscratch1, 0x80000000); // Develop 0 (EQ),
15572 __ cselw(rscratch1, rscratch1, zr, Assembler::NE); // or 0x80000000 (NE)
15573 __ cmpw(rscratch1, 1); // 0x80000000 - 1 => VS
15574 %}
15575
15576 ins_pipe(pipe_slow);
15577 %}
15578
15579 instruct overflowMulI_reg_branch(cmpOp cmp, iRegIorL2I op1, iRegIorL2I op2, label labl, rFlagsReg cr)
15580 %{
15581 match(If cmp (OverflowMulI op1 op2));
15582 predicate(n->in(1)->as_Bool()->_test._test == BoolTest::overflow
15583 || n->in(1)->as_Bool()->_test._test == BoolTest::no_overflow);
15584 effect(USE labl, KILL cr);
15585
15586 format %{ "smull rscratch1, $op1, $op2\t# overflow check int\n\t"
15587 "cmp rscratch1, rscratch1, sxtw\n\t"
15588 "b$cmp $labl" %}
15589 ins_cost(3 * INSN_COST); // Branch is rare so treat as INSN_COST
15590 ins_encode %{
15591 Label* L = $labl$$label;
15592 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15593 __ smull(rscratch1, $op1$$Register, $op2$$Register);
15594 __ subs(zr, rscratch1, rscratch1, ext::sxtw); // NE => overflow
15595 __ br(cond == Assembler::VS ? Assembler::NE : Assembler::EQ, *L);
15596 %}
15597
15598 ins_pipe(pipe_serial);
15599 %}
15600
15601 instruct overflowMulL_reg(rFlagsReg cr, iRegL op1, iRegL op2)
15602 %{
15603 match(Set cr (OverflowMulL op1 op2));
15604
15605 format %{ "mul rscratch1, $op1, $op2\t#overflow check long\n\t"
15606 "smulh rscratch2, $op1, $op2\n\t"
15607 "cmp rscratch2, rscratch1, ASR #63\n\t"
15608 "movw rscratch1, #0x80000000\n\t"
15609 "cselw rscratch1, rscratch1, zr, NE\n\t"
15610 "cmpw rscratch1, #1" %}
15611 ins_cost(6 * INSN_COST);
15612 ins_encode %{
15613 __ mul(rscratch1, $op1$$Register, $op2$$Register); // Result bits 0..63
15614 __ smulh(rscratch2, $op1$$Register, $op2$$Register); // Result bits 64..127
15615 __ cmp(rscratch2, rscratch1, Assembler::ASR, 63); // Top is pure sign ext
15616 __ movw(rscratch1, 0x80000000); // Develop 0 (EQ),
15617 __ cselw(rscratch1, rscratch1, zr, Assembler::NE); // or 0x80000000 (NE)
15618 __ cmpw(rscratch1, 1); // 0x80000000 - 1 => VS
15619 %}
15620
15621 ins_pipe(pipe_slow);
15622 %}
15623
15624 instruct overflowMulL_reg_branch(cmpOp cmp, iRegL op1, iRegL op2, label labl, rFlagsReg cr)
15625 %{
15626 match(If cmp (OverflowMulL op1 op2));
15627 predicate(n->in(1)->as_Bool()->_test._test == BoolTest::overflow
15628 || n->in(1)->as_Bool()->_test._test == BoolTest::no_overflow);
15629 effect(USE labl, KILL cr);
15630
15631 format %{ "mul rscratch1, $op1, $op2\t#overflow check long\n\t"
15632 "smulh rscratch2, $op1, $op2\n\t"
15633 "cmp rscratch2, rscratch1, ASR #63\n\t"
15634 "b$cmp $labl" %}
15635 ins_cost(4 * INSN_COST); // Branch is rare so treat as INSN_COST
15636 ins_encode %{
15637 Label* L = $labl$$label;
15638 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15639 __ mul(rscratch1, $op1$$Register, $op2$$Register); // Result bits 0..63
15640 __ smulh(rscratch2, $op1$$Register, $op2$$Register); // Result bits 64..127
15641 __ cmp(rscratch2, rscratch1, Assembler::ASR, 63); // Top is pure sign ext
15642 __ br(cond == Assembler::VS ? Assembler::NE : Assembler::EQ, *L);
15643 %}
15644
15645 ins_pipe(pipe_serial);
15646 %}
15647
15648 // ============================================================================
15649 // Compare Instructions
15650
15651 instruct compI_reg_reg(rFlagsReg cr, iRegI op1, iRegI op2)
15652 %{
15653 match(Set cr (CmpI op1 op2));
15654
15655 effect(DEF cr, USE op1, USE op2);
15656
15657 ins_cost(INSN_COST);
15658 format %{ "cmpw $op1, $op2" %}
15659
15660 ins_encode(aarch64_enc_cmpw(op1, op2));
15661
15662 ins_pipe(icmp_reg_reg);
15663 %}
15664
15665 instruct compI_reg_immI0(rFlagsReg cr, iRegI op1, immI0 zero)
15666 %{
15667 match(Set cr (CmpI op1 zero));
15668
15669 effect(DEF cr, USE op1);
15670
15671 ins_cost(INSN_COST);
15672 format %{ "cmpw $op1, 0" %}
15673
15674 ins_encode(aarch64_enc_cmpw_imm_addsub(op1, zero));
15675
15676 ins_pipe(icmp_reg_imm);
15677 %}
15678
15679 instruct compI_reg_immIAddSub(rFlagsReg cr, iRegI op1, immIAddSub op2)
15680 %{
15681 match(Set cr (CmpI op1 op2));
15682
15683 effect(DEF cr, USE op1);
15684
15685 ins_cost(INSN_COST);
15686 format %{ "cmpw $op1, $op2" %}
15687
15688 ins_encode(aarch64_enc_cmpw_imm_addsub(op1, op2));
15689
15690 ins_pipe(icmp_reg_imm);
15691 %}
15692
15693 instruct compI_reg_immI(rFlagsReg cr, iRegI op1, immI op2)
15694 %{
15695 match(Set cr (CmpI op1 op2));
15696
15697 effect(DEF cr, USE op1);
15698
15699 ins_cost(INSN_COST * 2);
15700 format %{ "cmpw $op1, $op2" %}
15701
15702 ins_encode(aarch64_enc_cmpw_imm(op1, op2));
15703
15704 ins_pipe(icmp_reg_imm);
15705 %}
15706
15707 // Unsigned compare Instructions; really, same as signed compare
15708 // except it should only be used to feed an If or a CMovI which takes a
15709 // cmpOpU.
15710
15711 instruct compU_reg_reg(rFlagsRegU cr, iRegI op1, iRegI op2)
15712 %{
15713 match(Set cr (CmpU op1 op2));
15714
15715 effect(DEF cr, USE op1, USE op2);
15716
15717 ins_cost(INSN_COST);
15718 format %{ "cmpw $op1, $op2\t# unsigned" %}
15719
15720 ins_encode(aarch64_enc_cmpw(op1, op2));
15721
15722 ins_pipe(icmp_reg_reg);
15723 %}
15724
15725 instruct compU_reg_immI0(rFlagsRegU cr, iRegI op1, immI0 zero)
15726 %{
15727 match(Set cr (CmpU op1 zero));
15728
15729 effect(DEF cr, USE op1);
15730
15731 ins_cost(INSN_COST);
15732 format %{ "cmpw $op1, #0\t# unsigned" %}
15733
15734 ins_encode(aarch64_enc_cmpw_imm_addsub(op1, zero));
15735
15736 ins_pipe(icmp_reg_imm);
15737 %}
15738
15739 instruct compU_reg_immIAddSub(rFlagsRegU cr, iRegI op1, immIAddSub op2)
15740 %{
15741 match(Set cr (CmpU op1 op2));
15742
15743 effect(DEF cr, USE op1);
15744
15745 ins_cost(INSN_COST);
15746 format %{ "cmpw $op1, $op2\t# unsigned" %}
15747
15748 ins_encode(aarch64_enc_cmpw_imm_addsub(op1, op2));
15749
15750 ins_pipe(icmp_reg_imm);
15751 %}
15752
15753 instruct compU_reg_immI(rFlagsRegU cr, iRegI op1, immI op2)
15754 %{
15755 match(Set cr (CmpU op1 op2));
15756
15757 effect(DEF cr, USE op1);
15758
15759 ins_cost(INSN_COST * 2);
15760 format %{ "cmpw $op1, $op2\t# unsigned" %}
15761
15762 ins_encode(aarch64_enc_cmpw_imm(op1, op2));
15763
15764 ins_pipe(icmp_reg_imm);
15765 %}
15766
15767 instruct compL_reg_reg(rFlagsReg cr, iRegL op1, iRegL op2)
15768 %{
15769 match(Set cr (CmpL op1 op2));
15770
15771 effect(DEF cr, USE op1, USE op2);
15772
15773 ins_cost(INSN_COST);
15774 format %{ "cmp $op1, $op2" %}
15775
15776 ins_encode(aarch64_enc_cmp(op1, op2));
15777
15778 ins_pipe(icmp_reg_reg);
15779 %}
15780
15781 instruct compL_reg_immL0(rFlagsReg cr, iRegL op1, immL0 zero)
15782 %{
15783 match(Set cr (CmpL op1 zero));
15784
15785 effect(DEF cr, USE op1);
15786
15787 ins_cost(INSN_COST);
15788 format %{ "tst $op1" %}
15789
15790 ins_encode(aarch64_enc_cmp_imm_addsub(op1, zero));
15791
15792 ins_pipe(icmp_reg_imm);
15793 %}
15794
15795 instruct compL_reg_immLAddSub(rFlagsReg cr, iRegL op1, immLAddSub op2)
15796 %{
15797 match(Set cr (CmpL op1 op2));
15798
15799 effect(DEF cr, USE op1);
15800
15801 ins_cost(INSN_COST);
15802 format %{ "cmp $op1, $op2" %}
15803
15804 ins_encode(aarch64_enc_cmp_imm_addsub(op1, op2));
15805
15806 ins_pipe(icmp_reg_imm);
15807 %}
15808
15809 instruct compL_reg_immL(rFlagsReg cr, iRegL op1, immL op2)
15810 %{
15811 match(Set cr (CmpL op1 op2));
15812
15813 effect(DEF cr, USE op1);
15814
15815 ins_cost(INSN_COST * 2);
15816 format %{ "cmp $op1, $op2" %}
15817
15818 ins_encode(aarch64_enc_cmp_imm(op1, op2));
15819
15820 ins_pipe(icmp_reg_imm);
15821 %}
15822
15823 instruct compUL_reg_reg(rFlagsRegU cr, iRegL op1, iRegL op2)
15824 %{
15825 match(Set cr (CmpUL op1 op2));
15826
15827 effect(DEF cr, USE op1, USE op2);
15828
15829 ins_cost(INSN_COST);
15830 format %{ "cmp $op1, $op2" %}
15831
15832 ins_encode(aarch64_enc_cmp(op1, op2));
15833
15834 ins_pipe(icmp_reg_reg);
15835 %}
15836
15837 instruct compUL_reg_immL0(rFlagsRegU cr, iRegL op1, immL0 zero)
15838 %{
15839 match(Set cr (CmpUL op1 zero));
15840
15841 effect(DEF cr, USE op1);
15842
15843 ins_cost(INSN_COST);
15844 format %{ "tst $op1" %}
15845
15846 ins_encode(aarch64_enc_cmp_imm_addsub(op1, zero));
15847
15848 ins_pipe(icmp_reg_imm);
15849 %}
15850
15851 instruct compUL_reg_immLAddSub(rFlagsRegU cr, iRegL op1, immLAddSub op2)
15852 %{
15853 match(Set cr (CmpUL op1 op2));
15854
15855 effect(DEF cr, USE op1);
15856
15857 ins_cost(INSN_COST);
15858 format %{ "cmp $op1, $op2" %}
15859
15860 ins_encode(aarch64_enc_cmp_imm_addsub(op1, op2));
15861
15862 ins_pipe(icmp_reg_imm);
15863 %}
15864
15865 instruct compUL_reg_immL(rFlagsRegU cr, iRegL op1, immL op2)
15866 %{
15867 match(Set cr (CmpUL op1 op2));
15868
15869 effect(DEF cr, USE op1);
15870
15871 ins_cost(INSN_COST * 2);
15872 format %{ "cmp $op1, $op2" %}
15873
15874 ins_encode(aarch64_enc_cmp_imm(op1, op2));
15875
15876 ins_pipe(icmp_reg_imm);
15877 %}
15878
15879 instruct compP_reg_reg(rFlagsRegU cr, iRegP op1, iRegP op2)
15880 %{
15881 match(Set cr (CmpP op1 op2));
15882
15883 effect(DEF cr, USE op1, USE op2);
15884
15885 ins_cost(INSN_COST);
15886 format %{ "cmp $op1, $op2\t // ptr" %}
15887
15888 ins_encode(aarch64_enc_cmpp(op1, op2));
15889
15890 ins_pipe(icmp_reg_reg);
15891 %}
15892
15893 instruct compN_reg_reg(rFlagsRegU cr, iRegN op1, iRegN op2)
15894 %{
15895 match(Set cr (CmpN op1 op2));
15896
15897 effect(DEF cr, USE op1, USE op2);
15898
15899 ins_cost(INSN_COST);
15900 format %{ "cmp $op1, $op2\t // compressed ptr" %}
15901
15902 ins_encode(aarch64_enc_cmpn(op1, op2));
15903
15904 ins_pipe(icmp_reg_reg);
15905 %}
15906
15907 instruct testP_reg(rFlagsRegU cr, iRegP op1, immP0 zero)
15908 %{
15909 match(Set cr (CmpP op1 zero));
15910
15911 effect(DEF cr, USE op1, USE zero);
15912
15913 ins_cost(INSN_COST);
15914 format %{ "cmp $op1, 0\t // ptr" %}
15915
15916 ins_encode(aarch64_enc_testp(op1));
15917
15918 ins_pipe(icmp_reg_imm);
15919 %}
15920
15921 instruct testN_reg(rFlagsRegU cr, iRegN op1, immN0 zero)
15922 %{
15923 match(Set cr (CmpN op1 zero));
15924
15925 effect(DEF cr, USE op1, USE zero);
15926
15927 ins_cost(INSN_COST);
15928 format %{ "cmp $op1, 0\t // compressed ptr" %}
15929
15930 ins_encode(aarch64_enc_testn(op1));
15931
15932 ins_pipe(icmp_reg_imm);
15933 %}
15934
15935 // FP comparisons
15936 //
15937 // n.b. CmpF/CmpD set a normal flags reg which then gets compared
15938 // using normal cmpOp. See declaration of rFlagsReg for details.
15939
15940 instruct compF_reg_reg(rFlagsReg cr, vRegF src1, vRegF src2)
15941 %{
15942 match(Set cr (CmpF src1 src2));
15943
15944 ins_cost(3 * INSN_COST);
15945 format %{ "fcmps $src1, $src2" %}
15946
15947 ins_encode %{
15948 __ fcmps(as_FloatRegister($src1$$reg), as_FloatRegister($src2$$reg));
15949 %}
15950
15951 ins_pipe(pipe_class_compare);
15952 %}
15953
15954 instruct compF_reg_zero(rFlagsReg cr, vRegF src1, immF0 src2)
15955 %{
15956 match(Set cr (CmpF src1 src2));
15957
15958 ins_cost(3 * INSN_COST);
15959 format %{ "fcmps $src1, 0.0" %}
15960
15961 ins_encode %{
15962 __ fcmps(as_FloatRegister($src1$$reg), 0.0);
15963 %}
15964
15965 ins_pipe(pipe_class_compare);
15966 %}
15967 // FROM HERE
15968
15969 instruct compD_reg_reg(rFlagsReg cr, vRegD src1, vRegD src2)
15970 %{
15971 match(Set cr (CmpD src1 src2));
15972
15973 ins_cost(3 * INSN_COST);
15974 format %{ "fcmpd $src1, $src2" %}
15975
15976 ins_encode %{
15977 __ fcmpd(as_FloatRegister($src1$$reg), as_FloatRegister($src2$$reg));
15978 %}
15979
15980 ins_pipe(pipe_class_compare);
15981 %}
15982
15983 instruct compD_reg_zero(rFlagsReg cr, vRegD src1, immD0 src2)
15984 %{
15985 match(Set cr (CmpD src1 src2));
15986
15987 ins_cost(3 * INSN_COST);
15988 format %{ "fcmpd $src1, 0.0" %}
15989
15990 ins_encode %{
15991 __ fcmpd(as_FloatRegister($src1$$reg), 0.0);
15992 %}
15993
15994 ins_pipe(pipe_class_compare);
15995 %}
15996
15997 instruct compF3_reg_reg(iRegINoSp dst, vRegF src1, vRegF src2, rFlagsReg cr)
15998 %{
15999 match(Set dst (CmpF3 src1 src2));
16000 effect(KILL cr);
16001
16002 ins_cost(5 * INSN_COST);
16003 format %{ "fcmps $src1, $src2\n\t"
16004 "csinvw($dst, zr, zr, eq\n\t"
16005 "csnegw($dst, $dst, $dst, lt)"
16006 %}
16007
16008 ins_encode %{
16009 Label done;
16010 FloatRegister s1 = as_FloatRegister($src1$$reg);
16011 FloatRegister s2 = as_FloatRegister($src2$$reg);
16012 Register d = as_Register($dst$$reg);
16013 __ fcmps(s1, s2);
16014 // installs 0 if EQ else -1
16015 __ csinvw(d, zr, zr, Assembler::EQ);
16016 // keeps -1 if less or unordered else installs 1
16017 __ csnegw(d, d, d, Assembler::LT);
16018 __ bind(done);
16019 %}
16020
16021 ins_pipe(pipe_class_default);
16022
16023 %}
16024
16025 instruct compD3_reg_reg(iRegINoSp dst, vRegD src1, vRegD src2, rFlagsReg cr)
16026 %{
16027 match(Set dst (CmpD3 src1 src2));
16028 effect(KILL cr);
16029
16030 ins_cost(5 * INSN_COST);
16031 format %{ "fcmpd $src1, $src2\n\t"
16032 "csinvw($dst, zr, zr, eq\n\t"
16033 "csnegw($dst, $dst, $dst, lt)"
16034 %}
16035
16036 ins_encode %{
16037 Label done;
16038 FloatRegister s1 = as_FloatRegister($src1$$reg);
16039 FloatRegister s2 = as_FloatRegister($src2$$reg);
16040 Register d = as_Register($dst$$reg);
16041 __ fcmpd(s1, s2);
16042 // installs 0 if EQ else -1
16043 __ csinvw(d, zr, zr, Assembler::EQ);
16044 // keeps -1 if less or unordered else installs 1
16045 __ csnegw(d, d, d, Assembler::LT);
16046 __ bind(done);
16047 %}
16048 ins_pipe(pipe_class_default);
16049
16050 %}
16051
16052 instruct compF3_reg_immF0(iRegINoSp dst, vRegF src1, immF0 zero, rFlagsReg cr)
16053 %{
16054 match(Set dst (CmpF3 src1 zero));
16055 effect(KILL cr);
16056
16057 ins_cost(5 * INSN_COST);
16058 format %{ "fcmps $src1, 0.0\n\t"
16059 "csinvw($dst, zr, zr, eq\n\t"
16060 "csnegw($dst, $dst, $dst, lt)"
16061 %}
16062
16063 ins_encode %{
16064 Label done;
16065 FloatRegister s1 = as_FloatRegister($src1$$reg);
16066 Register d = as_Register($dst$$reg);
16067 __ fcmps(s1, 0.0);
16068 // installs 0 if EQ else -1
16069 __ csinvw(d, zr, zr, Assembler::EQ);
16070 // keeps -1 if less or unordered else installs 1
16071 __ csnegw(d, d, d, Assembler::LT);
16072 __ bind(done);
16073 %}
16074
16075 ins_pipe(pipe_class_default);
16076
16077 %}
16078
16079 instruct compD3_reg_immD0(iRegINoSp dst, vRegD src1, immD0 zero, rFlagsReg cr)
16080 %{
16081 match(Set dst (CmpD3 src1 zero));
16082 effect(KILL cr);
16083
16084 ins_cost(5 * INSN_COST);
16085 format %{ "fcmpd $src1, 0.0\n\t"
16086 "csinvw($dst, zr, zr, eq\n\t"
16087 "csnegw($dst, $dst, $dst, lt)"
16088 %}
16089
16090 ins_encode %{
16091 Label done;
16092 FloatRegister s1 = as_FloatRegister($src1$$reg);
16093 Register d = as_Register($dst$$reg);
16094 __ fcmpd(s1, 0.0);
16095 // installs 0 if EQ else -1
16096 __ csinvw(d, zr, zr, Assembler::EQ);
16097 // keeps -1 if less or unordered else installs 1
16098 __ csnegw(d, d, d, Assembler::LT);
16099 __ bind(done);
16100 %}
16101 ins_pipe(pipe_class_default);
16102
16103 %}
16104
16105 instruct cmpLTMask_reg_reg(iRegINoSp dst, iRegIorL2I p, iRegIorL2I q, rFlagsReg cr)
16106 %{
16107 match(Set dst (CmpLTMask p q));
16108 effect(KILL cr);
16109
16110 ins_cost(3 * INSN_COST);
16111
16112 format %{ "cmpw $p, $q\t# cmpLTMask\n\t"
16113 "csetw $dst, lt\n\t"
16114 "subw $dst, zr, $dst"
16115 %}
16116
16117 ins_encode %{
16118 __ cmpw(as_Register($p$$reg), as_Register($q$$reg));
16119 __ csetw(as_Register($dst$$reg), Assembler::LT);
16120 __ subw(as_Register($dst$$reg), zr, as_Register($dst$$reg));
16121 %}
16122
16123 ins_pipe(ialu_reg_reg);
16124 %}
16125
16126 instruct cmpLTMask_reg_zero(iRegINoSp dst, iRegIorL2I src, immI0 zero, rFlagsReg cr)
16127 %{
16128 match(Set dst (CmpLTMask src zero));
16129 effect(KILL cr);
16130
16131 ins_cost(INSN_COST);
16132
16133 format %{ "asrw $dst, $src, #31\t# cmpLTMask0" %}
16134
16135 ins_encode %{
16136 __ asrw(as_Register($dst$$reg), as_Register($src$$reg), 31);
16137 %}
16138
16139 ins_pipe(ialu_reg_shift);
16140 %}
16141
16142 // ============================================================================
16143 // Max and Min
16144
16145 instruct cmovI_reg_reg_lt(iRegINoSp dst, iRegI src1, iRegI src2, rFlagsReg cr)
16146 %{
16147 effect( DEF dst, USE src1, USE src2, USE cr );
16148
16149 ins_cost(INSN_COST * 2);
16150 format %{ "cselw $dst, $src1, $src2 lt\t" %}
16151
16152 ins_encode %{
16153 __ cselw(as_Register($dst$$reg),
16154 as_Register($src1$$reg),
16155 as_Register($src2$$reg),
16156 Assembler::LT);
16157 %}
16158
16159 ins_pipe(icond_reg_reg);
16160 %}
16161
16162 instruct minI_rReg(iRegINoSp dst, iRegI src1, iRegI src2)
16163 %{
16164 match(Set dst (MinI src1 src2));
16165 ins_cost(INSN_COST * 3);
16166
16167 expand %{
16168 rFlagsReg cr;
16169 compI_reg_reg(cr, src1, src2);
16170 cmovI_reg_reg_lt(dst, src1, src2, cr);
16171 %}
16172
16173 %}
16174 // FROM HERE
16175
16176 instruct cmovI_reg_reg_gt(iRegINoSp dst, iRegI src1, iRegI src2, rFlagsReg cr)
16177 %{
16178 effect( DEF dst, USE src1, USE src2, USE cr );
16179
16180 ins_cost(INSN_COST * 2);
16181 format %{ "cselw $dst, $src1, $src2 gt\t" %}
16182
16183 ins_encode %{
16184 __ cselw(as_Register($dst$$reg),
16185 as_Register($src1$$reg),
16186 as_Register($src2$$reg),
16187 Assembler::GT);
16188 %}
16189
16190 ins_pipe(icond_reg_reg);
16191 %}
16192
16193 instruct maxI_rReg(iRegINoSp dst, iRegI src1, iRegI src2)
16194 %{
16195 match(Set dst (MaxI src1 src2));
16196 ins_cost(INSN_COST * 3);
16197 expand %{
16198 rFlagsReg cr;
16199 compI_reg_reg(cr, src1, src2);
16200 cmovI_reg_reg_gt(dst, src1, src2, cr);
16201 %}
16202 %}
16203
16204 // ============================================================================
16205 // Branch Instructions
16206
16207 // Direct Branch.
16208 instruct branch(label lbl)
16209 %{
16210 match(Goto);
16211
16212 effect(USE lbl);
16213
16214 ins_cost(BRANCH_COST);
16215 format %{ "b $lbl" %}
16216
16217 ins_encode(aarch64_enc_b(lbl));
16218
16219 ins_pipe(pipe_branch);
16220 %}
16221
16222 // Conditional Near Branch
16223 instruct branchCon(cmpOp cmp, rFlagsReg cr, label lbl)
16224 %{
16225 // Same match rule as `branchConFar'.
16226 match(If cmp cr);
16227
16228 effect(USE lbl);
16229
16230 ins_cost(BRANCH_COST);
16231 // If set to 1 this indicates that the current instruction is a
16232 // short variant of a long branch. This avoids using this
16233 // instruction in first-pass matching. It will then only be used in
16234 // the `Shorten_branches' pass.
16235 // ins_short_branch(1);
16236 format %{ "b$cmp $lbl" %}
16237
16238 ins_encode(aarch64_enc_br_con(cmp, lbl));
16239
16240 ins_pipe(pipe_branch_cond);
16241 %}
16242
16243 // Conditional Near Branch Unsigned
16244 instruct branchConU(cmpOpU cmp, rFlagsRegU cr, label lbl)
16245 %{
16246 // Same match rule as `branchConFar'.
16247 match(If cmp cr);
16248
16249 effect(USE lbl);
16250
16251 ins_cost(BRANCH_COST);
16252 // If set to 1 this indicates that the current instruction is a
16253 // short variant of a long branch. This avoids using this
16254 // instruction in first-pass matching. It will then only be used in
16255 // the `Shorten_branches' pass.
16256 // ins_short_branch(1);
16257 format %{ "b$cmp $lbl\t# unsigned" %}
16258
16259 ins_encode(aarch64_enc_br_conU(cmp, lbl));
16260
16261 ins_pipe(pipe_branch_cond);
16262 %}
16263
16264 // Make use of CBZ and CBNZ. These instructions, as well as being
16265 // shorter than (cmp; branch), have the additional benefit of not
16266 // killing the flags.
16267
16268 instruct cmpI_imm0_branch(cmpOpEqNe cmp, iRegIorL2I op1, immI0 op2, label labl, rFlagsReg cr) %{
16269 match(If cmp (CmpI op1 op2));
16270 effect(USE labl);
16271
16272 ins_cost(BRANCH_COST);
16273 format %{ "cbw$cmp $op1, $labl" %}
16274 ins_encode %{
16275 Label* L = $labl$$label;
16276 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
16277 if (cond == Assembler::EQ)
16278 __ cbzw($op1$$Register, *L);
16279 else
16280 __ cbnzw($op1$$Register, *L);
16281 %}
16282 ins_pipe(pipe_cmp_branch);
16283 %}
16284
16285 instruct cmpL_imm0_branch(cmpOpEqNe cmp, iRegL op1, immL0 op2, label labl, rFlagsReg cr) %{
16286 match(If cmp (CmpL op1 op2));
16287 effect(USE labl);
16288
16289 ins_cost(BRANCH_COST);
16290 format %{ "cb$cmp $op1, $labl" %}
16291 ins_encode %{
16292 Label* L = $labl$$label;
16293 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
16294 if (cond == Assembler::EQ)
16295 __ cbz($op1$$Register, *L);
16296 else
16297 __ cbnz($op1$$Register, *L);
16298 %}
16299 ins_pipe(pipe_cmp_branch);
16300 %}
16301
16302 instruct cmpP_imm0_branch(cmpOpEqNe cmp, iRegP op1, immP0 op2, label labl, rFlagsReg cr) %{
16303 match(If cmp (CmpP op1 op2));
16304 effect(USE labl);
16305
16306 ins_cost(BRANCH_COST);
16307 format %{ "cb$cmp $op1, $labl" %}
16308 ins_encode %{
16309 Label* L = $labl$$label;
16310 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
16311 if (cond == Assembler::EQ)
16312 __ cbz($op1$$Register, *L);
16313 else
16314 __ cbnz($op1$$Register, *L);
16315 %}
16316 ins_pipe(pipe_cmp_branch);
16317 %}
16318
16319 instruct cmpN_imm0_branch(cmpOpEqNe cmp, iRegN op1, immN0 op2, label labl, rFlagsReg cr) %{
16320 match(If cmp (CmpN op1 op2));
16321 effect(USE labl);
16322
16323 ins_cost(BRANCH_COST);
16324 format %{ "cbw$cmp $op1, $labl" %}
16325 ins_encode %{
16326 Label* L = $labl$$label;
16327 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
16328 if (cond == Assembler::EQ)
16329 __ cbzw($op1$$Register, *L);
16330 else
16331 __ cbnzw($op1$$Register, *L);
16332 %}
16333 ins_pipe(pipe_cmp_branch);
16334 %}
16335
16336 instruct cmpP_narrowOop_imm0_branch(cmpOpEqNe cmp, iRegN oop, immP0 zero, label labl, rFlagsReg cr) %{
16337 match(If cmp (CmpP (DecodeN oop) zero));
16338 effect(USE labl);
16339
16340 ins_cost(BRANCH_COST);
16341 format %{ "cb$cmp $oop, $labl" %}
16342 ins_encode %{
16343 Label* L = $labl$$label;
16344 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
16345 if (cond == Assembler::EQ)
16346 __ cbzw($oop$$Register, *L);
16347 else
16348 __ cbnzw($oop$$Register, *L);
16349 %}
16350 ins_pipe(pipe_cmp_branch);
16351 %}
16352
16353 instruct cmpUI_imm0_branch(cmpOpUEqNeLtGe cmp, iRegIorL2I op1, immI0 op2, label labl, rFlagsRegU cr) %{
16354 match(If cmp (CmpU op1 op2));
16355 effect(USE labl);
16356
16357 ins_cost(BRANCH_COST);
16358 format %{ "cbw$cmp $op1, $labl" %}
16359 ins_encode %{
16360 Label* L = $labl$$label;
16361 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
16362 if (cond == Assembler::EQ || cond == Assembler::LS)
16363 __ cbzw($op1$$Register, *L);
16364 else
16365 __ cbnzw($op1$$Register, *L);
16366 %}
16367 ins_pipe(pipe_cmp_branch);
16368 %}
16369
16370 instruct cmpUL_imm0_branch(cmpOpUEqNeLtGe cmp, iRegL op1, immL0 op2, label labl, rFlagsRegU cr) %{
16371 match(If cmp (CmpUL op1 op2));
16372 effect(USE labl);
16373
16374 ins_cost(BRANCH_COST);
16375 format %{ "cb$cmp $op1, $labl" %}
16376 ins_encode %{
16377 Label* L = $labl$$label;
16378 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
16379 if (cond == Assembler::EQ || cond == Assembler::LS)
16380 __ cbz($op1$$Register, *L);
16381 else
16382 __ cbnz($op1$$Register, *L);
16383 %}
16384 ins_pipe(pipe_cmp_branch);
16385 %}
16386
16387 // Test bit and Branch
16388
16389 // Patterns for short (< 32KiB) variants
16390 instruct cmpL_branch_sign(cmpOpLtGe cmp, iRegL op1, immL0 op2, label labl) %{
16391 match(If cmp (CmpL op1 op2));
16392 effect(USE labl);
16393
16394 ins_cost(BRANCH_COST);
16395 format %{ "cb$cmp $op1, $labl # long" %}
16396 ins_encode %{
16397 Label* L = $labl$$label;
16398 Assembler::Condition cond =
16399 ((Assembler::Condition)$cmp$$cmpcode == Assembler::LT) ? Assembler::NE : Assembler::EQ;
16400 __ tbr(cond, $op1$$Register, 63, *L);
16401 %}
16402 ins_pipe(pipe_cmp_branch);
16403 ins_short_branch(1);
16404 %}
16405
16406 instruct cmpI_branch_sign(cmpOpLtGe cmp, iRegIorL2I op1, immI0 op2, label labl) %{
16407 match(If cmp (CmpI op1 op2));
16408 effect(USE labl);
16409
16410 ins_cost(BRANCH_COST);
16411 format %{ "cb$cmp $op1, $labl # int" %}
16412 ins_encode %{
16413 Label* L = $labl$$label;
16414 Assembler::Condition cond =
16415 ((Assembler::Condition)$cmp$$cmpcode == Assembler::LT) ? Assembler::NE : Assembler::EQ;
16416 __ tbr(cond, $op1$$Register, 31, *L);
16417 %}
16418 ins_pipe(pipe_cmp_branch);
16419 ins_short_branch(1);
16420 %}
16421
16422 instruct cmpL_branch_bit(cmpOpEqNe cmp, iRegL op1, immL op2, immL0 op3, label labl) %{
16423 match(If cmp (CmpL (AndL op1 op2) op3));
16424 predicate(is_power_of_2((julong)n->in(2)->in(1)->in(2)->get_long()));
16425 effect(USE labl);
16426
16427 ins_cost(BRANCH_COST);
16428 format %{ "tb$cmp $op1, $op2, $labl" %}
16429 ins_encode %{
16430 Label* L = $labl$$label;
16431 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
16432 int bit = exact_log2_long($op2$$constant);
16433 __ tbr(cond, $op1$$Register, bit, *L);
16434 %}
16435 ins_pipe(pipe_cmp_branch);
16436 ins_short_branch(1);
16437 %}
16438
16439 instruct cmpI_branch_bit(cmpOpEqNe cmp, iRegIorL2I op1, immI op2, immI0 op3, label labl) %{
16440 match(If cmp (CmpI (AndI op1 op2) op3));
16441 predicate(is_power_of_2((juint)n->in(2)->in(1)->in(2)->get_int()));
16442 effect(USE labl);
16443
16444 ins_cost(BRANCH_COST);
16445 format %{ "tb$cmp $op1, $op2, $labl" %}
16446 ins_encode %{
16447 Label* L = $labl$$label;
16448 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
16449 int bit = exact_log2((juint)$op2$$constant);
16450 __ tbr(cond, $op1$$Register, bit, *L);
16451 %}
16452 ins_pipe(pipe_cmp_branch);
16453 ins_short_branch(1);
16454 %}
16455
16456 // And far variants
16457 instruct far_cmpL_branch_sign(cmpOpLtGe cmp, iRegL op1, immL0 op2, label labl) %{
16458 match(If cmp (CmpL op1 op2));
16459 effect(USE labl);
16460
16461 ins_cost(BRANCH_COST);
16462 format %{ "cb$cmp $op1, $labl # long" %}
16463 ins_encode %{
16464 Label* L = $labl$$label;
16465 Assembler::Condition cond =
16466 ((Assembler::Condition)$cmp$$cmpcode == Assembler::LT) ? Assembler::NE : Assembler::EQ;
16467 __ tbr(cond, $op1$$Register, 63, *L, /*far*/true);
16468 %}
16469 ins_pipe(pipe_cmp_branch);
16470 %}
16471
16472 instruct far_cmpI_branch_sign(cmpOpLtGe cmp, iRegIorL2I op1, immI0 op2, label labl) %{
16473 match(If cmp (CmpI op1 op2));
16474 effect(USE labl);
16475
16476 ins_cost(BRANCH_COST);
16477 format %{ "cb$cmp $op1, $labl # int" %}
16478 ins_encode %{
16479 Label* L = $labl$$label;
16480 Assembler::Condition cond =
16481 ((Assembler::Condition)$cmp$$cmpcode == Assembler::LT) ? Assembler::NE : Assembler::EQ;
16482 __ tbr(cond, $op1$$Register, 31, *L, /*far*/true);
16483 %}
16484 ins_pipe(pipe_cmp_branch);
16485 %}
16486
16487 instruct far_cmpL_branch_bit(cmpOpEqNe cmp, iRegL op1, immL op2, immL0 op3, label labl) %{
16488 match(If cmp (CmpL (AndL op1 op2) op3));
16489 predicate(is_power_of_2((julong)n->in(2)->in(1)->in(2)->get_long()));
16490 effect(USE labl);
16491
16492 ins_cost(BRANCH_COST);
16493 format %{ "tb$cmp $op1, $op2, $labl" %}
16494 ins_encode %{
16495 Label* L = $labl$$label;
16496 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
16497 int bit = exact_log2_long($op2$$constant);
16498 __ tbr(cond, $op1$$Register, bit, *L, /*far*/true);
16499 %}
16500 ins_pipe(pipe_cmp_branch);
16501 %}
16502
16503 instruct far_cmpI_branch_bit(cmpOpEqNe cmp, iRegIorL2I op1, immI op2, immI0 op3, label labl) %{
16504 match(If cmp (CmpI (AndI op1 op2) op3));
16505 predicate(is_power_of_2((juint)n->in(2)->in(1)->in(2)->get_int()));
16506 effect(USE labl);
16507
16508 ins_cost(BRANCH_COST);
16509 format %{ "tb$cmp $op1, $op2, $labl" %}
16510 ins_encode %{
16511 Label* L = $labl$$label;
16512 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
16513 int bit = exact_log2((juint)$op2$$constant);
16514 __ tbr(cond, $op1$$Register, bit, *L, /*far*/true);
16515 %}
16516 ins_pipe(pipe_cmp_branch);
16517 %}
16518
16519 // Test bits
16520
16521 instruct cmpL_and(cmpOp cmp, iRegL op1, immL op2, immL0 op3, rFlagsReg cr) %{
16522 match(Set cr (CmpL (AndL op1 op2) op3));
16523 predicate(Assembler::operand_valid_for_logical_immediate
16524 (/*is_32*/false, n->in(1)->in(2)->get_long()));
16525
16526 ins_cost(INSN_COST);
16527 format %{ "tst $op1, $op2 # long" %}
16528 ins_encode %{
16529 __ tst($op1$$Register, $op2$$constant);
16530 %}
16531 ins_pipe(ialu_reg_reg);
16532 %}
16533
16534 instruct cmpI_and(cmpOp cmp, iRegIorL2I op1, immI op2, immI0 op3, rFlagsReg cr) %{
16535 match(Set cr (CmpI (AndI op1 op2) op3));
16536 predicate(Assembler::operand_valid_for_logical_immediate
16537 (/*is_32*/true, n->in(1)->in(2)->get_int()));
16538
16539 ins_cost(INSN_COST);
16540 format %{ "tst $op1, $op2 # int" %}
16541 ins_encode %{
16542 __ tstw($op1$$Register, $op2$$constant);
16543 %}
16544 ins_pipe(ialu_reg_reg);
16545 %}
16546
16547 instruct cmpL_and_reg(cmpOp cmp, iRegL op1, iRegL op2, immL0 op3, rFlagsReg cr) %{
16548 match(Set cr (CmpL (AndL op1 op2) op3));
16549
16550 ins_cost(INSN_COST);
16551 format %{ "tst $op1, $op2 # long" %}
16552 ins_encode %{
16553 __ tst($op1$$Register, $op2$$Register);
16554 %}
16555 ins_pipe(ialu_reg_reg);
16556 %}
16557
16558 instruct cmpI_and_reg(cmpOp cmp, iRegIorL2I op1, iRegIorL2I op2, immI0 op3, rFlagsReg cr) %{
16559 match(Set cr (CmpI (AndI op1 op2) op3));
16560
16561 ins_cost(INSN_COST);
16562 format %{ "tstw $op1, $op2 # int" %}
16563 ins_encode %{
16564 __ tstw($op1$$Register, $op2$$Register);
16565 %}
16566 ins_pipe(ialu_reg_reg);
16567 %}
16568
16569
16570 // Conditional Far Branch
16571 // Conditional Far Branch Unsigned
16572 // TODO: fixme
16573
16574 // counted loop end branch near
16575 instruct branchLoopEnd(cmpOp cmp, rFlagsReg cr, label lbl)
16576 %{
16577 match(CountedLoopEnd cmp cr);
16578
16579 effect(USE lbl);
16580
16581 ins_cost(BRANCH_COST);
16582 // short variant.
16583 // ins_short_branch(1);
16584 format %{ "b$cmp $lbl \t// counted loop end" %}
16585
16586 ins_encode(aarch64_enc_br_con(cmp, lbl));
16587
16588 ins_pipe(pipe_branch);
16589 %}
16590
16591 // counted loop end branch near Unsigned
16592 instruct branchLoopEndU(cmpOpU cmp, rFlagsRegU cr, label lbl)
16593 %{
16594 match(CountedLoopEnd cmp cr);
16595
16596 effect(USE lbl);
16597
16598 ins_cost(BRANCH_COST);
16599 // short variant.
16600 // ins_short_branch(1);
16601 format %{ "b$cmp $lbl \t// counted loop end unsigned" %}
16602
16603 ins_encode(aarch64_enc_br_conU(cmp, lbl));
16604
16605 ins_pipe(pipe_branch);
16606 %}
16607
16608 // counted loop end branch far
16609 // counted loop end branch far unsigned
16610 // TODO: fixme
16611
16612 // ============================================================================
16613 // inlined locking and unlocking
16614
16615 instruct cmpFastLock(rFlagsReg cr, iRegP object, iRegP box, iRegPNoSp tmp, iRegPNoSp tmp2)
16616 %{
16617 match(Set cr (FastLock object box));
16618 effect(TEMP tmp, TEMP tmp2);
16619
16620 // TODO
16621 // identify correct cost
16622 ins_cost(5 * INSN_COST);
16623 format %{ "fastlock $object,$box\t! kills $tmp,$tmp2" %}
16624
16625 ins_encode(aarch64_enc_fast_lock(object, box, tmp, tmp2));
16626
16627 ins_pipe(pipe_serial);
16628 %}
16629
16630 instruct cmpFastUnlock(rFlagsReg cr, iRegP object, iRegP box, iRegPNoSp tmp, iRegPNoSp tmp2)
16631 %{
16632 match(Set cr (FastUnlock object box));
16633 effect(TEMP tmp, TEMP tmp2);
16634
16635 ins_cost(5 * INSN_COST);
16636 format %{ "fastunlock $object,$box\t! kills $tmp, $tmp2" %}
16637
16638 ins_encode(aarch64_enc_fast_unlock(object, box, tmp, tmp2));
16639
16640 ins_pipe(pipe_serial);
16641 %}
16642
16643
16644 // ============================================================================
16645 // Safepoint Instructions
16646
16647 // TODO
16648 // provide a near and far version of this code
16649
16650 instruct safePoint(rFlagsReg cr, iRegP poll)
16651 %{
16652 match(SafePoint poll);
16653 effect(KILL cr);
16654
16655 format %{
16656 "ldrw zr, [$poll]\t# Safepoint: poll for GC"
16657 %}
16658 ins_encode %{
16659 __ read_polling_page(as_Register($poll$$reg), relocInfo::poll_type);
16660 %}
16661 ins_pipe(pipe_serial); // ins_pipe(iload_reg_mem);
16662 %}
16663
16664
16665 // ============================================================================
16666 // Procedure Call/Return Instructions
16667
16668 // Call Java Static Instruction
16669
16670 instruct CallStaticJavaDirect(method meth)
16671 %{
16672 match(CallStaticJava);
16673
16674 effect(USE meth);
16675
16676 ins_cost(CALL_COST);
16677
16678 format %{ "call,static $meth \t// ==> " %}
16679
16680 ins_encode(aarch64_enc_java_static_call(meth),
16681 aarch64_enc_call_epilog);
16682
16683 ins_pipe(pipe_class_call);
16684 %}
16685
16686 // TO HERE
16687
16688 // Call Java Dynamic Instruction
16689 instruct CallDynamicJavaDirect(method meth)
16690 %{
16691 match(CallDynamicJava);
16692
16693 effect(USE meth);
16694
16695 ins_cost(CALL_COST);
16696
16697 format %{ "CALL,dynamic $meth \t// ==> " %}
16698
16699 ins_encode(aarch64_enc_java_dynamic_call(meth),
16700 aarch64_enc_call_epilog);
16701
16702 ins_pipe(pipe_class_call);
16703 %}
16704
16705 // Call Runtime Instruction
16706
16707 instruct CallRuntimeDirect(method meth)
16708 %{
16709 match(CallRuntime);
16710
16711 effect(USE meth);
16712
16713 ins_cost(CALL_COST);
16714
16715 format %{ "CALL, runtime $meth" %}
16716
16717 ins_encode( aarch64_enc_java_to_runtime(meth) );
16718
16719 ins_pipe(pipe_class_call);
16720 %}
16721
16722 // Call Runtime Instruction
16723
16724 instruct CallLeafDirect(method meth)
16725 %{
16726 match(CallLeaf);
16727
16728 effect(USE meth);
16729
16730 ins_cost(CALL_COST);
16731
16732 format %{ "CALL, runtime leaf $meth" %}
16733
16734 ins_encode( aarch64_enc_java_to_runtime(meth) );
16735
16736 ins_pipe(pipe_class_call);
16737 %}
16738
16739 // Call Runtime Instruction
16740
16741 instruct CallLeafNoFPDirect(method meth)
16742 %{
16743 match(CallLeafNoFP);
16744
16745 effect(USE meth);
16746
16747 ins_cost(CALL_COST);
16748
16749 format %{ "CALL, runtime leaf nofp $meth" %}
16750
16751 ins_encode( aarch64_enc_java_to_runtime(meth) );
16752
16753 ins_pipe(pipe_class_call);
16754 %}
16755
16756 instruct CallNativeDirect(method meth)
16757 %{
16758 match(CallNative);
16759
16760 effect(USE meth);
16761
16762 ins_cost(CALL_COST);
16763
16764 format %{ "CALL, native $meth" %}
16765
16766 ins_encode( aarch64_enc_java_to_runtime(meth) );
16767
16768 ins_pipe(pipe_class_call);
16769 %}
16770
16771 // Tail Call; Jump from runtime stub to Java code.
16772 // Also known as an 'interprocedural jump'.
16773 // Target of jump will eventually return to caller.
16774 // TailJump below removes the return address.
16775 instruct TailCalljmpInd(iRegPNoSp jump_target, inline_cache_RegP method_ptr)
16776 %{
16777 match(TailCall jump_target method_ptr);
16778
16779 ins_cost(CALL_COST);
16780
16781 format %{ "br $jump_target\t# $method_ptr holds method" %}
16782
16783 ins_encode(aarch64_enc_tail_call(jump_target));
16784
16785 ins_pipe(pipe_class_call);
16786 %}
16787
16788 instruct TailjmpInd(iRegPNoSp jump_target, iRegP_R0 ex_oop)
16789 %{
16790 match(TailJump jump_target ex_oop);
16791
16792 ins_cost(CALL_COST);
16793
16794 format %{ "br $jump_target\t# $ex_oop holds exception oop" %}
16795
16796 ins_encode(aarch64_enc_tail_jmp(jump_target));
16797
16798 ins_pipe(pipe_class_call);
16799 %}
16800
16801 // Create exception oop: created by stack-crawling runtime code.
16802 // Created exception is now available to this handler, and is setup
16803 // just prior to jumping to this handler. No code emitted.
16804 // TODO check
16805 // should ex_oop be in r0? intel uses rax, ppc cannot use r0 so uses rarg1
16806 instruct CreateException(iRegP_R0 ex_oop)
16807 %{
16808 match(Set ex_oop (CreateEx));
16809
16810 format %{ " -- \t// exception oop; no code emitted" %}
16811
16812 size(0);
16813
16814 ins_encode( /*empty*/ );
16815
16816 ins_pipe(pipe_class_empty);
16817 %}
16818
16819 // Rethrow exception: The exception oop will come in the first
16820 // argument position. Then JUMP (not call) to the rethrow stub code.
16821 instruct RethrowException() %{
16822 match(Rethrow);
16823 ins_cost(CALL_COST);
16824
16825 format %{ "b rethrow_stub" %}
16826
16827 ins_encode( aarch64_enc_rethrow() );
16828
16829 ins_pipe(pipe_class_call);
16830 %}
16831
16832
16833 // Return Instruction
16834 // epilog node loads ret address into lr as part of frame pop
16835 instruct Ret()
16836 %{
16837 match(Return);
16838
16839 format %{ "ret\t// return register" %}
16840
16841 ins_encode( aarch64_enc_ret() );
16842
16843 ins_pipe(pipe_branch);
16844 %}
16845
16846 // Die now.
16847 instruct ShouldNotReachHere() %{
16848 match(Halt);
16849
16850 ins_cost(CALL_COST);
16851 format %{ "ShouldNotReachHere" %}
16852
16853 ins_encode %{
16854 if (is_reachable()) {
16855 __ stop(_halt_reason);
16856 }
16857 %}
16858
16859 ins_pipe(pipe_class_default);
16860 %}
16861
16862 // ============================================================================
16863 // Partial Subtype Check
16864 //
16865 // superklass array for an instance of the superklass. Set a hidden
16866 // internal cache on a hit (cache is checked with exposed code in
16867 // gen_subtype_check()). Return NZ for a miss or zero for a hit. The
16868 // encoding ALSO sets flags.
16869
16870 instruct partialSubtypeCheck(iRegP_R4 sub, iRegP_R0 super, iRegP_R2 temp, iRegP_R5 result, rFlagsReg cr)
16871 %{
16872 match(Set result (PartialSubtypeCheck sub super));
16873 effect(KILL cr, KILL temp);
16874
16875 ins_cost(1100); // slightly larger than the next version
16876 format %{ "partialSubtypeCheck $result, $sub, $super" %}
16877
16878 ins_encode(aarch64_enc_partial_subtype_check(sub, super, temp, result));
16879
16880 opcode(0x1); // Force zero of result reg on hit
16881
16882 ins_pipe(pipe_class_memory);
16883 %}
16884
16885 instruct partialSubtypeCheckVsZero(iRegP_R4 sub, iRegP_R0 super, iRegP_R2 temp, iRegP_R5 result, immP0 zero, rFlagsReg cr)
16886 %{
16887 match(Set cr (CmpP (PartialSubtypeCheck sub super) zero));
16888 effect(KILL temp, KILL result);
16889
16890 ins_cost(1100); // slightly larger than the next version
16891 format %{ "partialSubtypeCheck $result, $sub, $super == 0" %}
16892
16893 ins_encode(aarch64_enc_partial_subtype_check(sub, super, temp, result));
16894
16895 opcode(0x0); // Don't zero result reg on hit
16896
16897 ins_pipe(pipe_class_memory);
16898 %}
16899
16900 instruct string_compareU(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
16901 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2, rFlagsReg cr)
16902 %{
16903 predicate(((StrCompNode*)n)->encoding() == StrIntrinsicNode::UU);
16904 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
16905 effect(KILL tmp1, KILL tmp2, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
16906
16907 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # KILL $tmp1" %}
16908 ins_encode %{
16909 // Count is in 8-bit bytes; non-Compact chars are 16 bits.
16910 __ string_compare($str1$$Register, $str2$$Register,
16911 $cnt1$$Register, $cnt2$$Register, $result$$Register,
16912 $tmp1$$Register, $tmp2$$Register,
16913 fnoreg, fnoreg, fnoreg, StrIntrinsicNode::UU);
16914 %}
16915 ins_pipe(pipe_class_memory);
16916 %}
16917
16918 instruct string_compareL(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
16919 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2, rFlagsReg cr)
16920 %{
16921 predicate(((StrCompNode*)n)->encoding() == StrIntrinsicNode::LL);
16922 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
16923 effect(KILL tmp1, KILL tmp2, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
16924
16925 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # KILL $tmp1" %}
16926 ins_encode %{
16927 __ string_compare($str1$$Register, $str2$$Register,
16928 $cnt1$$Register, $cnt2$$Register, $result$$Register,
16929 $tmp1$$Register, $tmp2$$Register,
16930 fnoreg, fnoreg, fnoreg, StrIntrinsicNode::LL);
16931 %}
16932 ins_pipe(pipe_class_memory);
16933 %}
16934
16935 instruct string_compareUL(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
16936 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
16937 vRegD_V0 vtmp1, vRegD_V1 vtmp2, vRegD_V2 vtmp3, rFlagsReg cr)
16938 %{
16939 predicate(((StrCompNode*)n)->encoding() == StrIntrinsicNode::UL);
16940 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
16941 effect(KILL tmp1, KILL tmp2, KILL vtmp1, KILL vtmp2, KILL vtmp3,
16942 USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
16943
16944 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # KILL $tmp1, $tmp2, $vtmp1, $vtmp2, $vtmp3" %}
16945 ins_encode %{
16946 __ string_compare($str1$$Register, $str2$$Register,
16947 $cnt1$$Register, $cnt2$$Register, $result$$Register,
16948 $tmp1$$Register, $tmp2$$Register,
16949 $vtmp1$$FloatRegister, $vtmp2$$FloatRegister,
16950 $vtmp3$$FloatRegister, StrIntrinsicNode::UL);
16951 %}
16952 ins_pipe(pipe_class_memory);
16953 %}
16954
16955 instruct string_compareLU(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
16956 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
16957 vRegD_V0 vtmp1, vRegD_V1 vtmp2, vRegD_V2 vtmp3, rFlagsReg cr)
16958 %{
16959 predicate(((StrCompNode*)n)->encoding() == StrIntrinsicNode::LU);
16960 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
16961 effect(KILL tmp1, KILL tmp2, KILL vtmp1, KILL vtmp2, KILL vtmp3,
16962 USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
16963
16964 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # KILL $tmp1, $tmp2, $vtmp1, $vtmp2, $vtmp3" %}
16965 ins_encode %{
16966 __ string_compare($str1$$Register, $str2$$Register,
16967 $cnt1$$Register, $cnt2$$Register, $result$$Register,
16968 $tmp1$$Register, $tmp2$$Register,
16969 $vtmp1$$FloatRegister, $vtmp2$$FloatRegister,
16970 $vtmp3$$FloatRegister,StrIntrinsicNode::LU);
16971 %}
16972 ins_pipe(pipe_class_memory);
16973 %}
16974
16975 instruct string_indexofUU(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2, iRegI_R2 cnt2,
16976 iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2, iRegINoSp tmp3,
16977 iRegINoSp tmp4, iRegINoSp tmp5, iRegINoSp tmp6, rFlagsReg cr)
16978 %{
16979 predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UU);
16980 match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 cnt2)));
16981 effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2,
16982 TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, TEMP tmp5, TEMP tmp6, KILL cr);
16983 format %{ "String IndexOf $str1,$cnt1,$str2,$cnt2 -> $result (UU)" %}
16984
16985 ins_encode %{
16986 __ string_indexof($str1$$Register, $str2$$Register,
16987 $cnt1$$Register, $cnt2$$Register,
16988 $tmp1$$Register, $tmp2$$Register,
16989 $tmp3$$Register, $tmp4$$Register,
16990 $tmp5$$Register, $tmp6$$Register,
16991 -1, $result$$Register, StrIntrinsicNode::UU);
16992 %}
16993 ins_pipe(pipe_class_memory);
16994 %}
16995
16996 instruct string_indexofLL(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2, iRegI_R2 cnt2,
16997 iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2, iRegINoSp tmp3,
16998 iRegINoSp tmp4, iRegINoSp tmp5, iRegINoSp tmp6, rFlagsReg cr)
16999 %{
17000 predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::LL);
17001 match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 cnt2)));
17002 effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2,
17003 TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, TEMP tmp5, TEMP tmp6, KILL cr);
17004 format %{ "String IndexOf $str1,$cnt1,$str2,$cnt2 -> $result (LL)" %}
17005
17006 ins_encode %{
17007 __ string_indexof($str1$$Register, $str2$$Register,
17008 $cnt1$$Register, $cnt2$$Register,
17009 $tmp1$$Register, $tmp2$$Register,
17010 $tmp3$$Register, $tmp4$$Register,
17011 $tmp5$$Register, $tmp6$$Register,
17012 -1, $result$$Register, StrIntrinsicNode::LL);
17013 %}
17014 ins_pipe(pipe_class_memory);
17015 %}
17016
17017 instruct string_indexofUL(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2, iRegI_R2 cnt2,
17018 iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2, iRegINoSp tmp3,
17019 iRegINoSp tmp4, iRegINoSp tmp5, iRegINoSp tmp6, rFlagsReg cr)
17020 %{
17021 predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UL);
17022 match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 cnt2)));
17023 effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2,
17024 TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, TEMP tmp5, TEMP tmp6, KILL cr);
17025 format %{ "String IndexOf $str1,$cnt1,$str2,$cnt2 -> $result (UL)" %}
17026
17027 ins_encode %{
17028 __ string_indexof($str1$$Register, $str2$$Register,
17029 $cnt1$$Register, $cnt2$$Register,
17030 $tmp1$$Register, $tmp2$$Register,
17031 $tmp3$$Register, $tmp4$$Register,
17032 $tmp5$$Register, $tmp6$$Register,
17033 -1, $result$$Register, StrIntrinsicNode::UL);
17034 %}
17035 ins_pipe(pipe_class_memory);
17036 %}
17037
17038 instruct string_indexof_conUU(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2,
17039 immI_le_4 int_cnt2, iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2,
17040 iRegINoSp tmp3, iRegINoSp tmp4, rFlagsReg cr)
17041 %{
17042 predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UU);
17043 match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 int_cnt2)));
17044 effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1,
17045 TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, KILL cr);
17046 format %{ "String IndexOf $str1,$cnt1,$str2,$int_cnt2 -> $result (UU)" %}
17047
17048 ins_encode %{
17049 int icnt2 = (int)$int_cnt2$$constant;
17050 __ string_indexof($str1$$Register, $str2$$Register,
17051 $cnt1$$Register, zr,
17052 $tmp1$$Register, $tmp2$$Register,
17053 $tmp3$$Register, $tmp4$$Register, zr, zr,
17054 icnt2, $result$$Register, StrIntrinsicNode::UU);
17055 %}
17056 ins_pipe(pipe_class_memory);
17057 %}
17058
17059 instruct string_indexof_conLL(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2,
17060 immI_le_4 int_cnt2, iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2,
17061 iRegINoSp tmp3, iRegINoSp tmp4, rFlagsReg cr)
17062 %{
17063 predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::LL);
17064 match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 int_cnt2)));
17065 effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1,
17066 TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, KILL cr);
17067 format %{ "String IndexOf $str1,$cnt1,$str2,$int_cnt2 -> $result (LL)" %}
17068
17069 ins_encode %{
17070 int icnt2 = (int)$int_cnt2$$constant;
17071 __ string_indexof($str1$$Register, $str2$$Register,
17072 $cnt1$$Register, zr,
17073 $tmp1$$Register, $tmp2$$Register,
17074 $tmp3$$Register, $tmp4$$Register, zr, zr,
17075 icnt2, $result$$Register, StrIntrinsicNode::LL);
17076 %}
17077 ins_pipe(pipe_class_memory);
17078 %}
17079
17080 instruct string_indexof_conUL(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2,
17081 immI_1 int_cnt2, iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2,
17082 iRegINoSp tmp3, iRegINoSp tmp4, rFlagsReg cr)
17083 %{
17084 predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UL);
17085 match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 int_cnt2)));
17086 effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1,
17087 TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, KILL cr);
17088 format %{ "String IndexOf $str1,$cnt1,$str2,$int_cnt2 -> $result (UL)" %}
17089
17090 ins_encode %{
17091 int icnt2 = (int)$int_cnt2$$constant;
17092 __ string_indexof($str1$$Register, $str2$$Register,
17093 $cnt1$$Register, zr,
17094 $tmp1$$Register, $tmp2$$Register,
17095 $tmp3$$Register, $tmp4$$Register, zr, zr,
17096 icnt2, $result$$Register, StrIntrinsicNode::UL);
17097 %}
17098 ins_pipe(pipe_class_memory);
17099 %}
17100
17101 instruct string_indexof_char(iRegP_R1 str1, iRegI_R2 cnt1, iRegI_R3 ch,
17102 iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2,
17103 iRegINoSp tmp3, rFlagsReg cr)
17104 %{
17105 match(Set result (StrIndexOfChar (Binary str1 cnt1) ch));
17106 predicate((UseSVE == 0) && (((StrIndexOfCharNode*)n)->encoding() == StrIntrinsicNode::U));
17107 effect(USE_KILL str1, USE_KILL cnt1, USE_KILL ch,
17108 TEMP tmp1, TEMP tmp2, TEMP tmp3, KILL cr);
17109
17110 format %{ "StringUTF16 IndexOf char[] $str1,$cnt1,$ch -> $result" %}
17111
17112 ins_encode %{
17113 __ string_indexof_char($str1$$Register, $cnt1$$Register, $ch$$Register,
17114 $result$$Register, $tmp1$$Register, $tmp2$$Register,
17115 $tmp3$$Register);
17116 %}
17117 ins_pipe(pipe_class_memory);
17118 %}
17119
17120 instruct stringL_indexof_char(iRegP_R1 str1, iRegI_R2 cnt1, iRegI_R3 ch,
17121 iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2,
17122 iRegINoSp tmp3, rFlagsReg cr)
17123 %{
17124 match(Set result (StrIndexOfChar (Binary str1 cnt1) ch));
17125 predicate((UseSVE == 0) && (((StrIndexOfCharNode*)n)->encoding() == StrIntrinsicNode::L));
17126 effect(USE_KILL str1, USE_KILL cnt1, USE_KILL ch,
17127 TEMP tmp1, TEMP tmp2, TEMP tmp3, KILL cr);
17128
17129 format %{ "StringLatin1 IndexOf char[] $str1,$cnt1,$ch -> $result" %}
17130
17131 ins_encode %{
17132 __ stringL_indexof_char($str1$$Register, $cnt1$$Register, $ch$$Register,
17133 $result$$Register, $tmp1$$Register, $tmp2$$Register,
17134 $tmp3$$Register);
17135 %}
17136 ins_pipe(pipe_class_memory);
17137 %}
17138
17139 instruct string_equalsL(iRegP_R1 str1, iRegP_R3 str2, iRegI_R4 cnt,
17140 iRegI_R0 result, rFlagsReg cr)
17141 %{
17142 predicate(((StrEqualsNode*)n)->encoding() == StrIntrinsicNode::LL);
17143 match(Set result (StrEquals (Binary str1 str2) cnt));
17144 effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt, KILL cr);
17145
17146 format %{ "String Equals $str1,$str2,$cnt -> $result" %}
17147 ins_encode %{
17148 // Count is in 8-bit bytes; non-Compact chars are 16 bits.
17149 __ string_equals($str1$$Register, $str2$$Register,
17150 $result$$Register, $cnt$$Register, 1);
17151 %}
17152 ins_pipe(pipe_class_memory);
17153 %}
17154
17155 instruct string_equalsU(iRegP_R1 str1, iRegP_R3 str2, iRegI_R4 cnt,
17156 iRegI_R0 result, rFlagsReg cr)
17157 %{
17158 predicate(((StrEqualsNode*)n)->encoding() == StrIntrinsicNode::UU);
17159 match(Set result (StrEquals (Binary str1 str2) cnt));
17160 effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt, KILL cr);
17161
17162 format %{ "String Equals $str1,$str2,$cnt -> $result" %}
17163 ins_encode %{
17164 // Count is in 8-bit bytes; non-Compact chars are 16 bits.
17165 __ string_equals($str1$$Register, $str2$$Register,
17166 $result$$Register, $cnt$$Register, 2);
17167 %}
17168 ins_pipe(pipe_class_memory);
17169 %}
17170
17171 instruct array_equalsB(iRegP_R1 ary1, iRegP_R2 ary2, iRegI_R0 result,
17172 iRegP_R3 tmp1, iRegP_R4 tmp2, iRegP_R5 tmp3,
17173 iRegP_R10 tmp, rFlagsReg cr)
17174 %{
17175 predicate(((AryEqNode*)n)->encoding() == StrIntrinsicNode::LL);
17176 match(Set result (AryEq ary1 ary2));
17177 effect(KILL tmp, USE_KILL ary1, USE_KILL ary2, TEMP tmp1, TEMP tmp2, TEMP tmp3, KILL cr);
17178
17179 format %{ "Array Equals $ary1,ary2 -> $result // KILL $tmp" %}
17180 ins_encode %{
17181 address tpc = __ arrays_equals($ary1$$Register, $ary2$$Register,
17182 $tmp1$$Register, $tmp2$$Register, $tmp3$$Register,
17183 $result$$Register, $tmp$$Register, 1);
17184 if (tpc == NULL) {
17185 ciEnv::current()->record_failure("CodeCache is full");
17186 return;
17187 }
17188 %}
17189 ins_pipe(pipe_class_memory);
17190 %}
17191
17192 instruct array_equalsC(iRegP_R1 ary1, iRegP_R2 ary2, iRegI_R0 result,
17193 iRegP_R3 tmp1, iRegP_R4 tmp2, iRegP_R5 tmp3,
17194 iRegP_R10 tmp, rFlagsReg cr)
17195 %{
17196 predicate(((AryEqNode*)n)->encoding() == StrIntrinsicNode::UU);
17197 match(Set result (AryEq ary1 ary2));
17198 effect(KILL tmp, USE_KILL ary1, USE_KILL ary2, TEMP tmp1, TEMP tmp2, TEMP tmp3, KILL cr);
17199
17200 format %{ "Array Equals $ary1,ary2 -> $result // KILL $tmp" %}
17201 ins_encode %{
17202 address tpc = __ arrays_equals($ary1$$Register, $ary2$$Register,
17203 $tmp1$$Register, $tmp2$$Register, $tmp3$$Register,
17204 $result$$Register, $tmp$$Register, 2);
17205 if (tpc == NULL) {
17206 ciEnv::current()->record_failure("CodeCache is full");
17207 return;
17208 }
17209 %}
17210 ins_pipe(pipe_class_memory);
17211 %}
17212
17213 instruct count_positives(iRegP_R1 ary1, iRegI_R2 len, iRegI_R0 result, rFlagsReg cr)
17214 %{
17215 match(Set result (CountPositives ary1 len));
17216 effect(USE_KILL ary1, USE_KILL len, KILL cr);
17217 format %{ "count positives byte[] $ary1,$len -> $result" %}
17218 ins_encode %{
17219 address tpc = __ count_positives($ary1$$Register, $len$$Register, $result$$Register);
17220 if (tpc == NULL) {
17221 ciEnv::current()->record_failure("CodeCache is full");
17222 return;
17223 }
17224 %}
17225 ins_pipe( pipe_slow );
17226 %}
17227
17228 // fast char[] to byte[] compression
17229 instruct string_compress(iRegP_R2 src, iRegP_R1 dst, iRegI_R3 len,
17230 vRegD_V0 tmp1, vRegD_V1 tmp2,
17231 vRegD_V2 tmp3, vRegD_V3 tmp4,
17232 iRegI_R0 result, rFlagsReg cr)
17233 %{
17234 match(Set result (StrCompressedCopy src (Binary dst len)));
17235 effect(TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4,
17236 USE_KILL src, USE_KILL dst, USE len, KILL cr);
17237
17238 format %{ "String Compress $src,$dst,$len -> $result // KILL $src,$dst" %}
17239 ins_encode %{
17240 __ char_array_compress($src$$Register, $dst$$Register, $len$$Register,
17241 $result$$Register,
17242 $tmp1$$FloatRegister, $tmp2$$FloatRegister,
17243 $tmp3$$FloatRegister, $tmp4$$FloatRegister);
17244 %}
17245 ins_pipe(pipe_slow);
17246 %}
17247
17248 // fast byte[] to char[] inflation
17249 instruct string_inflate(Universe dummy, iRegP_R0 src, iRegP_R1 dst, iRegI_R2 len,
17250 vRegD_V0 tmp1, vRegD_V1 tmp2, vRegD_V2 tmp3, iRegP_R3 tmp4, rFlagsReg cr)
17251 %{
17252 match(Set dummy (StrInflatedCopy src (Binary dst len)));
17253 effect(TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, USE_KILL src, USE_KILL dst, USE_KILL len, KILL cr);
17254
17255 format %{ "String Inflate $src,$dst // KILL $tmp1, $tmp2" %}
17256 ins_encode %{
17257 address tpc = __ byte_array_inflate($src$$Register, $dst$$Register, $len$$Register,
17258 $tmp1$$FloatRegister, $tmp2$$FloatRegister,
17259 $tmp3$$FloatRegister, $tmp4$$Register);
17260 if (tpc == NULL) {
17261 ciEnv::current()->record_failure("CodeCache is full");
17262 return;
17263 }
17264 %}
17265 ins_pipe(pipe_class_memory);
17266 %}
17267
17268 // encode char[] to byte[] in ISO_8859_1
17269 instruct encode_iso_array(iRegP_R2 src, iRegP_R1 dst, iRegI_R3 len,
17270 vRegD_V0 vtmp0, vRegD_V1 vtmp1,
17271 vRegD_V2 vtmp2, vRegD_V3 vtmp3,
17272 iRegI_R0 result, rFlagsReg cr)
17273 %{
17274 predicate(!((EncodeISOArrayNode*)n)->is_ascii());
17275 match(Set result (EncodeISOArray src (Binary dst len)));
17276 effect(USE_KILL src, USE_KILL dst, USE len,
17277 KILL vtmp0, KILL vtmp1, KILL vtmp2, KILL vtmp3, KILL cr);
17278
17279 format %{ "Encode ISO array $src,$dst,$len -> $result" %}
17280 ins_encode %{
17281 __ encode_iso_array($src$$Register, $dst$$Register, $len$$Register,
17282 $result$$Register, false,
17283 $vtmp0$$FloatRegister, $vtmp1$$FloatRegister,
17284 $vtmp2$$FloatRegister, $vtmp3$$FloatRegister);
17285 %}
17286 ins_pipe(pipe_class_memory);
17287 %}
17288
17289 instruct encode_ascii_array(iRegP_R2 src, iRegP_R1 dst, iRegI_R3 len,
17290 vRegD_V0 vtmp0, vRegD_V1 vtmp1,
17291 vRegD_V2 vtmp2, vRegD_V3 vtmp3,
17292 iRegI_R0 result, rFlagsReg cr)
17293 %{
17294 predicate(((EncodeISOArrayNode*)n)->is_ascii());
17295 match(Set result (EncodeISOArray src (Binary dst len)));
17296 effect(USE_KILL src, USE_KILL dst, USE len,
17297 KILL vtmp0, KILL vtmp1, KILL vtmp2, KILL vtmp3, KILL cr);
17298
17299 format %{ "Encode ASCII array $src,$dst,$len -> $result" %}
17300 ins_encode %{
17301 __ encode_iso_array($src$$Register, $dst$$Register, $len$$Register,
17302 $result$$Register, true,
17303 $vtmp0$$FloatRegister, $vtmp1$$FloatRegister,
17304 $vtmp2$$FloatRegister, $vtmp3$$FloatRegister);
17305 %}
17306 ins_pipe(pipe_class_memory);
17307 %}
17308
17309 // ============================================================================
17310 // This name is KNOWN by the ADLC and cannot be changed.
17311 // The ADLC forces a 'TypeRawPtr::BOTTOM' output type
17312 // for this guy.
17313 instruct tlsLoadP(thread_RegP dst)
17314 %{
17315 match(Set dst (ThreadLocal));
17316
17317 ins_cost(0);
17318
17319 format %{ " -- \t// $dst=Thread::current(), empty" %}
17320
17321 size(0);
17322
17323 ins_encode( /*empty*/ );
17324
17325 ins_pipe(pipe_class_empty);
17326 %}
17327
17328 //----------PEEPHOLE RULES-----------------------------------------------------
17329 // These must follow all instruction definitions as they use the names
17330 // defined in the instructions definitions.
17331 //
17332 // peepmatch ( root_instr_name [preceding_instruction]* );
17333 //
17334 // peepconstraint %{
17335 // (instruction_number.operand_name relational_op instruction_number.operand_name
17336 // [, ...] );
17337 // // instruction numbers are zero-based using left to right order in peepmatch
17338 //
17339 // peepreplace ( instr_name ( [instruction_number.operand_name]* ) );
17340 // // provide an instruction_number.operand_name for each operand that appears
17341 // // in the replacement instruction's match rule
17342 //
17343 // ---------VM FLAGS---------------------------------------------------------
17344 //
17345 // All peephole optimizations can be turned off using -XX:-OptoPeephole
17346 //
17347 // Each peephole rule is given an identifying number starting with zero and
17348 // increasing by one in the order seen by the parser. An individual peephole
17349 // can be enabled, and all others disabled, by using -XX:OptoPeepholeAt=#
17350 // on the command-line.
17351 //
17352 // ---------CURRENT LIMITATIONS----------------------------------------------
17353 //
17354 // Only match adjacent instructions in same basic block
17355 // Only equality constraints
17356 // Only constraints between operands, not (0.dest_reg == RAX_enc)
17357 // Only one replacement instruction
17358 //
17359 // ---------EXAMPLE----------------------------------------------------------
17360 //
17361 // // pertinent parts of existing instructions in architecture description
17362 // instruct movI(iRegINoSp dst, iRegI src)
17363 // %{
17364 // match(Set dst (CopyI src));
17365 // %}
17366 //
17367 // instruct incI_iReg(iRegINoSp dst, immI1 src, rFlagsReg cr)
17368 // %{
17369 // match(Set dst (AddI dst src));
17370 // effect(KILL cr);
17371 // %}
17372 //
17373 // // Change (inc mov) to lea
17374 // peephole %{
17375 // // increment preceded by register-register move
17376 // peepmatch ( incI_iReg movI );
17377 // // require that the destination register of the increment
17378 // // match the destination register of the move
17379 // peepconstraint ( 0.dst == 1.dst );
17380 // // construct a replacement instruction that sets
17381 // // the destination to ( move's source register + one )
17382 // peepreplace ( leaI_iReg_immI( 0.dst 1.src 0.src ) );
17383 // %}
17384 //
17385
17386 // Implementation no longer uses movX instructions since
17387 // machine-independent system no longer uses CopyX nodes.
17388 //
17389 // peephole
17390 // %{
17391 // peepmatch (incI_iReg movI);
17392 // peepconstraint (0.dst == 1.dst);
17393 // peepreplace (leaI_iReg_immI(0.dst 1.src 0.src));
17394 // %}
17395
17396 // peephole
17397 // %{
17398 // peepmatch (decI_iReg movI);
17399 // peepconstraint (0.dst == 1.dst);
17400 // peepreplace (leaI_iReg_immI(0.dst 1.src 0.src));
17401 // %}
17402
17403 // peephole
17404 // %{
17405 // peepmatch (addI_iReg_imm movI);
17406 // peepconstraint (0.dst == 1.dst);
17407 // peepreplace (leaI_iReg_immI(0.dst 1.src 0.src));
17408 // %}
17409
17410 // peephole
17411 // %{
17412 // peepmatch (incL_iReg movL);
17413 // peepconstraint (0.dst == 1.dst);
17414 // peepreplace (leaL_iReg_immL(0.dst 1.src 0.src));
17415 // %}
17416
17417 // peephole
17418 // %{
17419 // peepmatch (decL_iReg movL);
17420 // peepconstraint (0.dst == 1.dst);
17421 // peepreplace (leaL_iReg_immL(0.dst 1.src 0.src));
17422 // %}
17423
17424 // peephole
17425 // %{
17426 // peepmatch (addL_iReg_imm movL);
17427 // peepconstraint (0.dst == 1.dst);
17428 // peepreplace (leaL_iReg_immL(0.dst 1.src 0.src));
17429 // %}
17430
17431 // peephole
17432 // %{
17433 // peepmatch (addP_iReg_imm movP);
17434 // peepconstraint (0.dst == 1.dst);
17435 // peepreplace (leaP_iReg_imm(0.dst 1.src 0.src));
17436 // %}
17437
17438 // // Change load of spilled value to only a spill
17439 // instruct storeI(memory mem, iRegI src)
17440 // %{
17441 // match(Set mem (StoreI mem src));
17442 // %}
17443 //
17444 // instruct loadI(iRegINoSp dst, memory mem)
17445 // %{
17446 // match(Set dst (LoadI mem));
17447 // %}
17448 //
17449
17450 //----------SMARTSPILL RULES---------------------------------------------------
17451 // These must follow all instruction definitions as they use the names
17452 // defined in the instructions definitions.
17453
17454 // Local Variables:
17455 // mode: c++
17456 // End: