1 //
2 // Copyright (c) 2003, 2021, Oracle and/or its affiliates. All rights reserved.
3 // Copyright (c) 2014, 2021, Red Hat, Inc. All rights reserved.
4 // DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
5 //
6 // This code is free software; you can redistribute it and/or modify it
7 // under the terms of the GNU General Public License version 2 only, as
8 // published by the Free Software Foundation.
9 //
10 // This code is distributed in the hope that it will be useful, but WITHOUT
11 // ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
12 // FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
13 // version 2 for more details (a copy is included in the LICENSE file that
14 // accompanied this code).
15 //
16 // You should have received a copy of the GNU General Public License version
17 // 2 along with this work; if not, write to the Free Software Foundation,
18 // Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
19 //
20 // Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
21 // or visit www.oracle.com if you need additional information or have any
22 // questions.
23 //
24 //
25
26 // AArch64 Architecture Description File
27
28 //----------REGISTER DEFINITION BLOCK------------------------------------------
29 // This information is used by the matcher and the register allocator to
30 // describe individual registers and classes of registers within the target
31 // archtecture.
32
33 register %{
34 //----------Architecture Description Register Definitions----------------------
35 // General Registers
36 // "reg_def" name ( register save type, C convention save type,
37 // ideal register type, encoding );
38 // Register Save Types:
39 //
40 // NS = No-Save: The register allocator assumes that these registers
41 // can be used without saving upon entry to the method, &
42 // that they do not need to be saved at call sites.
43 //
44 // SOC = Save-On-Call: The register allocator assumes that these registers
45 // can be used without saving upon entry to the method,
46 // but that they must be saved at call sites.
47 //
48 // SOE = Save-On-Entry: The register allocator assumes that these registers
49 // must be saved before using them upon entry to the
50 // method, but they do not need to be saved at call
51 // sites.
52 //
53 // AS = Always-Save: The register allocator assumes that these registers
54 // must be saved before using them upon entry to the
55 // method, & that they must be saved at call sites.
56 //
57 // Ideal Register Type is used to determine how to save & restore a
58 // register. Op_RegI will get spilled with LoadI/StoreI, Op_RegP will get
59 // spilled with LoadP/StoreP. If the register supports both, use Op_RegI.
60 //
61 // The encoding number is the actual bit-pattern placed into the opcodes.
62
63 // We must define the 64 bit int registers in two 32 bit halves, the
64 // real lower register and a virtual upper half register. upper halves
65 // are used by the register allocator but are not actually supplied as
66 // operands to memory ops.
67 //
68 // follow the C1 compiler in making registers
69 //
70 // r0-r7,r10-r26 volatile (caller save)
71 // r27-r32 system (no save, no allocate)
72 // r8-r9 non-allocatable (so we can use them as scratch regs)
73 //
74 // as regards Java usage. we don't use any callee save registers
75 // because this makes it difficult to de-optimise a frame (see comment
76 // in x86 implementation of Deoptimization::unwind_callee_save_values)
77 //
78
79 // General Registers
80
81 reg_def R0 ( SOC, SOC, Op_RegI, 0, r0->as_VMReg() );
82 reg_def R0_H ( SOC, SOC, Op_RegI, 0, r0->as_VMReg()->next() );
83 reg_def R1 ( SOC, SOC, Op_RegI, 1, r1->as_VMReg() );
84 reg_def R1_H ( SOC, SOC, Op_RegI, 1, r1->as_VMReg()->next() );
85 reg_def R2 ( SOC, SOC, Op_RegI, 2, r2->as_VMReg() );
86 reg_def R2_H ( SOC, SOC, Op_RegI, 2, r2->as_VMReg()->next() );
87 reg_def R3 ( SOC, SOC, Op_RegI, 3, r3->as_VMReg() );
88 reg_def R3_H ( SOC, SOC, Op_RegI, 3, r3->as_VMReg()->next() );
89 reg_def R4 ( SOC, SOC, Op_RegI, 4, r4->as_VMReg() );
90 reg_def R4_H ( SOC, SOC, Op_RegI, 4, r4->as_VMReg()->next() );
91 reg_def R5 ( SOC, SOC, Op_RegI, 5, r5->as_VMReg() );
92 reg_def R5_H ( SOC, SOC, Op_RegI, 5, r5->as_VMReg()->next() );
93 reg_def R6 ( SOC, SOC, Op_RegI, 6, r6->as_VMReg() );
94 reg_def R6_H ( SOC, SOC, Op_RegI, 6, r6->as_VMReg()->next() );
95 reg_def R7 ( SOC, SOC, Op_RegI, 7, r7->as_VMReg() );
96 reg_def R7_H ( SOC, SOC, Op_RegI, 7, r7->as_VMReg()->next() );
97 reg_def R8 ( NS, SOC, Op_RegI, 8, r8->as_VMReg() ); // rscratch1, non-allocatable
98 reg_def R8_H ( NS, SOC, Op_RegI, 8, r8->as_VMReg()->next() );
99 reg_def R9 ( NS, SOC, Op_RegI, 9, r9->as_VMReg() ); // rscratch2, non-allocatable
100 reg_def R9_H ( NS, SOC, Op_RegI, 9, r9->as_VMReg()->next() );
101 reg_def R10 ( SOC, SOC, Op_RegI, 10, r10->as_VMReg() );
102 reg_def R10_H ( SOC, SOC, Op_RegI, 10, r10->as_VMReg()->next());
103 reg_def R11 ( SOC, SOC, Op_RegI, 11, r11->as_VMReg() );
104 reg_def R11_H ( SOC, SOC, Op_RegI, 11, r11->as_VMReg()->next());
105 reg_def R12 ( SOC, SOC, Op_RegI, 12, r12->as_VMReg() );
106 reg_def R12_H ( SOC, SOC, Op_RegI, 12, r12->as_VMReg()->next());
107 reg_def R13 ( SOC, SOC, Op_RegI, 13, r13->as_VMReg() );
108 reg_def R13_H ( SOC, SOC, Op_RegI, 13, r13->as_VMReg()->next());
109 reg_def R14 ( SOC, SOC, Op_RegI, 14, r14->as_VMReg() );
110 reg_def R14_H ( SOC, SOC, Op_RegI, 14, r14->as_VMReg()->next());
111 reg_def R15 ( SOC, SOC, Op_RegI, 15, r15->as_VMReg() );
112 reg_def R15_H ( SOC, SOC, Op_RegI, 15, r15->as_VMReg()->next());
113 reg_def R16 ( SOC, SOC, Op_RegI, 16, r16->as_VMReg() );
114 reg_def R16_H ( SOC, SOC, Op_RegI, 16, r16->as_VMReg()->next());
115 reg_def R17 ( SOC, SOC, Op_RegI, 17, r17->as_VMReg() );
116 reg_def R17_H ( SOC, SOC, Op_RegI, 17, r17->as_VMReg()->next());
117 reg_def R18 ( SOC, SOC, Op_RegI, 18, r18_tls->as_VMReg() );
118 reg_def R18_H ( SOC, SOC, Op_RegI, 18, r18_tls->as_VMReg()->next());
119 reg_def R19 ( SOC, SOE, Op_RegI, 19, r19->as_VMReg() );
120 reg_def R19_H ( SOC, SOE, Op_RegI, 19, r19->as_VMReg()->next());
121 reg_def R20 ( SOC, SOE, Op_RegI, 20, r20->as_VMReg() ); // caller esp
122 reg_def R20_H ( SOC, SOE, Op_RegI, 20, r20->as_VMReg()->next());
123 reg_def R21 ( SOC, SOE, Op_RegI, 21, r21->as_VMReg() );
124 reg_def R21_H ( SOC, SOE, Op_RegI, 21, r21->as_VMReg()->next());
125 reg_def R22 ( SOC, SOE, Op_RegI, 22, r22->as_VMReg() );
126 reg_def R22_H ( SOC, SOE, Op_RegI, 22, r22->as_VMReg()->next());
127 reg_def R23 ( SOC, SOE, Op_RegI, 23, r23->as_VMReg() );
128 reg_def R23_H ( SOC, SOE, Op_RegI, 23, r23->as_VMReg()->next());
129 reg_def R24 ( SOC, SOE, Op_RegI, 24, r24->as_VMReg() );
130 reg_def R24_H ( SOC, SOE, Op_RegI, 24, r24->as_VMReg()->next());
131 reg_def R25 ( SOC, SOE, Op_RegI, 25, r25->as_VMReg() );
132 reg_def R25_H ( SOC, SOE, Op_RegI, 25, r25->as_VMReg()->next());
133 reg_def R26 ( SOC, SOE, Op_RegI, 26, r26->as_VMReg() );
134 reg_def R26_H ( SOC, SOE, Op_RegI, 26, r26->as_VMReg()->next());
135 reg_def R27 ( SOC, SOE, Op_RegI, 27, r27->as_VMReg() ); // heapbase
136 reg_def R27_H ( SOC, SOE, Op_RegI, 27, r27->as_VMReg()->next());
137 reg_def R28 ( NS, SOE, Op_RegI, 28, r28->as_VMReg() ); // thread
138 reg_def R28_H ( NS, SOE, Op_RegI, 28, r28->as_VMReg()->next());
139 reg_def R29 ( NS, NS, Op_RegI, 29, r29->as_VMReg() ); // fp
140 reg_def R29_H ( NS, NS, Op_RegI, 29, r29->as_VMReg()->next());
141 reg_def R30 ( NS, NS, Op_RegI, 30, r30->as_VMReg() ); // lr
142 reg_def R30_H ( NS, NS, Op_RegI, 30, r30->as_VMReg()->next());
143 reg_def R31 ( NS, NS, Op_RegI, 31, r31_sp->as_VMReg() ); // sp
144 reg_def R31_H ( NS, NS, Op_RegI, 31, r31_sp->as_VMReg()->next());
145
146 // ----------------------------
147 // Float/Double/Vector Registers
148 // ----------------------------
149
150 // Double Registers
151
152 // The rules of ADL require that double registers be defined in pairs.
153 // Each pair must be two 32-bit values, but not necessarily a pair of
154 // single float registers. In each pair, ADLC-assigned register numbers
155 // must be adjacent, with the lower number even. Finally, when the
156 // CPU stores such a register pair to memory, the word associated with
157 // the lower ADLC-assigned number must be stored to the lower address.
158
159 // AArch64 has 32 floating-point registers. Each can store a vector of
160 // single or double precision floating-point values up to 8 * 32
161 // floats, 4 * 64 bit floats or 2 * 128 bit floats. We currently only
162 // use the first float or double element of the vector.
163
164 // for Java use float registers v0-v15 are always save on call whereas
165 // the platform ABI treats v8-v15 as callee save). float registers
166 // v16-v31 are SOC as per the platform spec
167
168 // For SVE vector registers, we simply extend vector register size to 8
169 // 'logical' slots. This is nominally 256 bits but it actually covers
170 // all possible 'physical' SVE vector register lengths from 128 ~ 2048
171 // bits. The 'physical' SVE vector register length is detected during
172 // startup, so the register allocator is able to identify the correct
173 // number of bytes needed for an SVE spill/unspill.
174 // Note that a vector register with 4 slots denotes a 128-bit NEON
175 // register allowing it to be distinguished from the corresponding SVE
176 // vector register when the SVE vector length is 128 bits.
177
178 reg_def V0 ( SOC, SOC, Op_RegF, 0, v0->as_VMReg() );
179 reg_def V0_H ( SOC, SOC, Op_RegF, 0, v0->as_VMReg()->next() );
180 reg_def V0_J ( SOC, SOC, Op_RegF, 0, v0->as_VMReg()->next(2) );
181 reg_def V0_K ( SOC, SOC, Op_RegF, 0, v0->as_VMReg()->next(3) );
182 reg_def V0_L ( SOC, SOC, Op_RegF, 0, v0->as_VMReg()->next(4) );
183 reg_def V0_M ( SOC, SOC, Op_RegF, 0, v0->as_VMReg()->next(5) );
184 reg_def V0_N ( SOC, SOC, Op_RegF, 0, v0->as_VMReg()->next(6) );
185 reg_def V0_O ( SOC, SOC, Op_RegF, 0, v0->as_VMReg()->next(7) );
186
187 reg_def V1 ( SOC, SOC, Op_RegF, 1, v1->as_VMReg() );
188 reg_def V1_H ( SOC, SOC, Op_RegF, 1, v1->as_VMReg()->next() );
189 reg_def V1_J ( SOC, SOC, Op_RegF, 1, v1->as_VMReg()->next(2) );
190 reg_def V1_K ( SOC, SOC, Op_RegF, 1, v1->as_VMReg()->next(3) );
191 reg_def V1_L ( SOC, SOC, Op_RegF, 1, v1->as_VMReg()->next(4) );
192 reg_def V1_M ( SOC, SOC, Op_RegF, 1, v1->as_VMReg()->next(5) );
193 reg_def V1_N ( SOC, SOC, Op_RegF, 1, v1->as_VMReg()->next(6) );
194 reg_def V1_O ( SOC, SOC, Op_RegF, 1, v1->as_VMReg()->next(7) );
195
196 reg_def V2 ( SOC, SOC, Op_RegF, 2, v2->as_VMReg() );
197 reg_def V2_H ( SOC, SOC, Op_RegF, 2, v2->as_VMReg()->next() );
198 reg_def V2_J ( SOC, SOC, Op_RegF, 2, v2->as_VMReg()->next(2) );
199 reg_def V2_K ( SOC, SOC, Op_RegF, 2, v2->as_VMReg()->next(3) );
200 reg_def V2_L ( SOC, SOC, Op_RegF, 2, v2->as_VMReg()->next(4) );
201 reg_def V2_M ( SOC, SOC, Op_RegF, 2, v2->as_VMReg()->next(5) );
202 reg_def V2_N ( SOC, SOC, Op_RegF, 2, v2->as_VMReg()->next(6) );
203 reg_def V2_O ( SOC, SOC, Op_RegF, 2, v2->as_VMReg()->next(7) );
204
205 reg_def V3 ( SOC, SOC, Op_RegF, 3, v3->as_VMReg() );
206 reg_def V3_H ( SOC, SOC, Op_RegF, 3, v3->as_VMReg()->next() );
207 reg_def V3_J ( SOC, SOC, Op_RegF, 3, v3->as_VMReg()->next(2) );
208 reg_def V3_K ( SOC, SOC, Op_RegF, 3, v3->as_VMReg()->next(3) );
209 reg_def V3_L ( SOC, SOC, Op_RegF, 3, v3->as_VMReg()->next(4) );
210 reg_def V3_M ( SOC, SOC, Op_RegF, 3, v3->as_VMReg()->next(5) );
211 reg_def V3_N ( SOC, SOC, Op_RegF, 3, v3->as_VMReg()->next(6) );
212 reg_def V3_O ( SOC, SOC, Op_RegF, 3, v3->as_VMReg()->next(7) );
213
214 reg_def V4 ( SOC, SOC, Op_RegF, 4, v4->as_VMReg() );
215 reg_def V4_H ( SOC, SOC, Op_RegF, 4, v4->as_VMReg()->next() );
216 reg_def V4_J ( SOC, SOC, Op_RegF, 4, v4->as_VMReg()->next(2) );
217 reg_def V4_K ( SOC, SOC, Op_RegF, 4, v4->as_VMReg()->next(3) );
218 reg_def V4_L ( SOC, SOC, Op_RegF, 4, v4->as_VMReg()->next(4) );
219 reg_def V4_M ( SOC, SOC, Op_RegF, 4, v4->as_VMReg()->next(5) );
220 reg_def V4_N ( SOC, SOC, Op_RegF, 4, v4->as_VMReg()->next(6) );
221 reg_def V4_O ( SOC, SOC, Op_RegF, 4, v4->as_VMReg()->next(7) );
222
223 reg_def V5 ( SOC, SOC, Op_RegF, 5, v5->as_VMReg() );
224 reg_def V5_H ( SOC, SOC, Op_RegF, 5, v5->as_VMReg()->next() );
225 reg_def V5_J ( SOC, SOC, Op_RegF, 5, v5->as_VMReg()->next(2) );
226 reg_def V5_K ( SOC, SOC, Op_RegF, 5, v5->as_VMReg()->next(3) );
227 reg_def V5_L ( SOC, SOC, Op_RegF, 5, v5->as_VMReg()->next(4) );
228 reg_def V5_M ( SOC, SOC, Op_RegF, 5, v5->as_VMReg()->next(5) );
229 reg_def V5_N ( SOC, SOC, Op_RegF, 5, v5->as_VMReg()->next(6) );
230 reg_def V5_O ( SOC, SOC, Op_RegF, 5, v5->as_VMReg()->next(7) );
231
232 reg_def V6 ( SOC, SOC, Op_RegF, 6, v6->as_VMReg() );
233 reg_def V6_H ( SOC, SOC, Op_RegF, 6, v6->as_VMReg()->next() );
234 reg_def V6_J ( SOC, SOC, Op_RegF, 6, v6->as_VMReg()->next(2) );
235 reg_def V6_K ( SOC, SOC, Op_RegF, 6, v6->as_VMReg()->next(3) );
236 reg_def V6_L ( SOC, SOC, Op_RegF, 6, v6->as_VMReg()->next(4) );
237 reg_def V6_M ( SOC, SOC, Op_RegF, 6, v6->as_VMReg()->next(5) );
238 reg_def V6_N ( SOC, SOC, Op_RegF, 6, v6->as_VMReg()->next(6) );
239 reg_def V6_O ( SOC, SOC, Op_RegF, 6, v6->as_VMReg()->next(7) );
240
241 reg_def V7 ( SOC, SOC, Op_RegF, 7, v7->as_VMReg() );
242 reg_def V7_H ( SOC, SOC, Op_RegF, 7, v7->as_VMReg()->next() );
243 reg_def V7_J ( SOC, SOC, Op_RegF, 7, v7->as_VMReg()->next(2) );
244 reg_def V7_K ( SOC, SOC, Op_RegF, 7, v7->as_VMReg()->next(3) );
245 reg_def V7_L ( SOC, SOC, Op_RegF, 7, v7->as_VMReg()->next(4) );
246 reg_def V7_M ( SOC, SOC, Op_RegF, 7, v7->as_VMReg()->next(5) );
247 reg_def V7_N ( SOC, SOC, Op_RegF, 7, v7->as_VMReg()->next(6) );
248 reg_def V7_O ( SOC, SOC, Op_RegF, 7, v7->as_VMReg()->next(7) );
249
250 reg_def V8 ( SOC, SOE, Op_RegF, 8, v8->as_VMReg() );
251 reg_def V8_H ( SOC, SOE, Op_RegF, 8, v8->as_VMReg()->next() );
252 reg_def V8_J ( SOC, SOC, Op_RegF, 8, v8->as_VMReg()->next(2) );
253 reg_def V8_K ( SOC, SOC, Op_RegF, 8, v8->as_VMReg()->next(3) );
254 reg_def V8_L ( SOC, SOC, Op_RegF, 8, v8->as_VMReg()->next(4) );
255 reg_def V8_M ( SOC, SOC, Op_RegF, 8, v8->as_VMReg()->next(5) );
256 reg_def V8_N ( SOC, SOC, Op_RegF, 8, v8->as_VMReg()->next(6) );
257 reg_def V8_O ( SOC, SOC, Op_RegF, 8, v8->as_VMReg()->next(7) );
258
259 reg_def V9 ( SOC, SOE, Op_RegF, 9, v9->as_VMReg() );
260 reg_def V9_H ( SOC, SOE, Op_RegF, 9, v9->as_VMReg()->next() );
261 reg_def V9_J ( SOC, SOC, Op_RegF, 9, v9->as_VMReg()->next(2) );
262 reg_def V9_K ( SOC, SOC, Op_RegF, 9, v9->as_VMReg()->next(3) );
263 reg_def V9_L ( SOC, SOC, Op_RegF, 9, v9->as_VMReg()->next(4) );
264 reg_def V9_M ( SOC, SOC, Op_RegF, 9, v9->as_VMReg()->next(5) );
265 reg_def V9_N ( SOC, SOC, Op_RegF, 9, v9->as_VMReg()->next(6) );
266 reg_def V9_O ( SOC, SOC, Op_RegF, 9, v9->as_VMReg()->next(7) );
267
268 reg_def V10 ( SOC, SOE, Op_RegF, 10, v10->as_VMReg() );
269 reg_def V10_H ( SOC, SOE, Op_RegF, 10, v10->as_VMReg()->next() );
270 reg_def V10_J ( SOC, SOC, Op_RegF, 10, v10->as_VMReg()->next(2) );
271 reg_def V10_K ( SOC, SOC, Op_RegF, 10, v10->as_VMReg()->next(3) );
272 reg_def V10_L ( SOC, SOC, Op_RegF, 10, v10->as_VMReg()->next(4) );
273 reg_def V10_M ( SOC, SOC, Op_RegF, 10, v10->as_VMReg()->next(5) );
274 reg_def V10_N ( SOC, SOC, Op_RegF, 10, v10->as_VMReg()->next(6) );
275 reg_def V10_O ( SOC, SOC, Op_RegF, 10, v10->as_VMReg()->next(7) );
276
277 reg_def V11 ( SOC, SOE, Op_RegF, 11, v11->as_VMReg() );
278 reg_def V11_H ( SOC, SOE, Op_RegF, 11, v11->as_VMReg()->next() );
279 reg_def V11_J ( SOC, SOC, Op_RegF, 11, v11->as_VMReg()->next(2) );
280 reg_def V11_K ( SOC, SOC, Op_RegF, 11, v11->as_VMReg()->next(3) );
281 reg_def V11_L ( SOC, SOC, Op_RegF, 11, v11->as_VMReg()->next(4) );
282 reg_def V11_M ( SOC, SOC, Op_RegF, 11, v11->as_VMReg()->next(5) );
283 reg_def V11_N ( SOC, SOC, Op_RegF, 11, v11->as_VMReg()->next(6) );
284 reg_def V11_O ( SOC, SOC, Op_RegF, 11, v11->as_VMReg()->next(7) );
285
286 reg_def V12 ( SOC, SOE, Op_RegF, 12, v12->as_VMReg() );
287 reg_def V12_H ( SOC, SOE, Op_RegF, 12, v12->as_VMReg()->next() );
288 reg_def V12_J ( SOC, SOC, Op_RegF, 12, v12->as_VMReg()->next(2) );
289 reg_def V12_K ( SOC, SOC, Op_RegF, 12, v12->as_VMReg()->next(3) );
290 reg_def V12_L ( SOC, SOC, Op_RegF, 12, v12->as_VMReg()->next(4) );
291 reg_def V12_M ( SOC, SOC, Op_RegF, 12, v12->as_VMReg()->next(5) );
292 reg_def V12_N ( SOC, SOC, Op_RegF, 12, v12->as_VMReg()->next(6) );
293 reg_def V12_O ( SOC, SOC, Op_RegF, 12, v12->as_VMReg()->next(7) );
294
295 reg_def V13 ( SOC, SOE, Op_RegF, 13, v13->as_VMReg() );
296 reg_def V13_H ( SOC, SOE, Op_RegF, 13, v13->as_VMReg()->next() );
297 reg_def V13_J ( SOC, SOC, Op_RegF, 13, v13->as_VMReg()->next(2) );
298 reg_def V13_K ( SOC, SOC, Op_RegF, 13, v13->as_VMReg()->next(3) );
299 reg_def V13_L ( SOC, SOC, Op_RegF, 13, v13->as_VMReg()->next(4) );
300 reg_def V13_M ( SOC, SOC, Op_RegF, 13, v13->as_VMReg()->next(5) );
301 reg_def V13_N ( SOC, SOC, Op_RegF, 13, v13->as_VMReg()->next(6) );
302 reg_def V13_O ( SOC, SOC, Op_RegF, 13, v13->as_VMReg()->next(7) );
303
304 reg_def V14 ( SOC, SOE, Op_RegF, 14, v14->as_VMReg() );
305 reg_def V14_H ( SOC, SOE, Op_RegF, 14, v14->as_VMReg()->next() );
306 reg_def V14_J ( SOC, SOC, Op_RegF, 14, v14->as_VMReg()->next(2) );
307 reg_def V14_K ( SOC, SOC, Op_RegF, 14, v14->as_VMReg()->next(3) );
308 reg_def V14_L ( SOC, SOC, Op_RegF, 14, v14->as_VMReg()->next(4) );
309 reg_def V14_M ( SOC, SOC, Op_RegF, 14, v14->as_VMReg()->next(5) );
310 reg_def V14_N ( SOC, SOC, Op_RegF, 14, v14->as_VMReg()->next(6) );
311 reg_def V14_O ( SOC, SOC, Op_RegF, 14, v14->as_VMReg()->next(7) );
312
313 reg_def V15 ( SOC, SOE, Op_RegF, 15, v15->as_VMReg() );
314 reg_def V15_H ( SOC, SOE, Op_RegF, 15, v15->as_VMReg()->next() );
315 reg_def V15_J ( SOC, SOC, Op_RegF, 15, v15->as_VMReg()->next(2) );
316 reg_def V15_K ( SOC, SOC, Op_RegF, 15, v15->as_VMReg()->next(3) );
317 reg_def V15_L ( SOC, SOC, Op_RegF, 15, v15->as_VMReg()->next(4) );
318 reg_def V15_M ( SOC, SOC, Op_RegF, 15, v15->as_VMReg()->next(5) );
319 reg_def V15_N ( SOC, SOC, Op_RegF, 15, v15->as_VMReg()->next(6) );
320 reg_def V15_O ( SOC, SOC, Op_RegF, 15, v15->as_VMReg()->next(7) );
321
322 reg_def V16 ( SOC, SOC, Op_RegF, 16, v16->as_VMReg() );
323 reg_def V16_H ( SOC, SOC, Op_RegF, 16, v16->as_VMReg()->next() );
324 reg_def V16_J ( SOC, SOC, Op_RegF, 16, v16->as_VMReg()->next(2) );
325 reg_def V16_K ( SOC, SOC, Op_RegF, 16, v16->as_VMReg()->next(3) );
326 reg_def V16_L ( SOC, SOC, Op_RegF, 16, v16->as_VMReg()->next(4) );
327 reg_def V16_M ( SOC, SOC, Op_RegF, 16, v16->as_VMReg()->next(5) );
328 reg_def V16_N ( SOC, SOC, Op_RegF, 16, v16->as_VMReg()->next(6) );
329 reg_def V16_O ( SOC, SOC, Op_RegF, 16, v16->as_VMReg()->next(7) );
330
331 reg_def V17 ( SOC, SOC, Op_RegF, 17, v17->as_VMReg() );
332 reg_def V17_H ( SOC, SOC, Op_RegF, 17, v17->as_VMReg()->next() );
333 reg_def V17_J ( SOC, SOC, Op_RegF, 17, v17->as_VMReg()->next(2) );
334 reg_def V17_K ( SOC, SOC, Op_RegF, 17, v17->as_VMReg()->next(3) );
335 reg_def V17_L ( SOC, SOC, Op_RegF, 17, v17->as_VMReg()->next(4) );
336 reg_def V17_M ( SOC, SOC, Op_RegF, 17, v17->as_VMReg()->next(5) );
337 reg_def V17_N ( SOC, SOC, Op_RegF, 17, v17->as_VMReg()->next(6) );
338 reg_def V17_O ( SOC, SOC, Op_RegF, 17, v17->as_VMReg()->next(7) );
339
340 reg_def V18 ( SOC, SOC, Op_RegF, 18, v18->as_VMReg() );
341 reg_def V18_H ( SOC, SOC, Op_RegF, 18, v18->as_VMReg()->next() );
342 reg_def V18_J ( SOC, SOC, Op_RegF, 18, v18->as_VMReg()->next(2) );
343 reg_def V18_K ( SOC, SOC, Op_RegF, 18, v18->as_VMReg()->next(3) );
344 reg_def V18_L ( SOC, SOC, Op_RegF, 18, v18->as_VMReg()->next(4) );
345 reg_def V18_M ( SOC, SOC, Op_RegF, 18, v18->as_VMReg()->next(5) );
346 reg_def V18_N ( SOC, SOC, Op_RegF, 18, v18->as_VMReg()->next(6) );
347 reg_def V18_O ( SOC, SOC, Op_RegF, 18, v18->as_VMReg()->next(7) );
348
349 reg_def V19 ( SOC, SOC, Op_RegF, 19, v19->as_VMReg() );
350 reg_def V19_H ( SOC, SOC, Op_RegF, 19, v19->as_VMReg()->next() );
351 reg_def V19_J ( SOC, SOC, Op_RegF, 19, v19->as_VMReg()->next(2) );
352 reg_def V19_K ( SOC, SOC, Op_RegF, 19, v19->as_VMReg()->next(3) );
353 reg_def V19_L ( SOC, SOC, Op_RegF, 19, v19->as_VMReg()->next(4) );
354 reg_def V19_M ( SOC, SOC, Op_RegF, 19, v19->as_VMReg()->next(5) );
355 reg_def V19_N ( SOC, SOC, Op_RegF, 19, v19->as_VMReg()->next(6) );
356 reg_def V19_O ( SOC, SOC, Op_RegF, 19, v19->as_VMReg()->next(7) );
357
358 reg_def V20 ( SOC, SOC, Op_RegF, 20, v20->as_VMReg() );
359 reg_def V20_H ( SOC, SOC, Op_RegF, 20, v20->as_VMReg()->next() );
360 reg_def V20_J ( SOC, SOC, Op_RegF, 20, v20->as_VMReg()->next(2) );
361 reg_def V20_K ( SOC, SOC, Op_RegF, 20, v20->as_VMReg()->next(3) );
362 reg_def V20_L ( SOC, SOC, Op_RegF, 20, v20->as_VMReg()->next(4) );
363 reg_def V20_M ( SOC, SOC, Op_RegF, 20, v20->as_VMReg()->next(5) );
364 reg_def V20_N ( SOC, SOC, Op_RegF, 20, v20->as_VMReg()->next(6) );
365 reg_def V20_O ( SOC, SOC, Op_RegF, 20, v20->as_VMReg()->next(7) );
366
367 reg_def V21 ( SOC, SOC, Op_RegF, 21, v21->as_VMReg() );
368 reg_def V21_H ( SOC, SOC, Op_RegF, 21, v21->as_VMReg()->next() );
369 reg_def V21_J ( SOC, SOC, Op_RegF, 21, v21->as_VMReg()->next(2) );
370 reg_def V21_K ( SOC, SOC, Op_RegF, 21, v21->as_VMReg()->next(3) );
371 reg_def V21_L ( SOC, SOC, Op_RegF, 21, v21->as_VMReg()->next(4) );
372 reg_def V21_M ( SOC, SOC, Op_RegF, 21, v21->as_VMReg()->next(5) );
373 reg_def V21_N ( SOC, SOC, Op_RegF, 21, v21->as_VMReg()->next(6) );
374 reg_def V21_O ( SOC, SOC, Op_RegF, 21, v21->as_VMReg()->next(7) );
375
376 reg_def V22 ( SOC, SOC, Op_RegF, 22, v22->as_VMReg() );
377 reg_def V22_H ( SOC, SOC, Op_RegF, 22, v22->as_VMReg()->next() );
378 reg_def V22_J ( SOC, SOC, Op_RegF, 22, v22->as_VMReg()->next(2) );
379 reg_def V22_K ( SOC, SOC, Op_RegF, 22, v22->as_VMReg()->next(3) );
380 reg_def V22_L ( SOC, SOC, Op_RegF, 22, v22->as_VMReg()->next(4) );
381 reg_def V22_M ( SOC, SOC, Op_RegF, 22, v22->as_VMReg()->next(5) );
382 reg_def V22_N ( SOC, SOC, Op_RegF, 22, v22->as_VMReg()->next(6) );
383 reg_def V22_O ( SOC, SOC, Op_RegF, 22, v22->as_VMReg()->next(7) );
384
385 reg_def V23 ( SOC, SOC, Op_RegF, 23, v23->as_VMReg() );
386 reg_def V23_H ( SOC, SOC, Op_RegF, 23, v23->as_VMReg()->next() );
387 reg_def V23_J ( SOC, SOC, Op_RegF, 23, v23->as_VMReg()->next(2) );
388 reg_def V23_K ( SOC, SOC, Op_RegF, 23, v23->as_VMReg()->next(3) );
389 reg_def V23_L ( SOC, SOC, Op_RegF, 23, v23->as_VMReg()->next(4) );
390 reg_def V23_M ( SOC, SOC, Op_RegF, 23, v23->as_VMReg()->next(5) );
391 reg_def V23_N ( SOC, SOC, Op_RegF, 23, v23->as_VMReg()->next(6) );
392 reg_def V23_O ( SOC, SOC, Op_RegF, 23, v23->as_VMReg()->next(7) );
393
394 reg_def V24 ( SOC, SOC, Op_RegF, 24, v24->as_VMReg() );
395 reg_def V24_H ( SOC, SOC, Op_RegF, 24, v24->as_VMReg()->next() );
396 reg_def V24_J ( SOC, SOC, Op_RegF, 24, v24->as_VMReg()->next(2) );
397 reg_def V24_K ( SOC, SOC, Op_RegF, 24, v24->as_VMReg()->next(3) );
398 reg_def V24_L ( SOC, SOC, Op_RegF, 24, v24->as_VMReg()->next(4) );
399 reg_def V24_M ( SOC, SOC, Op_RegF, 24, v24->as_VMReg()->next(5) );
400 reg_def V24_N ( SOC, SOC, Op_RegF, 24, v24->as_VMReg()->next(6) );
401 reg_def V24_O ( SOC, SOC, Op_RegF, 24, v24->as_VMReg()->next(7) );
402
403 reg_def V25 ( SOC, SOC, Op_RegF, 25, v25->as_VMReg() );
404 reg_def V25_H ( SOC, SOC, Op_RegF, 25, v25->as_VMReg()->next() );
405 reg_def V25_J ( SOC, SOC, Op_RegF, 25, v25->as_VMReg()->next(2) );
406 reg_def V25_K ( SOC, SOC, Op_RegF, 25, v25->as_VMReg()->next(3) );
407 reg_def V25_L ( SOC, SOC, Op_RegF, 25, v25->as_VMReg()->next(4) );
408 reg_def V25_M ( SOC, SOC, Op_RegF, 25, v25->as_VMReg()->next(5) );
409 reg_def V25_N ( SOC, SOC, Op_RegF, 25, v25->as_VMReg()->next(6) );
410 reg_def V25_O ( SOC, SOC, Op_RegF, 25, v25->as_VMReg()->next(7) );
411
412 reg_def V26 ( SOC, SOC, Op_RegF, 26, v26->as_VMReg() );
413 reg_def V26_H ( SOC, SOC, Op_RegF, 26, v26->as_VMReg()->next() );
414 reg_def V26_J ( SOC, SOC, Op_RegF, 26, v26->as_VMReg()->next(2) );
415 reg_def V26_K ( SOC, SOC, Op_RegF, 26, v26->as_VMReg()->next(3) );
416 reg_def V26_L ( SOC, SOC, Op_RegF, 26, v26->as_VMReg()->next(4) );
417 reg_def V26_M ( SOC, SOC, Op_RegF, 26, v26->as_VMReg()->next(5) );
418 reg_def V26_N ( SOC, SOC, Op_RegF, 26, v26->as_VMReg()->next(6) );
419 reg_def V26_O ( SOC, SOC, Op_RegF, 26, v26->as_VMReg()->next(7) );
420
421 reg_def V27 ( SOC, SOC, Op_RegF, 27, v27->as_VMReg() );
422 reg_def V27_H ( SOC, SOC, Op_RegF, 27, v27->as_VMReg()->next() );
423 reg_def V27_J ( SOC, SOC, Op_RegF, 27, v27->as_VMReg()->next(2) );
424 reg_def V27_K ( SOC, SOC, Op_RegF, 27, v27->as_VMReg()->next(3) );
425 reg_def V27_L ( SOC, SOC, Op_RegF, 27, v27->as_VMReg()->next(4) );
426 reg_def V27_M ( SOC, SOC, Op_RegF, 27, v27->as_VMReg()->next(5) );
427 reg_def V27_N ( SOC, SOC, Op_RegF, 27, v27->as_VMReg()->next(6) );
428 reg_def V27_O ( SOC, SOC, Op_RegF, 27, v27->as_VMReg()->next(7) );
429
430 reg_def V28 ( SOC, SOC, Op_RegF, 28, v28->as_VMReg() );
431 reg_def V28_H ( SOC, SOC, Op_RegF, 28, v28->as_VMReg()->next() );
432 reg_def V28_J ( SOC, SOC, Op_RegF, 28, v28->as_VMReg()->next(2) );
433 reg_def V28_K ( SOC, SOC, Op_RegF, 28, v28->as_VMReg()->next(3) );
434 reg_def V28_L ( SOC, SOC, Op_RegF, 28, v28->as_VMReg()->next(4) );
435 reg_def V28_M ( SOC, SOC, Op_RegF, 28, v28->as_VMReg()->next(5) );
436 reg_def V28_N ( SOC, SOC, Op_RegF, 28, v28->as_VMReg()->next(6) );
437 reg_def V28_O ( SOC, SOC, Op_RegF, 28, v28->as_VMReg()->next(7) );
438
439 reg_def V29 ( SOC, SOC, Op_RegF, 29, v29->as_VMReg() );
440 reg_def V29_H ( SOC, SOC, Op_RegF, 29, v29->as_VMReg()->next() );
441 reg_def V29_J ( SOC, SOC, Op_RegF, 29, v29->as_VMReg()->next(2) );
442 reg_def V29_K ( SOC, SOC, Op_RegF, 29, v29->as_VMReg()->next(3) );
443 reg_def V29_L ( SOC, SOC, Op_RegF, 29, v29->as_VMReg()->next(4) );
444 reg_def V29_M ( SOC, SOC, Op_RegF, 29, v29->as_VMReg()->next(5) );
445 reg_def V29_N ( SOC, SOC, Op_RegF, 29, v29->as_VMReg()->next(6) );
446 reg_def V29_O ( SOC, SOC, Op_RegF, 29, v29->as_VMReg()->next(7) );
447
448 reg_def V30 ( SOC, SOC, Op_RegF, 30, v30->as_VMReg() );
449 reg_def V30_H ( SOC, SOC, Op_RegF, 30, v30->as_VMReg()->next() );
450 reg_def V30_J ( SOC, SOC, Op_RegF, 30, v30->as_VMReg()->next(2) );
451 reg_def V30_K ( SOC, SOC, Op_RegF, 30, v30->as_VMReg()->next(3) );
452 reg_def V30_L ( SOC, SOC, Op_RegF, 30, v30->as_VMReg()->next(4) );
453 reg_def V30_M ( SOC, SOC, Op_RegF, 30, v30->as_VMReg()->next(5) );
454 reg_def V30_N ( SOC, SOC, Op_RegF, 30, v30->as_VMReg()->next(6) );
455 reg_def V30_O ( SOC, SOC, Op_RegF, 30, v30->as_VMReg()->next(7) );
456
457 reg_def V31 ( SOC, SOC, Op_RegF, 31, v31->as_VMReg() );
458 reg_def V31_H ( SOC, SOC, Op_RegF, 31, v31->as_VMReg()->next() );
459 reg_def V31_J ( SOC, SOC, Op_RegF, 31, v31->as_VMReg()->next(2) );
460 reg_def V31_K ( SOC, SOC, Op_RegF, 31, v31->as_VMReg()->next(3) );
461 reg_def V31_L ( SOC, SOC, Op_RegF, 31, v31->as_VMReg()->next(4) );
462 reg_def V31_M ( SOC, SOC, Op_RegF, 31, v31->as_VMReg()->next(5) );
463 reg_def V31_N ( SOC, SOC, Op_RegF, 31, v31->as_VMReg()->next(6) );
464 reg_def V31_O ( SOC, SOC, Op_RegF, 31, v31->as_VMReg()->next(7) );
465
466
467 // ----------------------------
468 // SVE Predicate Registers
469 // ----------------------------
470 reg_def P0 (SOC, SOC, Op_RegVectMask, 0, p0->as_VMReg());
471 reg_def P1 (SOC, SOC, Op_RegVectMask, 1, p1->as_VMReg());
472 reg_def P2 (SOC, SOC, Op_RegVectMask, 2, p2->as_VMReg());
473 reg_def P3 (SOC, SOC, Op_RegVectMask, 3, p3->as_VMReg());
474 reg_def P4 (SOC, SOC, Op_RegVectMask, 4, p4->as_VMReg());
475 reg_def P5 (SOC, SOC, Op_RegVectMask, 5, p5->as_VMReg());
476 reg_def P6 (SOC, SOC, Op_RegVectMask, 6, p6->as_VMReg());
477 reg_def P7 (SOC, SOC, Op_RegVectMask, 7, p7->as_VMReg());
478 reg_def P8 (SOC, SOC, Op_RegVectMask, 8, p8->as_VMReg());
479 reg_def P9 (SOC, SOC, Op_RegVectMask, 9, p9->as_VMReg());
480 reg_def P10 (SOC, SOC, Op_RegVectMask, 10, p10->as_VMReg());
481 reg_def P11 (SOC, SOC, Op_RegVectMask, 11, p11->as_VMReg());
482 reg_def P12 (SOC, SOC, Op_RegVectMask, 12, p12->as_VMReg());
483 reg_def P13 (SOC, SOC, Op_RegVectMask, 13, p13->as_VMReg());
484 reg_def P14 (SOC, SOC, Op_RegVectMask, 14, p14->as_VMReg());
485 reg_def P15 (SOC, SOC, Op_RegVectMask, 15, p15->as_VMReg());
486
487 // ----------------------------
488 // Special Registers
489 // ----------------------------
490
491 // the AArch64 CSPR status flag register is not directly acessible as
492 // instruction operand. the FPSR status flag register is a system
493 // register which can be written/read using MSR/MRS but again does not
494 // appear as an operand (a code identifying the FSPR occurs as an
495 // immediate value in the instruction).
496
497 reg_def RFLAGS(SOC, SOC, 0, 32, VMRegImpl::Bad());
498
499 // Specify priority of register selection within phases of register
500 // allocation. Highest priority is first. A useful heuristic is to
501 // give registers a low priority when they are required by machine
502 // instructions, like EAX and EDX on I486, and choose no-save registers
503 // before save-on-call, & save-on-call before save-on-entry. Registers
504 // which participate in fixed calling sequences should come last.
505 // Registers which are used as pairs must fall on an even boundary.
506
507 alloc_class chunk0(
508 // volatiles
509 R10, R10_H,
510 R11, R11_H,
511 R12, R12_H,
512 R13, R13_H,
513 R14, R14_H,
514 R15, R15_H,
515 R16, R16_H,
516 R17, R17_H,
517 R18, R18_H,
518
519 // arg registers
520 R0, R0_H,
521 R1, R1_H,
522 R2, R2_H,
523 R3, R3_H,
524 R4, R4_H,
525 R5, R5_H,
526 R6, R6_H,
527 R7, R7_H,
528
529 // non-volatiles
530 R19, R19_H,
531 R20, R20_H,
532 R21, R21_H,
533 R22, R22_H,
534 R23, R23_H,
535 R24, R24_H,
536 R25, R25_H,
537 R26, R26_H,
538
539 // non-allocatable registers
540
541 R27, R27_H, // heapbase
542 R28, R28_H, // thread
543 R29, R29_H, // fp
544 R30, R30_H, // lr
545 R31, R31_H, // sp
546 R8, R8_H, // rscratch1
547 R9, R9_H, // rscratch2
548 );
549
550 alloc_class chunk1(
551
552 // no save
553 V16, V16_H, V16_J, V16_K, V16_L, V16_M, V16_N, V16_O,
554 V17, V17_H, V17_J, V17_K, V17_L, V17_M, V17_N, V17_O,
555 V18, V18_H, V18_J, V18_K, V18_L, V18_M, V18_N, V18_O,
556 V19, V19_H, V19_J, V19_K, V19_L, V19_M, V19_N, V19_O,
557 V20, V20_H, V20_J, V20_K, V20_L, V20_M, V20_N, V20_O,
558 V21, V21_H, V21_J, V21_K, V21_L, V21_M, V21_N, V21_O,
559 V22, V22_H, V22_J, V22_K, V22_L, V22_M, V22_N, V22_O,
560 V23, V23_H, V23_J, V23_K, V23_L, V23_M, V23_N, V23_O,
561 V24, V24_H, V24_J, V24_K, V24_L, V24_M, V24_N, V24_O,
562 V25, V25_H, V25_J, V25_K, V25_L, V25_M, V25_N, V25_O,
563 V26, V26_H, V26_J, V26_K, V26_L, V26_M, V26_N, V26_O,
564 V27, V27_H, V27_J, V27_K, V27_L, V27_M, V27_N, V27_O,
565 V28, V28_H, V28_J, V28_K, V28_L, V28_M, V28_N, V28_O,
566 V29, V29_H, V29_J, V29_K, V29_L, V29_M, V29_N, V29_O,
567 V30, V30_H, V30_J, V30_K, V30_L, V30_M, V30_N, V30_O,
568 V31, V31_H, V31_J, V31_K, V31_L, V31_M, V31_N, V31_O,
569
570 // arg registers
571 V0, V0_H, V0_J, V0_K, V0_L, V0_M, V0_N, V0_O,
572 V1, V1_H, V1_J, V1_K, V1_L, V1_M, V1_N, V1_O,
573 V2, V2_H, V2_J, V2_K, V2_L, V2_M, V2_N, V2_O,
574 V3, V3_H, V3_J, V3_K, V3_L, V3_M, V3_N, V3_O,
575 V4, V4_H, V4_J, V4_K, V4_L, V4_M, V4_N, V4_O,
576 V5, V5_H, V5_J, V5_K, V5_L, V5_M, V5_N, V5_O,
577 V6, V6_H, V6_J, V6_K, V6_L, V6_M, V6_N, V6_O,
578 V7, V7_H, V7_J, V7_K, V7_L, V7_M, V7_N, V7_O,
579
580 // non-volatiles
581 V8, V8_H, V8_J, V8_K, V8_L, V8_M, V8_N, V8_O,
582 V9, V9_H, V9_J, V9_K, V9_L, V9_M, V9_N, V9_O,
583 V10, V10_H, V10_J, V10_K, V10_L, V10_M, V10_N, V10_O,
584 V11, V11_H, V11_J, V11_K, V11_L, V11_M, V11_N, V11_O,
585 V12, V12_H, V12_J, V12_K, V12_L, V12_M, V12_N, V12_O,
586 V13, V13_H, V13_J, V13_K, V13_L, V13_M, V13_N, V13_O,
587 V14, V14_H, V14_J, V14_K, V14_L, V14_M, V14_N, V14_O,
588 V15, V15_H, V15_J, V15_K, V15_L, V15_M, V15_N, V15_O,
589 );
590
591 alloc_class chunk2 (
592 P0,
593 P1,
594 P2,
595 P3,
596 P4,
597 P5,
598 P6,
599 P7,
600
601 P8,
602 P9,
603 P10,
604 P11,
605 P12,
606 P13,
607 P14,
608 P15,
609 );
610
611 alloc_class chunk3(RFLAGS);
612
613 //----------Architecture Description Register Classes--------------------------
614 // Several register classes are automatically defined based upon information in
615 // this architecture description.
616 // 1) reg_class inline_cache_reg ( /* as def'd in frame section */ )
617 // 2) reg_class stack_slots( /* one chunk of stack-based "registers" */ )
618 //
619
620 // Class for all 32 bit general purpose registers
621 reg_class all_reg32(
622 R0,
623 R1,
624 R2,
625 R3,
626 R4,
627 R5,
628 R6,
629 R7,
630 R10,
631 R11,
632 R12,
633 R13,
634 R14,
635 R15,
636 R16,
637 R17,
638 R18,
639 R19,
640 R20,
641 R21,
642 R22,
643 R23,
644 R24,
645 R25,
646 R26,
647 R27,
648 R28,
649 R29,
650 R30,
651 R31
652 );
653
654
655 // Class for all 32 bit integer registers (excluding SP which
656 // will never be used as an integer register)
657 reg_class any_reg32 %{
658 return _ANY_REG32_mask;
659 %}
660
661 // Singleton class for R0 int register
662 reg_class int_r0_reg(R0);
663
664 // Singleton class for R2 int register
665 reg_class int_r2_reg(R2);
666
667 // Singleton class for R3 int register
668 reg_class int_r3_reg(R3);
669
670 // Singleton class for R4 int register
671 reg_class int_r4_reg(R4);
672
673 // Singleton class for R31 int register
674 reg_class int_r31_reg(R31);
675
676 // Class for all 64 bit general purpose registers
677 reg_class all_reg(
678 R0, R0_H,
679 R1, R1_H,
680 R2, R2_H,
681 R3, R3_H,
682 R4, R4_H,
683 R5, R5_H,
684 R6, R6_H,
685 R7, R7_H,
686 R10, R10_H,
687 R11, R11_H,
688 R12, R12_H,
689 R13, R13_H,
690 R14, R14_H,
691 R15, R15_H,
692 R16, R16_H,
693 R17, R17_H,
694 R18, R18_H,
695 R19, R19_H,
696 R20, R20_H,
697 R21, R21_H,
698 R22, R22_H,
699 R23, R23_H,
700 R24, R24_H,
701 R25, R25_H,
702 R26, R26_H,
703 R27, R27_H,
704 R28, R28_H,
705 R29, R29_H,
706 R30, R30_H,
707 R31, R31_H
708 );
709
710 // Class for all long integer registers (including SP)
711 reg_class any_reg %{
712 return _ANY_REG_mask;
713 %}
714
715 // Class for non-allocatable 32 bit registers
716 reg_class non_allocatable_reg32(
717 #ifdef R18_RESERVED
718 // See comment in register_aarch64.hpp
719 R18, // tls on Windows
720 #endif
721 R28, // thread
722 R30, // lr
723 R31 // sp
724 );
725
726 // Class for non-allocatable 64 bit registers
727 reg_class non_allocatable_reg(
728 #ifdef R18_RESERVED
729 // See comment in register_aarch64.hpp
730 R18, R18_H, // tls on Windows, platform register on macOS
731 #endif
732 R28, R28_H, // thread
733 R30, R30_H, // lr
734 R31, R31_H // sp
735 );
736
737 // Class for all non-special integer registers
738 reg_class no_special_reg32 %{
739 return _NO_SPECIAL_REG32_mask;
740 %}
741
742 // Class for all non-special long integer registers
743 reg_class no_special_reg %{
744 return _NO_SPECIAL_REG_mask;
745 %}
746
747 // Class for 64 bit register r0
748 reg_class r0_reg(
749 R0, R0_H
750 );
751
752 // Class for 64 bit register r1
753 reg_class r1_reg(
754 R1, R1_H
755 );
756
757 // Class for 64 bit register r2
758 reg_class r2_reg(
759 R2, R2_H
760 );
761
762 // Class for 64 bit register r3
763 reg_class r3_reg(
764 R3, R3_H
765 );
766
767 // Class for 64 bit register r4
768 reg_class r4_reg(
769 R4, R4_H
770 );
771
772 // Class for 64 bit register r5
773 reg_class r5_reg(
774 R5, R5_H
775 );
776
777 // Class for 64 bit register r10
778 reg_class r10_reg(
779 R10, R10_H
780 );
781
782 // Class for 64 bit register r11
783 reg_class r11_reg(
784 R11, R11_H
785 );
786
787 // Class for method register
788 reg_class method_reg(
789 R12, R12_H
790 );
791
792 // Class for heapbase register
793 reg_class heapbase_reg(
794 R27, R27_H
795 );
796
797 // Class for thread register
798 reg_class thread_reg(
799 R28, R28_H
800 );
801
802 // Class for frame pointer register
803 reg_class fp_reg(
804 R29, R29_H
805 );
806
807 // Class for link register
808 reg_class lr_reg(
809 R30, R30_H
810 );
811
812 // Class for long sp register
813 reg_class sp_reg(
814 R31, R31_H
815 );
816
817 // Class for all pointer registers
818 reg_class ptr_reg %{
819 return _PTR_REG_mask;
820 %}
821
822 // Class for all non_special pointer registers
823 reg_class no_special_ptr_reg %{
824 return _NO_SPECIAL_PTR_REG_mask;
825 %}
826
827 // Class for all float registers
828 reg_class float_reg(
829 V0,
830 V1,
831 V2,
832 V3,
833 V4,
834 V5,
835 V6,
836 V7,
837 V8,
838 V9,
839 V10,
840 V11,
841 V12,
842 V13,
843 V14,
844 V15,
845 V16,
846 V17,
847 V18,
848 V19,
849 V20,
850 V21,
851 V22,
852 V23,
853 V24,
854 V25,
855 V26,
856 V27,
857 V28,
858 V29,
859 V30,
860 V31
861 );
862
863 // Double precision float registers have virtual `high halves' that
864 // are needed by the allocator.
865 // Class for all double registers
866 reg_class double_reg(
867 V0, V0_H,
868 V1, V1_H,
869 V2, V2_H,
870 V3, V3_H,
871 V4, V4_H,
872 V5, V5_H,
873 V6, V6_H,
874 V7, V7_H,
875 V8, V8_H,
876 V9, V9_H,
877 V10, V10_H,
878 V11, V11_H,
879 V12, V12_H,
880 V13, V13_H,
881 V14, V14_H,
882 V15, V15_H,
883 V16, V16_H,
884 V17, V17_H,
885 V18, V18_H,
886 V19, V19_H,
887 V20, V20_H,
888 V21, V21_H,
889 V22, V22_H,
890 V23, V23_H,
891 V24, V24_H,
892 V25, V25_H,
893 V26, V26_H,
894 V27, V27_H,
895 V28, V28_H,
896 V29, V29_H,
897 V30, V30_H,
898 V31, V31_H
899 );
900
901 // Class for all SVE vector registers.
902 reg_class vectora_reg (
903 V0, V0_H, V0_J, V0_K, V0_L, V0_M, V0_N, V0_O,
904 V1, V1_H, V1_J, V1_K, V1_L, V1_M, V1_N, V1_O,
905 V2, V2_H, V2_J, V2_K, V2_L, V2_M, V2_N, V2_O,
906 V3, V3_H, V3_J, V3_K, V3_L, V3_M, V3_N, V3_O,
907 V4, V4_H, V4_J, V4_K, V4_L, V4_M, V4_N, V4_O,
908 V5, V5_H, V5_J, V5_K, V5_L, V5_M, V5_N, V5_O,
909 V6, V6_H, V6_J, V6_K, V6_L, V6_M, V6_N, V6_O,
910 V7, V7_H, V7_J, V7_K, V7_L, V7_M, V7_N, V7_O,
911 V8, V8_H, V8_J, V8_K, V8_L, V8_M, V8_N, V8_O,
912 V9, V9_H, V9_J, V9_K, V9_L, V9_M, V9_N, V9_O,
913 V10, V10_H, V10_J, V10_K, V10_L, V10_M, V10_N, V10_O,
914 V11, V11_H, V11_J, V11_K, V11_L, V11_M, V11_N, V11_O,
915 V12, V12_H, V12_J, V12_K, V12_L, V12_M, V12_N, V12_O,
916 V13, V13_H, V13_J, V13_K, V13_L, V13_M, V13_N, V13_O,
917 V14, V14_H, V14_J, V14_K, V14_L, V14_M, V14_N, V14_O,
918 V15, V15_H, V15_J, V15_K, V15_L, V15_M, V15_N, V15_O,
919 V16, V16_H, V16_J, V16_K, V16_L, V16_M, V16_N, V16_O,
920 V17, V17_H, V17_J, V17_K, V17_L, V17_M, V17_N, V17_O,
921 V18, V18_H, V18_J, V18_K, V18_L, V18_M, V18_N, V18_O,
922 V19, V19_H, V19_J, V19_K, V19_L, V19_M, V19_N, V19_O,
923 V20, V20_H, V20_J, V20_K, V20_L, V20_M, V20_N, V20_O,
924 V21, V21_H, V21_J, V21_K, V21_L, V21_M, V21_N, V21_O,
925 V22, V22_H, V22_J, V22_K, V22_L, V22_M, V22_N, V22_O,
926 V23, V23_H, V23_J, V23_K, V23_L, V23_M, V23_N, V23_O,
927 V24, V24_H, V24_J, V24_K, V24_L, V24_M, V24_N, V24_O,
928 V25, V25_H, V25_J, V25_K, V25_L, V25_M, V25_N, V25_O,
929 V26, V26_H, V26_J, V26_K, V26_L, V26_M, V26_N, V26_O,
930 V27, V27_H, V27_J, V27_K, V27_L, V27_M, V27_N, V27_O,
931 V28, V28_H, V28_J, V28_K, V28_L, V28_M, V28_N, V28_O,
932 V29, V29_H, V29_J, V29_K, V29_L, V29_M, V29_N, V29_O,
933 V30, V30_H, V30_J, V30_K, V30_L, V30_M, V30_N, V30_O,
934 V31, V31_H, V31_J, V31_K, V31_L, V31_M, V31_N, V31_O,
935 );
936
937 // Class for all 64bit vector registers
938 reg_class vectord_reg(
939 V0, V0_H,
940 V1, V1_H,
941 V2, V2_H,
942 V3, V3_H,
943 V4, V4_H,
944 V5, V5_H,
945 V6, V6_H,
946 V7, V7_H,
947 V8, V8_H,
948 V9, V9_H,
949 V10, V10_H,
950 V11, V11_H,
951 V12, V12_H,
952 V13, V13_H,
953 V14, V14_H,
954 V15, V15_H,
955 V16, V16_H,
956 V17, V17_H,
957 V18, V18_H,
958 V19, V19_H,
959 V20, V20_H,
960 V21, V21_H,
961 V22, V22_H,
962 V23, V23_H,
963 V24, V24_H,
964 V25, V25_H,
965 V26, V26_H,
966 V27, V27_H,
967 V28, V28_H,
968 V29, V29_H,
969 V30, V30_H,
970 V31, V31_H
971 );
972
973 // Class for all 128bit vector registers
974 reg_class vectorx_reg(
975 V0, V0_H, V0_J, V0_K,
976 V1, V1_H, V1_J, V1_K,
977 V2, V2_H, V2_J, V2_K,
978 V3, V3_H, V3_J, V3_K,
979 V4, V4_H, V4_J, V4_K,
980 V5, V5_H, V5_J, V5_K,
981 V6, V6_H, V6_J, V6_K,
982 V7, V7_H, V7_J, V7_K,
983 V8, V8_H, V8_J, V8_K,
984 V9, V9_H, V9_J, V9_K,
985 V10, V10_H, V10_J, V10_K,
986 V11, V11_H, V11_J, V11_K,
987 V12, V12_H, V12_J, V12_K,
988 V13, V13_H, V13_J, V13_K,
989 V14, V14_H, V14_J, V14_K,
990 V15, V15_H, V15_J, V15_K,
991 V16, V16_H, V16_J, V16_K,
992 V17, V17_H, V17_J, V17_K,
993 V18, V18_H, V18_J, V18_K,
994 V19, V19_H, V19_J, V19_K,
995 V20, V20_H, V20_J, V20_K,
996 V21, V21_H, V21_J, V21_K,
997 V22, V22_H, V22_J, V22_K,
998 V23, V23_H, V23_J, V23_K,
999 V24, V24_H, V24_J, V24_K,
1000 V25, V25_H, V25_J, V25_K,
1001 V26, V26_H, V26_J, V26_K,
1002 V27, V27_H, V27_J, V27_K,
1003 V28, V28_H, V28_J, V28_K,
1004 V29, V29_H, V29_J, V29_K,
1005 V30, V30_H, V30_J, V30_K,
1006 V31, V31_H, V31_J, V31_K
1007 );
1008
1009 // Class for 128 bit register v0
1010 reg_class v0_reg(
1011 V0, V0_H
1012 );
1013
1014 // Class for 128 bit register v1
1015 reg_class v1_reg(
1016 V1, V1_H
1017 );
1018
1019 // Class for 128 bit register v2
1020 reg_class v2_reg(
1021 V2, V2_H
1022 );
1023
1024 // Class for 128 bit register v3
1025 reg_class v3_reg(
1026 V3, V3_H
1027 );
1028
1029 // Class for 128 bit register v4
1030 reg_class v4_reg(
1031 V4, V4_H
1032 );
1033
1034 // Class for 128 bit register v5
1035 reg_class v5_reg(
1036 V5, V5_H
1037 );
1038
1039 // Class for 128 bit register v6
1040 reg_class v6_reg(
1041 V6, V6_H
1042 );
1043
1044 // Class for 128 bit register v7
1045 reg_class v7_reg(
1046 V7, V7_H
1047 );
1048
1049 // Class for 128 bit register v8
1050 reg_class v8_reg(
1051 V8, V8_H
1052 );
1053
1054 // Class for 128 bit register v9
1055 reg_class v9_reg(
1056 V9, V9_H
1057 );
1058
1059 // Class for 128 bit register v10
1060 reg_class v10_reg(
1061 V10, V10_H
1062 );
1063
1064 // Class for 128 bit register v11
1065 reg_class v11_reg(
1066 V11, V11_H
1067 );
1068
1069 // Class for 128 bit register v12
1070 reg_class v12_reg(
1071 V12, V12_H
1072 );
1073
1074 // Class for 128 bit register v13
1075 reg_class v13_reg(
1076 V13, V13_H
1077 );
1078
1079 // Class for 128 bit register v14
1080 reg_class v14_reg(
1081 V14, V14_H
1082 );
1083
1084 // Class for 128 bit register v15
1085 reg_class v15_reg(
1086 V15, V15_H
1087 );
1088
1089 // Class for 128 bit register v16
1090 reg_class v16_reg(
1091 V16, V16_H
1092 );
1093
1094 // Class for 128 bit register v17
1095 reg_class v17_reg(
1096 V17, V17_H
1097 );
1098
1099 // Class for 128 bit register v18
1100 reg_class v18_reg(
1101 V18, V18_H
1102 );
1103
1104 // Class for 128 bit register v19
1105 reg_class v19_reg(
1106 V19, V19_H
1107 );
1108
1109 // Class for 128 bit register v20
1110 reg_class v20_reg(
1111 V20, V20_H
1112 );
1113
1114 // Class for 128 bit register v21
1115 reg_class v21_reg(
1116 V21, V21_H
1117 );
1118
1119 // Class for 128 bit register v22
1120 reg_class v22_reg(
1121 V22, V22_H
1122 );
1123
1124 // Class for 128 bit register v23
1125 reg_class v23_reg(
1126 V23, V23_H
1127 );
1128
1129 // Class for 128 bit register v24
1130 reg_class v24_reg(
1131 V24, V24_H
1132 );
1133
1134 // Class for 128 bit register v25
1135 reg_class v25_reg(
1136 V25, V25_H
1137 );
1138
1139 // Class for 128 bit register v26
1140 reg_class v26_reg(
1141 V26, V26_H
1142 );
1143
1144 // Class for 128 bit register v27
1145 reg_class v27_reg(
1146 V27, V27_H
1147 );
1148
1149 // Class for 128 bit register v28
1150 reg_class v28_reg(
1151 V28, V28_H
1152 );
1153
1154 // Class for 128 bit register v29
1155 reg_class v29_reg(
1156 V29, V29_H
1157 );
1158
1159 // Class for 128 bit register v30
1160 reg_class v30_reg(
1161 V30, V30_H
1162 );
1163
1164 // Class for 128 bit register v31
1165 reg_class v31_reg(
1166 V31, V31_H
1167 );
1168
1169 // Class for all SVE predicate registers.
1170 reg_class pr_reg (
1171 P0,
1172 P1,
1173 P2,
1174 P3,
1175 P4,
1176 P5,
1177 P6,
1178 // P7, non-allocatable, preserved with all elements preset to TRUE.
1179 P8,
1180 P9,
1181 P10,
1182 P11,
1183 P12,
1184 P13,
1185 P14,
1186 P15
1187 );
1188
1189 // Class for SVE governing predicate registers, which are used
1190 // to determine the active elements of a predicated instruction.
1191 reg_class gov_pr (
1192 P0,
1193 P1,
1194 P2,
1195 P3,
1196 P4,
1197 P5,
1198 P6,
1199 // P7, non-allocatable, preserved with all elements preset to TRUE.
1200 );
1201
1202 // Singleton class for condition codes
1203 reg_class int_flags(RFLAGS);
1204
1205 %}
1206
1207 //----------DEFINITION BLOCK---------------------------------------------------
1208 // Define name --> value mappings to inform the ADLC of an integer valued name
1209 // Current support includes integer values in the range [0, 0x7FFFFFFF]
1210 // Format:
1211 // int_def <name> ( <int_value>, <expression>);
1212 // Generated Code in ad_<arch>.hpp
1213 // #define <name> (<expression>)
1214 // // value == <int_value>
1215 // Generated code in ad_<arch>.cpp adlc_verification()
1216 // assert( <name> == <int_value>, "Expect (<expression>) to equal <int_value>");
1217 //
1218
1219 // we follow the ppc-aix port in using a simple cost model which ranks
1220 // register operations as cheap, memory ops as more expensive and
1221 // branches as most expensive. the first two have a low as well as a
1222 // normal cost. huge cost appears to be a way of saying don't do
1223 // something
1224
1225 definitions %{
1226 // The default cost (of a register move instruction).
1227 int_def INSN_COST ( 100, 100);
1228 int_def BRANCH_COST ( 200, 2 * INSN_COST);
1229 int_def CALL_COST ( 200, 2 * INSN_COST);
1230 int_def VOLATILE_REF_COST ( 1000, 10 * INSN_COST);
1231 %}
1232
1233
1234 //----------SOURCE BLOCK-------------------------------------------------------
1235 // This is a block of C++ code which provides values, functions, and
1236 // definitions necessary in the rest of the architecture description
1237
1238 source_hpp %{
1239
1240 #include "asm/macroAssembler.hpp"
1241 #include "gc/shared/barrierSetAssembler.hpp"
1242 #include "gc/shared/cardTable.hpp"
1243 #include "gc/shared/cardTableBarrierSet.hpp"
1244 #include "gc/shared/collectedHeap.hpp"
1245 #include "opto/addnode.hpp"
1246 #include "opto/convertnode.hpp"
1247 #include "runtime/objectMonitor.hpp"
1248
1249 extern RegMask _ANY_REG32_mask;
1250 extern RegMask _ANY_REG_mask;
1251 extern RegMask _PTR_REG_mask;
1252 extern RegMask _NO_SPECIAL_REG32_mask;
1253 extern RegMask _NO_SPECIAL_REG_mask;
1254 extern RegMask _NO_SPECIAL_PTR_REG_mask;
1255
1256 class CallStubImpl {
1257
1258 //--------------------------------------------------------------
1259 //---< Used for optimization in Compile::shorten_branches >---
1260 //--------------------------------------------------------------
1261
1262 public:
1263 // Size of call trampoline stub.
1264 static uint size_call_trampoline() {
1265 return 0; // no call trampolines on this platform
1266 }
1267
1268 // number of relocations needed by a call trampoline stub
1269 static uint reloc_call_trampoline() {
1270 return 0; // no call trampolines on this platform
1271 }
1272 };
1273
1274 class HandlerImpl {
1275
1276 public:
1277
1278 static int emit_exception_handler(CodeBuffer &cbuf);
1279 static int emit_deopt_handler(CodeBuffer& cbuf);
1280
1281 static uint size_exception_handler() {
1282 return MacroAssembler::far_branch_size();
1283 }
1284
1285 static uint size_deopt_handler() {
1286 // count one adr and one far branch instruction
1287 return 4 * NativeInstruction::instruction_size;
1288 }
1289 };
1290
1291 class Node::PD {
1292 public:
1293 enum NodeFlags {
1294 _last_flag = Node::_last_flag
1295 };
1296 };
1297
1298 bool is_CAS(int opcode, bool maybe_volatile);
1299
1300 // predicates controlling emit of ldr<x>/ldar<x> and associated dmb
1301
1302 bool unnecessary_acquire(const Node *barrier);
1303 bool needs_acquiring_load(const Node *load);
1304
1305 // predicates controlling emit of str<x>/stlr<x> and associated dmbs
1306
1307 bool unnecessary_release(const Node *barrier);
1308 bool unnecessary_volatile(const Node *barrier);
1309 bool needs_releasing_store(const Node *store);
1310
1311 // predicate controlling translation of CompareAndSwapX
1312 bool needs_acquiring_load_exclusive(const Node *load);
1313
1314 // predicate controlling addressing modes
1315 bool size_fits_all_mem_uses(AddPNode* addp, int shift);
1316 %}
1317
1318 source %{
1319
1320 // Derived RegMask with conditionally allocatable registers
1321
1322 void PhaseOutput::pd_perform_mach_node_analysis() {
1323 }
1324
1325 int MachNode::pd_alignment_required() const {
1326 return 1;
1327 }
1328
1329 int MachNode::compute_padding(int current_offset) const {
1330 return 0;
1331 }
1332
1333 RegMask _ANY_REG32_mask;
1334 RegMask _ANY_REG_mask;
1335 RegMask _PTR_REG_mask;
1336 RegMask _NO_SPECIAL_REG32_mask;
1337 RegMask _NO_SPECIAL_REG_mask;
1338 RegMask _NO_SPECIAL_PTR_REG_mask;
1339
1340 void reg_mask_init() {
1341 // We derive below RegMask(s) from the ones which are auto-generated from
1342 // adlc register classes to make AArch64 rheapbase (r27) and rfp (r29)
1343 // registers conditionally reserved.
1344
1345 _ANY_REG32_mask = _ALL_REG32_mask;
1346 _ANY_REG32_mask.Remove(OptoReg::as_OptoReg(r31_sp->as_VMReg()));
1347
1348 _ANY_REG_mask = _ALL_REG_mask;
1349
1350 _PTR_REG_mask = _ALL_REG_mask;
1351
1352 _NO_SPECIAL_REG32_mask = _ALL_REG32_mask;
1353 _NO_SPECIAL_REG32_mask.SUBTRACT(_NON_ALLOCATABLE_REG32_mask);
1354
1355 _NO_SPECIAL_REG_mask = _ALL_REG_mask;
1356 _NO_SPECIAL_REG_mask.SUBTRACT(_NON_ALLOCATABLE_REG_mask);
1357
1358 _NO_SPECIAL_PTR_REG_mask = _ALL_REG_mask;
1359 _NO_SPECIAL_PTR_REG_mask.SUBTRACT(_NON_ALLOCATABLE_REG_mask);
1360
1361 // r27 is not allocatable when compressed oops is on and heapbase is not
1362 // zero, compressed klass pointers doesn't use r27 after JDK-8234794
1363 if (UseCompressedOops && (CompressedOops::ptrs_base() != NULL)) {
1364 _NO_SPECIAL_REG32_mask.Remove(OptoReg::as_OptoReg(r27->as_VMReg()));
1365 _NO_SPECIAL_REG_mask.SUBTRACT(_HEAPBASE_REG_mask);
1366 _NO_SPECIAL_PTR_REG_mask.SUBTRACT(_HEAPBASE_REG_mask);
1367 }
1368
1369 // r29 is not allocatable when PreserveFramePointer is on
1370 if (PreserveFramePointer) {
1371 _NO_SPECIAL_REG32_mask.Remove(OptoReg::as_OptoReg(r29->as_VMReg()));
1372 _NO_SPECIAL_REG_mask.SUBTRACT(_FP_REG_mask);
1373 _NO_SPECIAL_PTR_REG_mask.SUBTRACT(_FP_REG_mask);
1374 }
1375 }
1376
1377 // Optimizaton of volatile gets and puts
1378 // -------------------------------------
1379 //
1380 // AArch64 has ldar<x> and stlr<x> instructions which we can safely
1381 // use to implement volatile reads and writes. For a volatile read
1382 // we simply need
1383 //
1384 // ldar<x>
1385 //
1386 // and for a volatile write we need
1387 //
1388 // stlr<x>
1389 //
1390 // Alternatively, we can implement them by pairing a normal
1391 // load/store with a memory barrier. For a volatile read we need
1392 //
1393 // ldr<x>
1394 // dmb ishld
1395 //
1396 // for a volatile write
1397 //
1398 // dmb ish
1399 // str<x>
1400 // dmb ish
1401 //
1402 // We can also use ldaxr and stlxr to implement compare and swap CAS
1403 // sequences. These are normally translated to an instruction
1404 // sequence like the following
1405 //
1406 // dmb ish
1407 // retry:
1408 // ldxr<x> rval raddr
1409 // cmp rval rold
1410 // b.ne done
1411 // stlxr<x> rval, rnew, rold
1412 // cbnz rval retry
1413 // done:
1414 // cset r0, eq
1415 // dmb ishld
1416 //
1417 // Note that the exclusive store is already using an stlxr
1418 // instruction. That is required to ensure visibility to other
1419 // threads of the exclusive write (assuming it succeeds) before that
1420 // of any subsequent writes.
1421 //
1422 // The following instruction sequence is an improvement on the above
1423 //
1424 // retry:
1425 // ldaxr<x> rval raddr
1426 // cmp rval rold
1427 // b.ne done
1428 // stlxr<x> rval, rnew, rold
1429 // cbnz rval retry
1430 // done:
1431 // cset r0, eq
1432 //
1433 // We don't need the leading dmb ish since the stlxr guarantees
1434 // visibility of prior writes in the case that the swap is
1435 // successful. Crucially we don't have to worry about the case where
1436 // the swap is not successful since no valid program should be
1437 // relying on visibility of prior changes by the attempting thread
1438 // in the case where the CAS fails.
1439 //
1440 // Similarly, we don't need the trailing dmb ishld if we substitute
1441 // an ldaxr instruction since that will provide all the guarantees we
1442 // require regarding observation of changes made by other threads
1443 // before any change to the CAS address observed by the load.
1444 //
1445 // In order to generate the desired instruction sequence we need to
1446 // be able to identify specific 'signature' ideal graph node
1447 // sequences which i) occur as a translation of a volatile reads or
1448 // writes or CAS operations and ii) do not occur through any other
1449 // translation or graph transformation. We can then provide
1450 // alternative aldc matching rules which translate these node
1451 // sequences to the desired machine code sequences. Selection of the
1452 // alternative rules can be implemented by predicates which identify
1453 // the relevant node sequences.
1454 //
1455 // The ideal graph generator translates a volatile read to the node
1456 // sequence
1457 //
1458 // LoadX[mo_acquire]
1459 // MemBarAcquire
1460 //
1461 // As a special case when using the compressed oops optimization we
1462 // may also see this variant
1463 //
1464 // LoadN[mo_acquire]
1465 // DecodeN
1466 // MemBarAcquire
1467 //
1468 // A volatile write is translated to the node sequence
1469 //
1470 // MemBarRelease
1471 // StoreX[mo_release] {CardMark}-optional
1472 // MemBarVolatile
1473 //
1474 // n.b. the above node patterns are generated with a strict
1475 // 'signature' configuration of input and output dependencies (see
1476 // the predicates below for exact details). The card mark may be as
1477 // simple as a few extra nodes or, in a few GC configurations, may
1478 // include more complex control flow between the leading and
1479 // trailing memory barriers. However, whatever the card mark
1480 // configuration these signatures are unique to translated volatile
1481 // reads/stores -- they will not appear as a result of any other
1482 // bytecode translation or inlining nor as a consequence of
1483 // optimizing transforms.
1484 //
1485 // We also want to catch inlined unsafe volatile gets and puts and
1486 // be able to implement them using either ldar<x>/stlr<x> or some
1487 // combination of ldr<x>/stlr<x> and dmb instructions.
1488 //
1489 // Inlined unsafe volatiles puts manifest as a minor variant of the
1490 // normal volatile put node sequence containing an extra cpuorder
1491 // membar
1492 //
1493 // MemBarRelease
1494 // MemBarCPUOrder
1495 // StoreX[mo_release] {CardMark}-optional
1496 // MemBarCPUOrder
1497 // MemBarVolatile
1498 //
1499 // n.b. as an aside, a cpuorder membar is not itself subject to
1500 // matching and translation by adlc rules. However, the rule
1501 // predicates need to detect its presence in order to correctly
1502 // select the desired adlc rules.
1503 //
1504 // Inlined unsafe volatile gets manifest as a slightly different
1505 // node sequence to a normal volatile get because of the
1506 // introduction of some CPUOrder memory barriers to bracket the
1507 // Load. However, but the same basic skeleton of a LoadX feeding a
1508 // MemBarAcquire, possibly thorugh an optional DecodeN, is still
1509 // present
1510 //
1511 // MemBarCPUOrder
1512 // || \\
1513 // MemBarCPUOrder LoadX[mo_acquire]
1514 // || |
1515 // || {DecodeN} optional
1516 // || /
1517 // MemBarAcquire
1518 //
1519 // In this case the acquire membar does not directly depend on the
1520 // load. However, we can be sure that the load is generated from an
1521 // inlined unsafe volatile get if we see it dependent on this unique
1522 // sequence of membar nodes. Similarly, given an acquire membar we
1523 // can know that it was added because of an inlined unsafe volatile
1524 // get if it is fed and feeds a cpuorder membar and if its feed
1525 // membar also feeds an acquiring load.
1526 //
1527 // Finally an inlined (Unsafe) CAS operation is translated to the
1528 // following ideal graph
1529 //
1530 // MemBarRelease
1531 // MemBarCPUOrder
1532 // CompareAndSwapX {CardMark}-optional
1533 // MemBarCPUOrder
1534 // MemBarAcquire
1535 //
1536 // So, where we can identify these volatile read and write
1537 // signatures we can choose to plant either of the above two code
1538 // sequences. For a volatile read we can simply plant a normal
1539 // ldr<x> and translate the MemBarAcquire to a dmb. However, we can
1540 // also choose to inhibit translation of the MemBarAcquire and
1541 // inhibit planting of the ldr<x>, instead planting an ldar<x>.
1542 //
1543 // When we recognise a volatile store signature we can choose to
1544 // plant at a dmb ish as a translation for the MemBarRelease, a
1545 // normal str<x> and then a dmb ish for the MemBarVolatile.
1546 // Alternatively, we can inhibit translation of the MemBarRelease
1547 // and MemBarVolatile and instead plant a simple stlr<x>
1548 // instruction.
1549 //
1550 // when we recognise a CAS signature we can choose to plant a dmb
1551 // ish as a translation for the MemBarRelease, the conventional
1552 // macro-instruction sequence for the CompareAndSwap node (which
1553 // uses ldxr<x>) and then a dmb ishld for the MemBarAcquire.
1554 // Alternatively, we can elide generation of the dmb instructions
1555 // and plant the alternative CompareAndSwap macro-instruction
1556 // sequence (which uses ldaxr<x>).
1557 //
1558 // Of course, the above only applies when we see these signature
1559 // configurations. We still want to plant dmb instructions in any
1560 // other cases where we may see a MemBarAcquire, MemBarRelease or
1561 // MemBarVolatile. For example, at the end of a constructor which
1562 // writes final/volatile fields we will see a MemBarRelease
1563 // instruction and this needs a 'dmb ish' lest we risk the
1564 // constructed object being visible without making the
1565 // final/volatile field writes visible.
1566 //
1567 // n.b. the translation rules below which rely on detection of the
1568 // volatile signatures and insert ldar<x> or stlr<x> are failsafe.
1569 // If we see anything other than the signature configurations we
1570 // always just translate the loads and stores to ldr<x> and str<x>
1571 // and translate acquire, release and volatile membars to the
1572 // relevant dmb instructions.
1573 //
1574
1575 // is_CAS(int opcode, bool maybe_volatile)
1576 //
1577 // return true if opcode is one of the possible CompareAndSwapX
1578 // values otherwise false.
1579
1580 bool is_CAS(int opcode, bool maybe_volatile)
1581 {
1582 switch(opcode) {
1583 // We handle these
1584 case Op_CompareAndSwapI:
1585 case Op_CompareAndSwapL:
1586 case Op_CompareAndSwapP:
1587 case Op_CompareAndSwapN:
1588 case Op_ShenandoahCompareAndSwapP:
1589 case Op_ShenandoahCompareAndSwapN:
1590 case Op_CompareAndSwapB:
1591 case Op_CompareAndSwapS:
1592 case Op_GetAndSetI:
1593 case Op_GetAndSetL:
1594 case Op_GetAndSetP:
1595 case Op_GetAndSetN:
1596 case Op_GetAndAddI:
1597 case Op_GetAndAddL:
1598 return true;
1599 case Op_CompareAndExchangeI:
1600 case Op_CompareAndExchangeN:
1601 case Op_CompareAndExchangeB:
1602 case Op_CompareAndExchangeS:
1603 case Op_CompareAndExchangeL:
1604 case Op_CompareAndExchangeP:
1605 case Op_WeakCompareAndSwapB:
1606 case Op_WeakCompareAndSwapS:
1607 case Op_WeakCompareAndSwapI:
1608 case Op_WeakCompareAndSwapL:
1609 case Op_WeakCompareAndSwapP:
1610 case Op_WeakCompareAndSwapN:
1611 case Op_ShenandoahWeakCompareAndSwapP:
1612 case Op_ShenandoahWeakCompareAndSwapN:
1613 case Op_ShenandoahCompareAndExchangeP:
1614 case Op_ShenandoahCompareAndExchangeN:
1615 return maybe_volatile;
1616 default:
1617 return false;
1618 }
1619 }
1620
1621 // helper to determine the maximum number of Phi nodes we may need to
1622 // traverse when searching from a card mark membar for the merge mem
1623 // feeding a trailing membar or vice versa
1624
1625 // predicates controlling emit of ldr<x>/ldar<x>
1626
1627 bool unnecessary_acquire(const Node *barrier)
1628 {
1629 assert(barrier->is_MemBar(), "expecting a membar");
1630
1631 MemBarNode* mb = barrier->as_MemBar();
1632
1633 if (mb->trailing_load()) {
1634 return true;
1635 }
1636
1637 if (mb->trailing_load_store()) {
1638 Node* load_store = mb->in(MemBarNode::Precedent);
1639 assert(load_store->is_LoadStore(), "unexpected graph shape");
1640 return is_CAS(load_store->Opcode(), true);
1641 }
1642
1643 return false;
1644 }
1645
1646 bool needs_acquiring_load(const Node *n)
1647 {
1648 assert(n->is_Load(), "expecting a load");
1649 LoadNode *ld = n->as_Load();
1650 return ld->is_acquire();
1651 }
1652
1653 bool unnecessary_release(const Node *n)
1654 {
1655 assert((n->is_MemBar() &&
1656 n->Opcode() == Op_MemBarRelease),
1657 "expecting a release membar");
1658
1659 MemBarNode *barrier = n->as_MemBar();
1660 if (!barrier->leading()) {
1661 return false;
1662 } else {
1663 Node* trailing = barrier->trailing_membar();
1664 MemBarNode* trailing_mb = trailing->as_MemBar();
1665 assert(trailing_mb->trailing(), "Not a trailing membar?");
1666 assert(trailing_mb->leading_membar() == n, "inconsistent leading/trailing membars");
1667
1668 Node* mem = trailing_mb->in(MemBarNode::Precedent);
1669 if (mem->is_Store()) {
1670 assert(mem->as_Store()->is_release(), "");
1671 assert(trailing_mb->Opcode() == Op_MemBarVolatile, "");
1672 return true;
1673 } else {
1674 assert(mem->is_LoadStore(), "");
1675 assert(trailing_mb->Opcode() == Op_MemBarAcquire, "");
1676 return is_CAS(mem->Opcode(), true);
1677 }
1678 }
1679 return false;
1680 }
1681
1682 bool unnecessary_volatile(const Node *n)
1683 {
1684 // assert n->is_MemBar();
1685 MemBarNode *mbvol = n->as_MemBar();
1686
1687 bool release = mbvol->trailing_store();
1688 assert(!release || (mbvol->in(MemBarNode::Precedent)->is_Store() && mbvol->in(MemBarNode::Precedent)->as_Store()->is_release()), "");
1689 #ifdef ASSERT
1690 if (release) {
1691 Node* leading = mbvol->leading_membar();
1692 assert(leading->Opcode() == Op_MemBarRelease, "");
1693 assert(leading->as_MemBar()->leading_store(), "");
1694 assert(leading->as_MemBar()->trailing_membar() == mbvol, "");
1695 }
1696 #endif
1697
1698 return release;
1699 }
1700
1701 // predicates controlling emit of str<x>/stlr<x>
1702
1703 bool needs_releasing_store(const Node *n)
1704 {
1705 // assert n->is_Store();
1706 StoreNode *st = n->as_Store();
1707 return st->trailing_membar() != NULL;
1708 }
1709
1710 // predicate controlling translation of CAS
1711 //
1712 // returns true if CAS needs to use an acquiring load otherwise false
1713
1714 bool needs_acquiring_load_exclusive(const Node *n)
1715 {
1716 assert(is_CAS(n->Opcode(), true), "expecting a compare and swap");
1717 LoadStoreNode* ldst = n->as_LoadStore();
1718 if (is_CAS(n->Opcode(), false)) {
1719 assert(ldst->trailing_membar() != NULL, "expected trailing membar");
1720 } else {
1721 return ldst->trailing_membar() != NULL;
1722 }
1723
1724 // so we can just return true here
1725 return true;
1726 }
1727
1728 #define __ _masm.
1729
1730 // advance declarations for helper functions to convert register
1731 // indices to register objects
1732
1733 // the ad file has to provide implementations of certain methods
1734 // expected by the generic code
1735 //
1736 // REQUIRED FUNCTIONALITY
1737
1738 //=============================================================================
1739
1740 // !!!!! Special hack to get all types of calls to specify the byte offset
1741 // from the start of the call to the point where the return address
1742 // will point.
1743
1744 int MachCallStaticJavaNode::ret_addr_offset()
1745 {
1746 // call should be a simple bl
1747 int off = 4;
1748 return off;
1749 }
1750
1751 int MachCallDynamicJavaNode::ret_addr_offset()
1752 {
1753 return 16; // movz, movk, movk, bl
1754 }
1755
1756 int MachCallRuntimeNode::ret_addr_offset() {
1757 // for generated stubs the call will be
1758 // bl(addr)
1759 // or with far branches
1760 // bl(trampoline_stub)
1761 // for real runtime callouts it will be six instructions
1762 // see aarch64_enc_java_to_runtime
1763 // adr(rscratch2, retaddr)
1764 // lea(rscratch1, RuntimeAddress(addr)
1765 // stp(zr, rscratch2, Address(__ pre(sp, -2 * wordSize)))
1766 // blr(rscratch1)
1767 CodeBlob *cb = CodeCache::find_blob(_entry_point);
1768 if (cb) {
1769 return 1 * NativeInstruction::instruction_size;
1770 } else {
1771 return 6 * NativeInstruction::instruction_size;
1772 }
1773 }
1774
1775 int MachCallNativeNode::ret_addr_offset() {
1776 // This is implemented using aarch64_enc_java_to_runtime as above.
1777 CodeBlob *cb = CodeCache::find_blob(_entry_point);
1778 if (cb) {
1779 return 1 * NativeInstruction::instruction_size;
1780 } else {
1781 return 6 * NativeInstruction::instruction_size;
1782 }
1783 }
1784
1785 //=============================================================================
1786
1787 #ifndef PRODUCT
1788 void MachBreakpointNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
1789 st->print("BREAKPOINT");
1790 }
1791 #endif
1792
1793 void MachBreakpointNode::emit(CodeBuffer &cbuf, PhaseRegAlloc *ra_) const {
1794 C2_MacroAssembler _masm(&cbuf);
1795 __ brk(0);
1796 }
1797
1798 uint MachBreakpointNode::size(PhaseRegAlloc *ra_) const {
1799 return MachNode::size(ra_);
1800 }
1801
1802 //=============================================================================
1803
1804 #ifndef PRODUCT
1805 void MachNopNode::format(PhaseRegAlloc*, outputStream* st) const {
1806 st->print("nop \t# %d bytes pad for loops and calls", _count);
1807 }
1808 #endif
1809
1810 void MachNopNode::emit(CodeBuffer &cbuf, PhaseRegAlloc*) const {
1811 C2_MacroAssembler _masm(&cbuf);
1812 for (int i = 0; i < _count; i++) {
1813 __ nop();
1814 }
1815 }
1816
1817 uint MachNopNode::size(PhaseRegAlloc*) const {
1818 return _count * NativeInstruction::instruction_size;
1819 }
1820
1821 //=============================================================================
1822 const RegMask& MachConstantBaseNode::_out_RegMask = RegMask::Empty;
1823
1824 int ConstantTable::calculate_table_base_offset() const {
1825 return 0; // absolute addressing, no offset
1826 }
1827
1828 bool MachConstantBaseNode::requires_postalloc_expand() const { return false; }
1829 void MachConstantBaseNode::postalloc_expand(GrowableArray <Node *> *nodes, PhaseRegAlloc *ra_) {
1830 ShouldNotReachHere();
1831 }
1832
1833 void MachConstantBaseNode::emit(CodeBuffer& cbuf, PhaseRegAlloc* ra_) const {
1834 // Empty encoding
1835 }
1836
1837 uint MachConstantBaseNode::size(PhaseRegAlloc* ra_) const {
1838 return 0;
1839 }
1840
1841 #ifndef PRODUCT
1842 void MachConstantBaseNode::format(PhaseRegAlloc* ra_, outputStream* st) const {
1843 st->print("-- \t// MachConstantBaseNode (empty encoding)");
1844 }
1845 #endif
1846
1847 #ifndef PRODUCT
1848 void MachPrologNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
1849 Compile* C = ra_->C;
1850
1851 int framesize = C->output()->frame_slots() << LogBytesPerInt;
1852
1853 if (C->output()->need_stack_bang(framesize))
1854 st->print("# stack bang size=%d\n\t", framesize);
1855
1856 if (framesize < ((1 << 9) + 2 * wordSize)) {
1857 st->print("sub sp, sp, #%d\n\t", framesize);
1858 st->print("stp rfp, lr, [sp, #%d]", framesize - 2 * wordSize);
1859 if (PreserveFramePointer) st->print("\n\tadd rfp, sp, #%d", framesize - 2 * wordSize);
1860 } else {
1861 st->print("stp lr, rfp, [sp, #%d]!\n\t", -(2 * wordSize));
1862 if (PreserveFramePointer) st->print("mov rfp, sp\n\t");
1863 st->print("mov rscratch1, #%d\n\t", framesize - 2 * wordSize);
1864 st->print("sub sp, sp, rscratch1");
1865 }
1866 if (C->stub_function() == NULL && BarrierSet::barrier_set()->barrier_set_nmethod() != NULL) {
1867 st->print("\n\t");
1868 st->print("ldr rscratch1, [guard]\n\t");
1869 st->print("dmb ishld\n\t");
1870 st->print("ldr rscratch2, [rthread, #thread_disarmed_offset]\n\t");
1871 st->print("cmp rscratch1, rscratch2\n\t");
1872 st->print("b.eq skip");
1873 st->print("\n\t");
1874 st->print("blr #nmethod_entry_barrier_stub\n\t");
1875 st->print("b skip\n\t");
1876 st->print("guard: int\n\t");
1877 st->print("\n\t");
1878 st->print("skip:\n\t");
1879 }
1880 }
1881 #endif
1882
1883 void MachPrologNode::emit(CodeBuffer &cbuf, PhaseRegAlloc *ra_) const {
1884 Compile* C = ra_->C;
1885 C2_MacroAssembler _masm(&cbuf);
1886
1887 // n.b. frame size includes space for return pc and rfp
1888 const int framesize = C->output()->frame_size_in_bytes();
1889
1890 // insert a nop at the start of the prolog so we can patch in a
1891 // branch if we need to invalidate the method later
1892 __ nop();
1893
1894 if (C->clinit_barrier_on_entry()) {
1895 assert(!C->method()->holder()->is_not_initialized(), "initialization should have been started");
1896
1897 Label L_skip_barrier;
1898
1899 __ mov_metadata(rscratch2, C->method()->holder()->constant_encoding());
1900 __ clinit_barrier(rscratch2, rscratch1, &L_skip_barrier);
1901 __ far_jump(RuntimeAddress(SharedRuntime::get_handle_wrong_method_stub()));
1902 __ bind(L_skip_barrier);
1903 }
1904
1905 if (C->max_vector_size() > 0) {
1906 __ reinitialize_ptrue();
1907 }
1908
1909 int bangsize = C->output()->bang_size_in_bytes();
1910 if (C->output()->need_stack_bang(bangsize))
1911 __ generate_stack_overflow_check(bangsize);
1912
1913 __ build_frame(framesize);
1914
1915 if (C->stub_function() == NULL) {
1916 BarrierSetAssembler* bs = BarrierSet::barrier_set()->barrier_set_assembler();
1917 bs->nmethod_entry_barrier(&_masm);
1918 }
1919
1920 if (VerifyStackAtCalls) {
1921 Unimplemented();
1922 }
1923
1924 C->output()->set_frame_complete(cbuf.insts_size());
1925
1926 if (C->has_mach_constant_base_node()) {
1927 // NOTE: We set the table base offset here because users might be
1928 // emitted before MachConstantBaseNode.
1929 ConstantTable& constant_table = C->output()->constant_table();
1930 constant_table.set_table_base_offset(constant_table.calculate_table_base_offset());
1931 }
1932 }
1933
1934 uint MachPrologNode::size(PhaseRegAlloc* ra_) const
1935 {
1936 return MachNode::size(ra_); // too many variables; just compute it
1937 // the hard way
1938 }
1939
1940 int MachPrologNode::reloc() const
1941 {
1942 return 0;
1943 }
1944
1945 //=============================================================================
1946
1947 #ifndef PRODUCT
1948 void MachEpilogNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
1949 Compile* C = ra_->C;
1950 int framesize = C->output()->frame_slots() << LogBytesPerInt;
1951
1952 st->print("# pop frame %d\n\t",framesize);
1953
1954 if (framesize == 0) {
1955 st->print("ldp lr, rfp, [sp],#%d\n\t", (2 * wordSize));
1956 } else if (framesize < ((1 << 9) + 2 * wordSize)) {
1957 st->print("ldp lr, rfp, [sp,#%d]\n\t", framesize - 2 * wordSize);
1958 st->print("add sp, sp, #%d\n\t", framesize);
1959 } else {
1960 st->print("mov rscratch1, #%d\n\t", framesize - 2 * wordSize);
1961 st->print("add sp, sp, rscratch1\n\t");
1962 st->print("ldp lr, rfp, [sp],#%d\n\t", (2 * wordSize));
1963 }
1964
1965 if (do_polling() && C->is_method_compilation()) {
1966 st->print("# test polling word\n\t");
1967 st->print("ldr rscratch1, [rthread],#%d\n\t", in_bytes(JavaThread::polling_word_offset()));
1968 st->print("cmp sp, rscratch1\n\t");
1969 st->print("bhi #slow_path");
1970 }
1971 }
1972 #endif
1973
1974 void MachEpilogNode::emit(CodeBuffer &cbuf, PhaseRegAlloc *ra_) const {
1975 Compile* C = ra_->C;
1976 C2_MacroAssembler _masm(&cbuf);
1977 int framesize = C->output()->frame_slots() << LogBytesPerInt;
1978
1979 __ remove_frame(framesize);
1980
1981 if (StackReservedPages > 0 && C->has_reserved_stack_access()) {
1982 __ reserved_stack_check();
1983 }
1984
1985 if (do_polling() && C->is_method_compilation()) {
1986 Label dummy_label;
1987 Label* code_stub = &dummy_label;
1988 if (!C->output()->in_scratch_emit_size()) {
1989 code_stub = &C->output()->safepoint_poll_table()->add_safepoint(__ offset());
1990 }
1991 __ relocate(relocInfo::poll_return_type);
1992 __ safepoint_poll(*code_stub, true /* at_return */, false /* acquire */, true /* in_nmethod */);
1993 }
1994 }
1995
1996 uint MachEpilogNode::size(PhaseRegAlloc *ra_) const {
1997 // Variable size. Determine dynamically.
1998 return MachNode::size(ra_);
1999 }
2000
2001 int MachEpilogNode::reloc() const {
2002 // Return number of relocatable values contained in this instruction.
2003 return 1; // 1 for polling page.
2004 }
2005
2006 const Pipeline * MachEpilogNode::pipeline() const {
2007 return MachNode::pipeline_class();
2008 }
2009
2010 //=============================================================================
2011
2012 // Figure out which register class each belongs in: rc_int, rc_float or
2013 // rc_stack.
2014 enum RC { rc_bad, rc_int, rc_float, rc_predicate, rc_stack };
2015
2016 static enum RC rc_class(OptoReg::Name reg) {
2017
2018 if (reg == OptoReg::Bad) {
2019 return rc_bad;
2020 }
2021
2022 // we have 32 int registers * 2 halves
2023 int slots_of_int_registers = RegisterImpl::max_slots_per_register * RegisterImpl::number_of_registers;
2024
2025 if (reg < slots_of_int_registers) {
2026 return rc_int;
2027 }
2028
2029 // we have 32 float register * 8 halves
2030 int slots_of_float_registers = FloatRegisterImpl::max_slots_per_register * FloatRegisterImpl::number_of_registers;
2031 if (reg < slots_of_int_registers + slots_of_float_registers) {
2032 return rc_float;
2033 }
2034
2035 int slots_of_predicate_registers = PRegisterImpl::max_slots_per_register * PRegisterImpl::number_of_registers;
2036 if (reg < slots_of_int_registers + slots_of_float_registers + slots_of_predicate_registers) {
2037 return rc_predicate;
2038 }
2039
2040 // Between predicate regs & stack is the flags.
2041 assert(OptoReg::is_stack(reg), "blow up if spilling flags");
2042
2043 return rc_stack;
2044 }
2045
2046 uint MachSpillCopyNode::implementation(CodeBuffer *cbuf, PhaseRegAlloc *ra_, bool do_size, outputStream *st) const {
2047 Compile* C = ra_->C;
2048
2049 // Get registers to move.
2050 OptoReg::Name src_hi = ra_->get_reg_second(in(1));
2051 OptoReg::Name src_lo = ra_->get_reg_first(in(1));
2052 OptoReg::Name dst_hi = ra_->get_reg_second(this);
2053 OptoReg::Name dst_lo = ra_->get_reg_first(this);
2054
2055 enum RC src_hi_rc = rc_class(src_hi);
2056 enum RC src_lo_rc = rc_class(src_lo);
2057 enum RC dst_hi_rc = rc_class(dst_hi);
2058 enum RC dst_lo_rc = rc_class(dst_lo);
2059
2060 assert(src_lo != OptoReg::Bad && dst_lo != OptoReg::Bad, "must move at least 1 register");
2061
2062 if (src_hi != OptoReg::Bad) {
2063 assert((src_lo&1)==0 && src_lo+1==src_hi &&
2064 (dst_lo&1)==0 && dst_lo+1==dst_hi,
2065 "expected aligned-adjacent pairs");
2066 }
2067
2068 if (src_lo == dst_lo && src_hi == dst_hi) {
2069 return 0; // Self copy, no move.
2070 }
2071
2072 bool is64 = (src_lo & 1) == 0 && src_lo + 1 == src_hi &&
2073 (dst_lo & 1) == 0 && dst_lo + 1 == dst_hi;
2074 int src_offset = ra_->reg2offset(src_lo);
2075 int dst_offset = ra_->reg2offset(dst_lo);
2076
2077 if (bottom_type()->isa_vect() != NULL) {
2078 uint ireg = ideal_reg();
2079 if (ireg == Op_VecA && cbuf) {
2080 C2_MacroAssembler _masm(cbuf);
2081 int sve_vector_reg_size_in_bytes = Matcher::scalable_vector_reg_size(T_BYTE);
2082 if (src_lo_rc == rc_stack && dst_lo_rc == rc_stack) {
2083 // stack->stack
2084 __ spill_copy_sve_vector_stack_to_stack(src_offset, dst_offset,
2085 sve_vector_reg_size_in_bytes);
2086 } else if (src_lo_rc == rc_float && dst_lo_rc == rc_stack) {
2087 __ spill_sve_vector(as_FloatRegister(Matcher::_regEncode[src_lo]), ra_->reg2offset(dst_lo),
2088 sve_vector_reg_size_in_bytes);
2089 } else if (src_lo_rc == rc_stack && dst_lo_rc == rc_float) {
2090 __ unspill_sve_vector(as_FloatRegister(Matcher::_regEncode[dst_lo]), ra_->reg2offset(src_lo),
2091 sve_vector_reg_size_in_bytes);
2092 } else if (src_lo_rc == rc_float && dst_lo_rc == rc_float) {
2093 __ sve_orr(as_FloatRegister(Matcher::_regEncode[dst_lo]),
2094 as_FloatRegister(Matcher::_regEncode[src_lo]),
2095 as_FloatRegister(Matcher::_regEncode[src_lo]));
2096 } else {
2097 ShouldNotReachHere();
2098 }
2099 } else if (cbuf) {
2100 assert(ireg == Op_VecD || ireg == Op_VecX, "must be 64 bit or 128 bit vector");
2101 C2_MacroAssembler _masm(cbuf);
2102 assert((src_lo_rc != rc_int && dst_lo_rc != rc_int), "sanity");
2103 if (src_lo_rc == rc_stack && dst_lo_rc == rc_stack) {
2104 // stack->stack
2105 assert((src_offset & 7) == 0 && (dst_offset & 7) == 0, "unaligned stack offset");
2106 if (ireg == Op_VecD) {
2107 __ unspill(rscratch1, true, src_offset);
2108 __ spill(rscratch1, true, dst_offset);
2109 } else {
2110 __ spill_copy128(src_offset, dst_offset);
2111 }
2112 } else if (src_lo_rc == rc_float && dst_lo_rc == rc_float) {
2113 __ mov(as_FloatRegister(Matcher::_regEncode[dst_lo]),
2114 ireg == Op_VecD ? __ T8B : __ T16B,
2115 as_FloatRegister(Matcher::_regEncode[src_lo]));
2116 } else if (src_lo_rc == rc_float && dst_lo_rc == rc_stack) {
2117 __ spill(as_FloatRegister(Matcher::_regEncode[src_lo]),
2118 ireg == Op_VecD ? __ D : __ Q,
2119 ra_->reg2offset(dst_lo));
2120 } else if (src_lo_rc == rc_stack && dst_lo_rc == rc_float) {
2121 __ unspill(as_FloatRegister(Matcher::_regEncode[dst_lo]),
2122 ireg == Op_VecD ? __ D : __ Q,
2123 ra_->reg2offset(src_lo));
2124 } else {
2125 ShouldNotReachHere();
2126 }
2127 }
2128 } else if (cbuf) {
2129 C2_MacroAssembler _masm(cbuf);
2130 switch (src_lo_rc) {
2131 case rc_int:
2132 if (dst_lo_rc == rc_int) { // gpr --> gpr copy
2133 if (is64) {
2134 __ mov(as_Register(Matcher::_regEncode[dst_lo]),
2135 as_Register(Matcher::_regEncode[src_lo]));
2136 } else {
2137 C2_MacroAssembler _masm(cbuf);
2138 __ movw(as_Register(Matcher::_regEncode[dst_lo]),
2139 as_Register(Matcher::_regEncode[src_lo]));
2140 }
2141 } else if (dst_lo_rc == rc_float) { // gpr --> fpr copy
2142 if (is64) {
2143 __ fmovd(as_FloatRegister(Matcher::_regEncode[dst_lo]),
2144 as_Register(Matcher::_regEncode[src_lo]));
2145 } else {
2146 __ fmovs(as_FloatRegister(Matcher::_regEncode[dst_lo]),
2147 as_Register(Matcher::_regEncode[src_lo]));
2148 }
2149 } else { // gpr --> stack spill
2150 assert(dst_lo_rc == rc_stack, "spill to bad register class");
2151 __ spill(as_Register(Matcher::_regEncode[src_lo]), is64, dst_offset);
2152 }
2153 break;
2154 case rc_float:
2155 if (dst_lo_rc == rc_int) { // fpr --> gpr copy
2156 if (is64) {
2157 __ fmovd(as_Register(Matcher::_regEncode[dst_lo]),
2158 as_FloatRegister(Matcher::_regEncode[src_lo]));
2159 } else {
2160 __ fmovs(as_Register(Matcher::_regEncode[dst_lo]),
2161 as_FloatRegister(Matcher::_regEncode[src_lo]));
2162 }
2163 } else if (dst_lo_rc == rc_float) { // fpr --> fpr copy
2164 if (is64) {
2165 __ fmovd(as_FloatRegister(Matcher::_regEncode[dst_lo]),
2166 as_FloatRegister(Matcher::_regEncode[src_lo]));
2167 } else {
2168 __ fmovs(as_FloatRegister(Matcher::_regEncode[dst_lo]),
2169 as_FloatRegister(Matcher::_regEncode[src_lo]));
2170 }
2171 } else { // fpr --> stack spill
2172 assert(dst_lo_rc == rc_stack, "spill to bad register class");
2173 __ spill(as_FloatRegister(Matcher::_regEncode[src_lo]),
2174 is64 ? __ D : __ S, dst_offset);
2175 }
2176 break;
2177 case rc_stack:
2178 if (dst_lo_rc == rc_int) { // stack --> gpr load
2179 __ unspill(as_Register(Matcher::_regEncode[dst_lo]), is64, src_offset);
2180 } else if (dst_lo_rc == rc_float) { // stack --> fpr load
2181 __ unspill(as_FloatRegister(Matcher::_regEncode[dst_lo]),
2182 is64 ? __ D : __ S, src_offset);
2183 } else { // stack --> stack copy
2184 assert(dst_lo_rc == rc_stack, "spill to bad register class");
2185 __ unspill(rscratch1, is64, src_offset);
2186 __ spill(rscratch1, is64, dst_offset);
2187 }
2188 break;
2189 default:
2190 assert(false, "bad rc_class for spill");
2191 ShouldNotReachHere();
2192 }
2193 }
2194
2195 if (st) {
2196 st->print("spill ");
2197 if (src_lo_rc == rc_stack) {
2198 st->print("[sp, #%d] -> ", ra_->reg2offset(src_lo));
2199 } else {
2200 st->print("%s -> ", Matcher::regName[src_lo]);
2201 }
2202 if (dst_lo_rc == rc_stack) {
2203 st->print("[sp, #%d]", ra_->reg2offset(dst_lo));
2204 } else {
2205 st->print("%s", Matcher::regName[dst_lo]);
2206 }
2207 if (bottom_type()->isa_vect() != NULL) {
2208 int vsize = 0;
2209 switch (ideal_reg()) {
2210 case Op_VecD:
2211 vsize = 64;
2212 break;
2213 case Op_VecX:
2214 vsize = 128;
2215 break;
2216 case Op_VecA:
2217 vsize = Matcher::scalable_vector_reg_size(T_BYTE) * 8;
2218 break;
2219 default:
2220 assert(false, "bad register type for spill");
2221 ShouldNotReachHere();
2222 }
2223 st->print("\t# vector spill size = %d", vsize);
2224 } else {
2225 st->print("\t# spill size = %d", is64 ? 64 : 32);
2226 }
2227 }
2228
2229 return 0;
2230
2231 }
2232
2233 #ifndef PRODUCT
2234 void MachSpillCopyNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
2235 if (!ra_)
2236 st->print("N%d = SpillCopy(N%d)", _idx, in(1)->_idx);
2237 else
2238 implementation(NULL, ra_, false, st);
2239 }
2240 #endif
2241
2242 void MachSpillCopyNode::emit(CodeBuffer &cbuf, PhaseRegAlloc *ra_) const {
2243 implementation(&cbuf, ra_, false, NULL);
2244 }
2245
2246 uint MachSpillCopyNode::size(PhaseRegAlloc *ra_) const {
2247 return MachNode::size(ra_);
2248 }
2249
2250 //=============================================================================
2251
2252 #ifndef PRODUCT
2253 void BoxLockNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
2254 int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
2255 int reg = ra_->get_reg_first(this);
2256 st->print("add %s, rsp, #%d]\t# box lock",
2257 Matcher::regName[reg], offset);
2258 }
2259 #endif
2260
2261 void BoxLockNode::emit(CodeBuffer &cbuf, PhaseRegAlloc *ra_) const {
2262 C2_MacroAssembler _masm(&cbuf);
2263
2264 int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
2265 int reg = ra_->get_encode(this);
2266
2267 // This add will handle any 24-bit signed offset. 24 bits allows an
2268 // 8 megabyte stack frame.
2269 __ add(as_Register(reg), sp, offset);
2270 }
2271
2272 uint BoxLockNode::size(PhaseRegAlloc *ra_) const {
2273 // BoxLockNode is not a MachNode, so we can't just call MachNode::size(ra_).
2274 int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
2275
2276 if (Assembler::operand_valid_for_add_sub_immediate(offset)) {
2277 return NativeInstruction::instruction_size;
2278 } else {
2279 return 2 * NativeInstruction::instruction_size;
2280 }
2281 }
2282
2283 //=============================================================================
2284
2285 #ifndef PRODUCT
2286 void MachUEPNode::format(PhaseRegAlloc* ra_, outputStream* st) const
2287 {
2288 st->print_cr("# MachUEPNode");
2289 if (UseCompressedClassPointers) {
2290 st->print_cr("\tldrw rscratch1, j_rarg0 + oopDesc::klass_offset_in_bytes()]\t# compressed klass");
2291 if (CompressedKlassPointers::shift() != 0) {
2292 st->print_cr("\tdecode_klass_not_null rscratch1, rscratch1");
2293 }
2294 } else {
2295 st->print_cr("\tldr rscratch1, j_rarg0 + oopDesc::klass_offset_in_bytes()]\t# compressed klass");
2296 }
2297 st->print_cr("\tcmp r0, rscratch1\t # Inline cache check");
2298 st->print_cr("\tbne, SharedRuntime::_ic_miss_stub");
2299 }
2300 #endif
2301
2302 void MachUEPNode::emit(CodeBuffer& cbuf, PhaseRegAlloc* ra_) const
2303 {
2304 // This is the unverified entry point.
2305 C2_MacroAssembler _masm(&cbuf);
2306
2307 __ cmp_klass(j_rarg0, rscratch2, rscratch1);
2308 Label skip;
2309 // TODO
2310 // can we avoid this skip and still use a reloc?
2311 __ br(Assembler::EQ, skip);
2312 __ far_jump(RuntimeAddress(SharedRuntime::get_ic_miss_stub()));
2313 __ bind(skip);
2314 }
2315
2316 uint MachUEPNode::size(PhaseRegAlloc* ra_) const
2317 {
2318 return MachNode::size(ra_);
2319 }
2320
2321 // REQUIRED EMIT CODE
2322
2323 //=============================================================================
2324
2325 // Emit exception handler code.
2326 int HandlerImpl::emit_exception_handler(CodeBuffer& cbuf)
2327 {
2328 // mov rscratch1 #exception_blob_entry_point
2329 // br rscratch1
2330 // Note that the code buffer's insts_mark is always relative to insts.
2331 // That's why we must use the macroassembler to generate a handler.
2332 C2_MacroAssembler _masm(&cbuf);
2333 address base = __ start_a_stub(size_exception_handler());
2334 if (base == NULL) {
2335 ciEnv::current()->record_failure("CodeCache is full");
2336 return 0; // CodeBuffer::expand failed
2337 }
2338 int offset = __ offset();
2339 __ far_jump(RuntimeAddress(OptoRuntime::exception_blob()->entry_point()));
2340 assert(__ offset() - offset <= (int) size_exception_handler(), "overflow");
2341 __ end_a_stub();
2342 return offset;
2343 }
2344
2345 // Emit deopt handler code.
2346 int HandlerImpl::emit_deopt_handler(CodeBuffer& cbuf)
2347 {
2348 // Note that the code buffer's insts_mark is always relative to insts.
2349 // That's why we must use the macroassembler to generate a handler.
2350 C2_MacroAssembler _masm(&cbuf);
2351 address base = __ start_a_stub(size_deopt_handler());
2352 if (base == NULL) {
2353 ciEnv::current()->record_failure("CodeCache is full");
2354 return 0; // CodeBuffer::expand failed
2355 }
2356 int offset = __ offset();
2357
2358 __ adr(lr, __ pc());
2359 __ far_jump(RuntimeAddress(SharedRuntime::deopt_blob()->unpack()));
2360
2361 assert(__ offset() - offset <= (int) size_deopt_handler(), "overflow");
2362 __ end_a_stub();
2363 return offset;
2364 }
2365
2366 // REQUIRED MATCHER CODE
2367
2368 //=============================================================================
2369
2370 const bool Matcher::match_rule_supported(int opcode) {
2371 if (!has_match_rule(opcode))
2372 return false;
2373
2374 bool ret_value = true;
2375 switch (opcode) {
2376 case Op_CacheWB:
2377 case Op_CacheWBPreSync:
2378 case Op_CacheWBPostSync:
2379 if (!VM_Version::supports_data_cache_line_flush()) {
2380 ret_value = false;
2381 }
2382 break;
2383 }
2384
2385 return ret_value; // Per default match rules are supported.
2386 }
2387
2388 // Identify extra cases that we might want to provide match rules for vector nodes and
2389 // other intrinsics guarded with vector length (vlen) and element type (bt).
2390 const bool Matcher::match_rule_supported_vector(int opcode, int vlen, BasicType bt) {
2391 if (!match_rule_supported(opcode)) {
2392 return false;
2393 }
2394 int bit_size = vlen * type2aelembytes(bt) * 8;
2395 if (UseSVE == 0 && bit_size > 128) {
2396 return false;
2397 }
2398 if (UseSVE > 0) {
2399 return op_sve_supported(opcode, vlen, bt);
2400 } else { // NEON
2401 // Special cases
2402 switch (opcode) {
2403 case Op_VectorMaskCmp:
2404 // We don't have VectorReinterpret with bit_size less than 64 support for
2405 // now, even for byte type. To be refined with fully VectorCast support.
2406 case Op_VectorReinterpret:
2407 if (vlen < 2 || bit_size < 64) {
2408 return false;
2409 }
2410 break;
2411 case Op_MulAddVS2VI:
2412 if (bit_size < 128) {
2413 return false;
2414 }
2415 break;
2416 case Op_MulVL:
2417 return false;
2418 case Op_VectorLoadShuffle:
2419 case Op_VectorRearrange:
2420 if (vlen < 4) {
2421 return false;
2422 }
2423 break;
2424 // Some types of VectorCast are not implemented for now.
2425 case Op_VectorCastI2X:
2426 if (bt == T_BYTE) {
2427 return false;
2428 }
2429 break;
2430 case Op_VectorCastS2X:
2431 if (vlen < 4 || bit_size < 64) {
2432 return false;
2433 }
2434 break;
2435 case Op_VectorCastF2X:
2436 case Op_VectorCastD2X:
2437 if (bt == T_INT || bt == T_SHORT || bt == T_BYTE || bt == T_LONG) {
2438 return false;
2439 }
2440 break;
2441 case Op_LoadVectorGather:
2442 case Op_StoreVectorScatter:
2443 return false;
2444 default:
2445 break;
2446 }
2447 }
2448 return vector_size_supported(bt, vlen);
2449 }
2450
2451 const RegMask* Matcher::predicate_reg_mask(void) {
2452 return &_PR_REG_mask;
2453 }
2454
2455 const TypeVect* Matcher::predicate_reg_type(const Type* elemTy, int length) {
2456 return new TypeVectMask(elemTy, length);
2457 }
2458
2459 // Vector calling convention not yet implemented.
2460 const bool Matcher::supports_vector_calling_convention(void) {
2461 return false;
2462 }
2463
2464 OptoRegPair Matcher::vector_return_value(uint ideal_reg) {
2465 Unimplemented();
2466 return OptoRegPair(0, 0);
2467 }
2468
2469 // Is this branch offset short enough that a short branch can be used?
2470 //
2471 // NOTE: If the platform does not provide any short branch variants, then
2472 // this method should return false for offset 0.
2473 bool Matcher::is_short_branch_offset(int rule, int br_size, int offset) {
2474 // The passed offset is relative to address of the branch.
2475
2476 return (-32768 <= offset && offset < 32768);
2477 }
2478
2479 // Vector width in bytes.
2480 const int Matcher::vector_width_in_bytes(BasicType bt) {
2481 // The MaxVectorSize should have been set by detecting SVE max vector register size.
2482 int size = MIN2((UseSVE > 0) ? 256 : 16, (int)MaxVectorSize);
2483 // Minimum 2 values in vector
2484 if (size < 2*type2aelembytes(bt)) size = 0;
2485 // But never < 4
2486 if (size < 4) size = 0;
2487 return size;
2488 }
2489
2490 // Limits on vector size (number of elements) loaded into vector.
2491 const int Matcher::max_vector_size(const BasicType bt) {
2492 return vector_width_in_bytes(bt)/type2aelembytes(bt);
2493 }
2494
2495 const int Matcher::min_vector_size(const BasicType bt) {
2496 int max_size = max_vector_size(bt);
2497 // Limit the min vector size to 8 bytes.
2498 int size = 8 / type2aelembytes(bt);
2499 if (bt == T_BYTE) {
2500 // To support vector api shuffle/rearrange.
2501 size = 4;
2502 } else if (bt == T_BOOLEAN) {
2503 // To support vector api load/store mask.
2504 size = 2;
2505 }
2506 if (size < 2) size = 2;
2507 return MIN2(size, max_size);
2508 }
2509
2510 // Actual max scalable vector register length.
2511 const int Matcher::scalable_vector_reg_size(const BasicType bt) {
2512 return Matcher::max_vector_size(bt);
2513 }
2514
2515 // Vector ideal reg.
2516 const uint Matcher::vector_ideal_reg(int len) {
2517 if (UseSVE > 0 && 2 <= len && len <= 256) {
2518 return Op_VecA;
2519 }
2520 switch(len) {
2521 // For 16-bit/32-bit mask vector, reuse VecD.
2522 case 2:
2523 case 4:
2524 case 8: return Op_VecD;
2525 case 16: return Op_VecX;
2526 }
2527 ShouldNotReachHere();
2528 return 0;
2529 }
2530
2531 MachOper* Matcher::pd_specialize_generic_vector_operand(MachOper* original_opnd, uint ideal_reg, bool is_temp) {
2532 ShouldNotReachHere(); // generic vector operands not supported
2533 return NULL;
2534 }
2535
2536 bool Matcher::is_reg2reg_move(MachNode* m) {
2537 ShouldNotReachHere(); // generic vector operands not supported
2538 return false;
2539 }
2540
2541 bool Matcher::is_generic_vector(MachOper* opnd) {
2542 ShouldNotReachHere(); // generic vector operands not supported
2543 return false;
2544 }
2545
2546 // Return whether or not this register is ever used as an argument.
2547 // This function is used on startup to build the trampoline stubs in
2548 // generateOptoStub. Registers not mentioned will be killed by the VM
2549 // call in the trampoline, and arguments in those registers not be
2550 // available to the callee.
2551 bool Matcher::can_be_java_arg(int reg)
2552 {
2553 return
2554 reg == R0_num || reg == R0_H_num ||
2555 reg == R1_num || reg == R1_H_num ||
2556 reg == R2_num || reg == R2_H_num ||
2557 reg == R3_num || reg == R3_H_num ||
2558 reg == R4_num || reg == R4_H_num ||
2559 reg == R5_num || reg == R5_H_num ||
2560 reg == R6_num || reg == R6_H_num ||
2561 reg == R7_num || reg == R7_H_num ||
2562 reg == V0_num || reg == V0_H_num ||
2563 reg == V1_num || reg == V1_H_num ||
2564 reg == V2_num || reg == V2_H_num ||
2565 reg == V3_num || reg == V3_H_num ||
2566 reg == V4_num || reg == V4_H_num ||
2567 reg == V5_num || reg == V5_H_num ||
2568 reg == V6_num || reg == V6_H_num ||
2569 reg == V7_num || reg == V7_H_num;
2570 }
2571
2572 bool Matcher::is_spillable_arg(int reg)
2573 {
2574 return can_be_java_arg(reg);
2575 }
2576
2577 uint Matcher::int_pressure_limit()
2578 {
2579 // JDK-8183543: When taking the number of available registers as int
2580 // register pressure threshold, the jtreg test:
2581 // test/hotspot/jtreg/compiler/regalloc/TestC2IntPressure.java
2582 // failed due to C2 compilation failure with
2583 // "COMPILE SKIPPED: failed spill-split-recycle sanity check".
2584 //
2585 // A derived pointer is live at CallNode and then is flagged by RA
2586 // as a spilled LRG. Spilling heuristics(Spill-USE) explicitly skip
2587 // derived pointers and lastly fail to spill after reaching maximum
2588 // number of iterations. Lowering the default pressure threshold to
2589 // (_NO_SPECIAL_REG32_mask.Size() minus 1) forces CallNode to become
2590 // a high register pressure area of the code so that split_DEF can
2591 // generate DefinitionSpillCopy for the derived pointer.
2592 uint default_int_pressure_threshold = _NO_SPECIAL_REG32_mask.Size() - 1;
2593 if (!PreserveFramePointer) {
2594 // When PreserveFramePointer is off, frame pointer is allocatable,
2595 // but different from other SOC registers, it is excluded from
2596 // fatproj's mask because its save type is No-Save. Decrease 1 to
2597 // ensure high pressure at fatproj when PreserveFramePointer is off.
2598 // See check_pressure_at_fatproj().
2599 default_int_pressure_threshold--;
2600 }
2601 return (INTPRESSURE == -1) ? default_int_pressure_threshold : INTPRESSURE;
2602 }
2603
2604 uint Matcher::float_pressure_limit()
2605 {
2606 // _FLOAT_REG_mask is generated by adlc from the float_reg register class.
2607 return (FLOATPRESSURE == -1) ? _FLOAT_REG_mask.Size() : FLOATPRESSURE;
2608 }
2609
2610 bool Matcher::use_asm_for_ldiv_by_con(jlong divisor) {
2611 return false;
2612 }
2613
2614 RegMask Matcher::divI_proj_mask() {
2615 ShouldNotReachHere();
2616 return RegMask();
2617 }
2618
2619 // Register for MODI projection of divmodI.
2620 RegMask Matcher::modI_proj_mask() {
2621 ShouldNotReachHere();
2622 return RegMask();
2623 }
2624
2625 // Register for DIVL projection of divmodL.
2626 RegMask Matcher::divL_proj_mask() {
2627 ShouldNotReachHere();
2628 return RegMask();
2629 }
2630
2631 // Register for MODL projection of divmodL.
2632 RegMask Matcher::modL_proj_mask() {
2633 ShouldNotReachHere();
2634 return RegMask();
2635 }
2636
2637 const RegMask Matcher::method_handle_invoke_SP_save_mask() {
2638 return FP_REG_mask();
2639 }
2640
2641 bool size_fits_all_mem_uses(AddPNode* addp, int shift) {
2642 for (DUIterator_Fast imax, i = addp->fast_outs(imax); i < imax; i++) {
2643 Node* u = addp->fast_out(i);
2644 if (u->is_LoadStore()) {
2645 // On AArch64, LoadStoreNodes (i.e. compare and swap
2646 // instructions) only take register indirect as an operand, so
2647 // any attempt to use an AddPNode as an input to a LoadStoreNode
2648 // must fail.
2649 return false;
2650 }
2651 if (u->is_Mem()) {
2652 int opsize = u->as_Mem()->memory_size();
2653 assert(opsize > 0, "unexpected memory operand size");
2654 if (u->as_Mem()->memory_size() != (1<<shift)) {
2655 return false;
2656 }
2657 }
2658 }
2659 return true;
2660 }
2661
2662 // Should the matcher clone input 'm' of node 'n'?
2663 bool Matcher::pd_clone_node(Node* n, Node* m, Matcher::MStack& mstack) {
2664 if (is_vshift_con_pattern(n, m)) { // ShiftV src (ShiftCntV con)
2665 mstack.push(m, Visit); // m = ShiftCntV
2666 return true;
2667 }
2668 return false;
2669 }
2670
2671 // Should the Matcher clone shifts on addressing modes, expecting them
2672 // to be subsumed into complex addressing expressions or compute them
2673 // into registers?
2674 bool Matcher::pd_clone_address_expressions(AddPNode* m, Matcher::MStack& mstack, VectorSet& address_visited) {
2675 if (clone_base_plus_offset_address(m, mstack, address_visited)) {
2676 return true;
2677 }
2678
2679 Node *off = m->in(AddPNode::Offset);
2680 if (off->Opcode() == Op_LShiftL && off->in(2)->is_Con() &&
2681 size_fits_all_mem_uses(m, off->in(2)->get_int()) &&
2682 // Are there other uses besides address expressions?
2683 !is_visited(off)) {
2684 address_visited.set(off->_idx); // Flag as address_visited
2685 mstack.push(off->in(2), Visit);
2686 Node *conv = off->in(1);
2687 if (conv->Opcode() == Op_ConvI2L &&
2688 // Are there other uses besides address expressions?
2689 !is_visited(conv)) {
2690 address_visited.set(conv->_idx); // Flag as address_visited
2691 mstack.push(conv->in(1), Pre_Visit);
2692 } else {
2693 mstack.push(conv, Pre_Visit);
2694 }
2695 address_visited.test_set(m->_idx); // Flag as address_visited
2696 mstack.push(m->in(AddPNode::Address), Pre_Visit);
2697 mstack.push(m->in(AddPNode::Base), Pre_Visit);
2698 return true;
2699 } else if (off->Opcode() == Op_ConvI2L &&
2700 // Are there other uses besides address expressions?
2701 !is_visited(off)) {
2702 address_visited.test_set(m->_idx); // Flag as address_visited
2703 address_visited.set(off->_idx); // Flag as address_visited
2704 mstack.push(off->in(1), Pre_Visit);
2705 mstack.push(m->in(AddPNode::Address), Pre_Visit);
2706 mstack.push(m->in(AddPNode::Base), Pre_Visit);
2707 return true;
2708 }
2709 return false;
2710 }
2711
2712 #define MOV_VOLATILE(REG, BASE, INDEX, SCALE, DISP, SCRATCH, INSN) \
2713 C2_MacroAssembler _masm(&cbuf); \
2714 { \
2715 guarantee(INDEX == -1, "mode not permitted for volatile"); \
2716 guarantee(DISP == 0, "mode not permitted for volatile"); \
2717 guarantee(SCALE == 0, "mode not permitted for volatile"); \
2718 __ INSN(REG, as_Register(BASE)); \
2719 }
2720
2721
2722 static Address mem2address(int opcode, Register base, int index, int size, int disp)
2723 {
2724 Address::extend scale;
2725
2726 // Hooboy, this is fugly. We need a way to communicate to the
2727 // encoder that the index needs to be sign extended, so we have to
2728 // enumerate all the cases.
2729 switch (opcode) {
2730 case INDINDEXSCALEDI2L:
2731 case INDINDEXSCALEDI2LN:
2732 case INDINDEXI2L:
2733 case INDINDEXI2LN:
2734 scale = Address::sxtw(size);
2735 break;
2736 default:
2737 scale = Address::lsl(size);
2738 }
2739
2740 if (index == -1) {
2741 return Address(base, disp);
2742 } else {
2743 assert(disp == 0, "unsupported address mode: disp = %d", disp);
2744 return Address(base, as_Register(index), scale);
2745 }
2746 }
2747
2748
2749 typedef void (MacroAssembler::* mem_insn)(Register Rt, const Address &adr);
2750 typedef void (MacroAssembler::* mem_insn2)(Register Rt, Register adr);
2751 typedef void (MacroAssembler::* mem_float_insn)(FloatRegister Rt, const Address &adr);
2752 typedef void (MacroAssembler::* mem_vector_insn)(FloatRegister Rt,
2753 MacroAssembler::SIMD_RegVariant T, const Address &adr);
2754
2755 // Used for all non-volatile memory accesses. The use of
2756 // $mem->opcode() to discover whether this pattern uses sign-extended
2757 // offsets is something of a kludge.
2758 static void loadStore(C2_MacroAssembler masm, mem_insn insn,
2759 Register reg, int opcode,
2760 Register base, int index, int scale, int disp,
2761 int size_in_memory)
2762 {
2763 Address addr = mem2address(opcode, base, index, scale, disp);
2764 if (addr.getMode() == Address::base_plus_offset) {
2765 /* If we get an out-of-range offset it is a bug in the compiler,
2766 so we assert here. */
2767 assert(Address::offset_ok_for_immed(addr.offset(), exact_log2(size_in_memory)),
2768 "c2 compiler bug");
2769 /* Fix up any out-of-range offsets. */
2770 assert_different_registers(rscratch1, base);
2771 assert_different_registers(rscratch1, reg);
2772 addr = masm.legitimize_address(addr, size_in_memory, rscratch1);
2773 }
2774 (masm.*insn)(reg, addr);
2775 }
2776
2777 static void loadStore(C2_MacroAssembler masm, mem_float_insn insn,
2778 FloatRegister reg, int opcode,
2779 Register base, int index, int size, int disp,
2780 int size_in_memory)
2781 {
2782 Address::extend scale;
2783
2784 switch (opcode) {
2785 case INDINDEXSCALEDI2L:
2786 case INDINDEXSCALEDI2LN:
2787 scale = Address::sxtw(size);
2788 break;
2789 default:
2790 scale = Address::lsl(size);
2791 }
2792
2793 if (index == -1) {
2794 /* If we get an out-of-range offset it is a bug in the compiler,
2795 so we assert here. */
2796 assert(Address::offset_ok_for_immed(disp, exact_log2(size_in_memory)), "c2 compiler bug");
2797 /* Fix up any out-of-range offsets. */
2798 assert_different_registers(rscratch1, base);
2799 Address addr = Address(base, disp);
2800 addr = masm.legitimize_address(addr, size_in_memory, rscratch1);
2801 (masm.*insn)(reg, addr);
2802 } else {
2803 assert(disp == 0, "unsupported address mode: disp = %d", disp);
2804 (masm.*insn)(reg, Address(base, as_Register(index), scale));
2805 }
2806 }
2807
2808 static void loadStore(C2_MacroAssembler masm, mem_vector_insn insn,
2809 FloatRegister reg, MacroAssembler::SIMD_RegVariant T,
2810 int opcode, Register base, int index, int size, int disp)
2811 {
2812 if (index == -1) {
2813 (masm.*insn)(reg, T, Address(base, disp));
2814 } else {
2815 assert(disp == 0, "unsupported address mode");
2816 (masm.*insn)(reg, T, Address(base, as_Register(index), Address::lsl(size)));
2817 }
2818 }
2819
2820 %}
2821
2822
2823
2824 //----------ENCODING BLOCK-----------------------------------------------------
2825 // This block specifies the encoding classes used by the compiler to
2826 // output byte streams. Encoding classes are parameterized macros
2827 // used by Machine Instruction Nodes in order to generate the bit
2828 // encoding of the instruction. Operands specify their base encoding
2829 // interface with the interface keyword. There are currently
2830 // supported four interfaces, REG_INTER, CONST_INTER, MEMORY_INTER, &
2831 // COND_INTER. REG_INTER causes an operand to generate a function
2832 // which returns its register number when queried. CONST_INTER causes
2833 // an operand to generate a function which returns the value of the
2834 // constant when queried. MEMORY_INTER causes an operand to generate
2835 // four functions which return the Base Register, the Index Register,
2836 // the Scale Value, and the Offset Value of the operand when queried.
2837 // COND_INTER causes an operand to generate six functions which return
2838 // the encoding code (ie - encoding bits for the instruction)
2839 // associated with each basic boolean condition for a conditional
2840 // instruction.
2841 //
2842 // Instructions specify two basic values for encoding. Again, a
2843 // function is available to check if the constant displacement is an
2844 // oop. They use the ins_encode keyword to specify their encoding
2845 // classes (which must be a sequence of enc_class names, and their
2846 // parameters, specified in the encoding block), and they use the
2847 // opcode keyword to specify, in order, their primary, secondary, and
2848 // tertiary opcode. Only the opcode sections which a particular
2849 // instruction needs for encoding need to be specified.
2850 encode %{
2851 // Build emit functions for each basic byte or larger field in the
2852 // intel encoding scheme (opcode, rm, sib, immediate), and call them
2853 // from C++ code in the enc_class source block. Emit functions will
2854 // live in the main source block for now. In future, we can
2855 // generalize this by adding a syntax that specifies the sizes of
2856 // fields in an order, so that the adlc can build the emit functions
2857 // automagically
2858
2859 // catch all for unimplemented encodings
2860 enc_class enc_unimplemented %{
2861 C2_MacroAssembler _masm(&cbuf);
2862 __ unimplemented("C2 catch all");
2863 %}
2864
2865 // BEGIN Non-volatile memory access
2866
2867 // This encoding class is generated automatically from ad_encode.m4.
2868 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2869 enc_class aarch64_enc_ldrsbw(iRegI dst, memory1 mem) %{
2870 Register dst_reg = as_Register($dst$$reg);
2871 loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrsbw, dst_reg, $mem->opcode(),
2872 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
2873 %}
2874
2875 // This encoding class is generated automatically from ad_encode.m4.
2876 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2877 enc_class aarch64_enc_ldrsb(iRegI dst, memory1 mem) %{
2878 Register dst_reg = as_Register($dst$$reg);
2879 loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrsb, dst_reg, $mem->opcode(),
2880 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
2881 %}
2882
2883 // This encoding class is generated automatically from ad_encode.m4.
2884 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2885 enc_class aarch64_enc_ldrb(iRegI dst, memory1 mem) %{
2886 Register dst_reg = as_Register($dst$$reg);
2887 loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrb, dst_reg, $mem->opcode(),
2888 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
2889 %}
2890
2891 // This encoding class is generated automatically from ad_encode.m4.
2892 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2893 enc_class aarch64_enc_ldrb(iRegL dst, memory1 mem) %{
2894 Register dst_reg = as_Register($dst$$reg);
2895 loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrb, dst_reg, $mem->opcode(),
2896 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
2897 %}
2898
2899 // This encoding class is generated automatically from ad_encode.m4.
2900 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2901 enc_class aarch64_enc_ldrshw(iRegI dst, memory2 mem) %{
2902 Register dst_reg = as_Register($dst$$reg);
2903 loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrshw, dst_reg, $mem->opcode(),
2904 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
2905 %}
2906
2907 // This encoding class is generated automatically from ad_encode.m4.
2908 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2909 enc_class aarch64_enc_ldrsh(iRegI dst, memory2 mem) %{
2910 Register dst_reg = as_Register($dst$$reg);
2911 loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrsh, dst_reg, $mem->opcode(),
2912 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
2913 %}
2914
2915 // This encoding class is generated automatically from ad_encode.m4.
2916 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2917 enc_class aarch64_enc_ldrh(iRegI dst, memory2 mem) %{
2918 Register dst_reg = as_Register($dst$$reg);
2919 loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrh, dst_reg, $mem->opcode(),
2920 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
2921 %}
2922
2923 // This encoding class is generated automatically from ad_encode.m4.
2924 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2925 enc_class aarch64_enc_ldrh(iRegL dst, memory2 mem) %{
2926 Register dst_reg = as_Register($dst$$reg);
2927 loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrh, dst_reg, $mem->opcode(),
2928 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
2929 %}
2930
2931 // This encoding class is generated automatically from ad_encode.m4.
2932 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2933 enc_class aarch64_enc_ldrw(iRegI dst, memory4 mem) %{
2934 Register dst_reg = as_Register($dst$$reg);
2935 loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrw, dst_reg, $mem->opcode(),
2936 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
2937 %}
2938
2939 // This encoding class is generated automatically from ad_encode.m4.
2940 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2941 enc_class aarch64_enc_ldrw(iRegL dst, memory4 mem) %{
2942 Register dst_reg = as_Register($dst$$reg);
2943 loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrw, dst_reg, $mem->opcode(),
2944 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
2945 %}
2946
2947 // This encoding class is generated automatically from ad_encode.m4.
2948 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2949 enc_class aarch64_enc_ldrsw(iRegL dst, memory4 mem) %{
2950 Register dst_reg = as_Register($dst$$reg);
2951 loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrsw, dst_reg, $mem->opcode(),
2952 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
2953 %}
2954
2955 // This encoding class is generated automatically from ad_encode.m4.
2956 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2957 enc_class aarch64_enc_ldr(iRegL dst, memory8 mem) %{
2958 Register dst_reg = as_Register($dst$$reg);
2959 loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldr, dst_reg, $mem->opcode(),
2960 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
2961 %}
2962
2963 // This encoding class is generated automatically from ad_encode.m4.
2964 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2965 enc_class aarch64_enc_ldrs(vRegF dst, memory4 mem) %{
2966 FloatRegister dst_reg = as_FloatRegister($dst$$reg);
2967 loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrs, dst_reg, $mem->opcode(),
2968 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
2969 %}
2970
2971 // This encoding class is generated automatically from ad_encode.m4.
2972 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2973 enc_class aarch64_enc_ldrd(vRegD dst, memory8 mem) %{
2974 FloatRegister dst_reg = as_FloatRegister($dst$$reg);
2975 loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrd, dst_reg, $mem->opcode(),
2976 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
2977 %}
2978
2979 // This encoding class is generated automatically from ad_encode.m4.
2980 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2981 enc_class aarch64_enc_strb(iRegI src, memory1 mem) %{
2982 Register src_reg = as_Register($src$$reg);
2983 loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::strb, src_reg, $mem->opcode(),
2984 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
2985 %}
2986
2987 // This encoding class is generated automatically from ad_encode.m4.
2988 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2989 enc_class aarch64_enc_strb0(memory1 mem) %{
2990 C2_MacroAssembler _masm(&cbuf);
2991 loadStore(_masm, &MacroAssembler::strb, zr, $mem->opcode(),
2992 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
2993 %}
2994
2995 // This encoding class is generated automatically from ad_encode.m4.
2996 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2997 enc_class aarch64_enc_strh(iRegI src, memory2 mem) %{
2998 Register src_reg = as_Register($src$$reg);
2999 loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::strh, src_reg, $mem->opcode(),
3000 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
3001 %}
3002
3003 // This encoding class is generated automatically from ad_encode.m4.
3004 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
3005 enc_class aarch64_enc_strh0(memory2 mem) %{
3006 C2_MacroAssembler _masm(&cbuf);
3007 loadStore(_masm, &MacroAssembler::strh, zr, $mem->opcode(),
3008 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
3009 %}
3010
3011 // This encoding class is generated automatically from ad_encode.m4.
3012 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
3013 enc_class aarch64_enc_strw(iRegI src, memory4 mem) %{
3014 Register src_reg = as_Register($src$$reg);
3015 loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::strw, src_reg, $mem->opcode(),
3016 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
3017 %}
3018
3019 // This encoding class is generated automatically from ad_encode.m4.
3020 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
3021 enc_class aarch64_enc_strw0(memory4 mem) %{
3022 C2_MacroAssembler _masm(&cbuf);
3023 loadStore(_masm, &MacroAssembler::strw, zr, $mem->opcode(),
3024 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
3025 %}
3026
3027 // This encoding class is generated automatically from ad_encode.m4.
3028 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
3029 enc_class aarch64_enc_str(iRegL src, memory8 mem) %{
3030 Register src_reg = as_Register($src$$reg);
3031 // we sometimes get asked to store the stack pointer into the
3032 // current thread -- we cannot do that directly on AArch64
3033 if (src_reg == r31_sp) {
3034 C2_MacroAssembler _masm(&cbuf);
3035 assert(as_Register($mem$$base) == rthread, "unexpected store for sp");
3036 __ mov(rscratch2, sp);
3037 src_reg = rscratch2;
3038 }
3039 loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::str, src_reg, $mem->opcode(),
3040 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
3041 %}
3042
3043 // This encoding class is generated automatically from ad_encode.m4.
3044 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
3045 enc_class aarch64_enc_str0(memory8 mem) %{
3046 C2_MacroAssembler _masm(&cbuf);
3047 loadStore(_masm, &MacroAssembler::str, zr, $mem->opcode(),
3048 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
3049 %}
3050
3051 // This encoding class is generated automatically from ad_encode.m4.
3052 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
3053 enc_class aarch64_enc_strs(vRegF src, memory4 mem) %{
3054 FloatRegister src_reg = as_FloatRegister($src$$reg);
3055 loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::strs, src_reg, $mem->opcode(),
3056 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
3057 %}
3058
3059 // This encoding class is generated automatically from ad_encode.m4.
3060 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
3061 enc_class aarch64_enc_strd(vRegD src, memory8 mem) %{
3062 FloatRegister src_reg = as_FloatRegister($src$$reg);
3063 loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::strd, src_reg, $mem->opcode(),
3064 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
3065 %}
3066
3067 // This encoding class is generated automatically from ad_encode.m4.
3068 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
3069 enc_class aarch64_enc_strb0_ordered(memory4 mem) %{
3070 C2_MacroAssembler _masm(&cbuf);
3071 __ membar(Assembler::StoreStore);
3072 loadStore(_masm, &MacroAssembler::strb, zr, $mem->opcode(),
3073 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
3074 %}
3075
3076 // END Non-volatile memory access
3077
3078 // Vector loads and stores
3079 enc_class aarch64_enc_ldrvH(vecD dst, memory mem) %{
3080 FloatRegister dst_reg = as_FloatRegister($dst$$reg);
3081 loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldr, dst_reg, MacroAssembler::H,
3082 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3083 %}
3084
3085 enc_class aarch64_enc_ldrvS(vecD dst, memory mem) %{
3086 FloatRegister dst_reg = as_FloatRegister($dst$$reg);
3087 loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldr, dst_reg, MacroAssembler::S,
3088 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3089 %}
3090
3091 enc_class aarch64_enc_ldrvD(vecD dst, memory mem) %{
3092 FloatRegister dst_reg = as_FloatRegister($dst$$reg);
3093 loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldr, dst_reg, MacroAssembler::D,
3094 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3095 %}
3096
3097 enc_class aarch64_enc_ldrvQ(vecX dst, memory mem) %{
3098 FloatRegister dst_reg = as_FloatRegister($dst$$reg);
3099 loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldr, dst_reg, MacroAssembler::Q,
3100 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3101 %}
3102
3103 enc_class aarch64_enc_strvH(vecD src, memory mem) %{
3104 FloatRegister src_reg = as_FloatRegister($src$$reg);
3105 loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::str, src_reg, MacroAssembler::H,
3106 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3107 %}
3108
3109 enc_class aarch64_enc_strvS(vecD src, memory mem) %{
3110 FloatRegister src_reg = as_FloatRegister($src$$reg);
3111 loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::str, src_reg, MacroAssembler::S,
3112 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3113 %}
3114
3115 enc_class aarch64_enc_strvD(vecD src, memory mem) %{
3116 FloatRegister src_reg = as_FloatRegister($src$$reg);
3117 loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::str, src_reg, MacroAssembler::D,
3118 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3119 %}
3120
3121 enc_class aarch64_enc_strvQ(vecX src, memory mem) %{
3122 FloatRegister src_reg = as_FloatRegister($src$$reg);
3123 loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::str, src_reg, MacroAssembler::Q,
3124 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3125 %}
3126
3127 // volatile loads and stores
3128
3129 enc_class aarch64_enc_stlrb(iRegI src, memory mem) %{
3130 MOV_VOLATILE(as_Register($src$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3131 rscratch1, stlrb);
3132 %}
3133
3134 enc_class aarch64_enc_stlrh(iRegI src, memory mem) %{
3135 MOV_VOLATILE(as_Register($src$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3136 rscratch1, stlrh);
3137 %}
3138
3139 enc_class aarch64_enc_stlrw(iRegI src, memory mem) %{
3140 MOV_VOLATILE(as_Register($src$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3141 rscratch1, stlrw);
3142 %}
3143
3144
3145 enc_class aarch64_enc_ldarsbw(iRegI dst, memory mem) %{
3146 Register dst_reg = as_Register($dst$$reg);
3147 MOV_VOLATILE(dst_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3148 rscratch1, ldarb);
3149 __ sxtbw(dst_reg, dst_reg);
3150 %}
3151
3152 enc_class aarch64_enc_ldarsb(iRegL dst, memory mem) %{
3153 Register dst_reg = as_Register($dst$$reg);
3154 MOV_VOLATILE(dst_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3155 rscratch1, ldarb);
3156 __ sxtb(dst_reg, dst_reg);
3157 %}
3158
3159 enc_class aarch64_enc_ldarbw(iRegI dst, memory mem) %{
3160 MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3161 rscratch1, ldarb);
3162 %}
3163
3164 enc_class aarch64_enc_ldarb(iRegL dst, memory mem) %{
3165 MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3166 rscratch1, ldarb);
3167 %}
3168
3169 enc_class aarch64_enc_ldarshw(iRegI dst, memory mem) %{
3170 Register dst_reg = as_Register($dst$$reg);
3171 MOV_VOLATILE(dst_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3172 rscratch1, ldarh);
3173 __ sxthw(dst_reg, dst_reg);
3174 %}
3175
3176 enc_class aarch64_enc_ldarsh(iRegL dst, memory mem) %{
3177 Register dst_reg = as_Register($dst$$reg);
3178 MOV_VOLATILE(dst_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3179 rscratch1, ldarh);
3180 __ sxth(dst_reg, dst_reg);
3181 %}
3182
3183 enc_class aarch64_enc_ldarhw(iRegI dst, memory mem) %{
3184 MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3185 rscratch1, ldarh);
3186 %}
3187
3188 enc_class aarch64_enc_ldarh(iRegL dst, memory mem) %{
3189 MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3190 rscratch1, ldarh);
3191 %}
3192
3193 enc_class aarch64_enc_ldarw(iRegI dst, memory mem) %{
3194 MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3195 rscratch1, ldarw);
3196 %}
3197
3198 enc_class aarch64_enc_ldarw(iRegL dst, memory mem) %{
3199 MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3200 rscratch1, ldarw);
3201 %}
3202
3203 enc_class aarch64_enc_ldar(iRegL dst, memory mem) %{
3204 MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3205 rscratch1, ldar);
3206 %}
3207
3208 enc_class aarch64_enc_fldars(vRegF dst, memory mem) %{
3209 MOV_VOLATILE(rscratch1, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3210 rscratch1, ldarw);
3211 __ fmovs(as_FloatRegister($dst$$reg), rscratch1);
3212 %}
3213
3214 enc_class aarch64_enc_fldard(vRegD dst, memory mem) %{
3215 MOV_VOLATILE(rscratch1, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3216 rscratch1, ldar);
3217 __ fmovd(as_FloatRegister($dst$$reg), rscratch1);
3218 %}
3219
3220 enc_class aarch64_enc_stlr(iRegL src, memory mem) %{
3221 Register src_reg = as_Register($src$$reg);
3222 // we sometimes get asked to store the stack pointer into the
3223 // current thread -- we cannot do that directly on AArch64
3224 if (src_reg == r31_sp) {
3225 C2_MacroAssembler _masm(&cbuf);
3226 assert(as_Register($mem$$base) == rthread, "unexpected store for sp");
3227 __ mov(rscratch2, sp);
3228 src_reg = rscratch2;
3229 }
3230 MOV_VOLATILE(src_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3231 rscratch1, stlr);
3232 %}
3233
3234 enc_class aarch64_enc_fstlrs(vRegF src, memory mem) %{
3235 {
3236 C2_MacroAssembler _masm(&cbuf);
3237 FloatRegister src_reg = as_FloatRegister($src$$reg);
3238 __ fmovs(rscratch2, src_reg);
3239 }
3240 MOV_VOLATILE(rscratch2, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3241 rscratch1, stlrw);
3242 %}
3243
3244 enc_class aarch64_enc_fstlrd(vRegD src, memory mem) %{
3245 {
3246 C2_MacroAssembler _masm(&cbuf);
3247 FloatRegister src_reg = as_FloatRegister($src$$reg);
3248 __ fmovd(rscratch2, src_reg);
3249 }
3250 MOV_VOLATILE(rscratch2, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3251 rscratch1, stlr);
3252 %}
3253
3254 // synchronized read/update encodings
3255
3256 enc_class aarch64_enc_ldaxr(iRegL dst, memory8 mem) %{
3257 C2_MacroAssembler _masm(&cbuf);
3258 Register dst_reg = as_Register($dst$$reg);
3259 Register base = as_Register($mem$$base);
3260 int index = $mem$$index;
3261 int scale = $mem$$scale;
3262 int disp = $mem$$disp;
3263 if (index == -1) {
3264 if (disp != 0) {
3265 __ lea(rscratch1, Address(base, disp));
3266 __ ldaxr(dst_reg, rscratch1);
3267 } else {
3268 // TODO
3269 // should we ever get anything other than this case?
3270 __ ldaxr(dst_reg, base);
3271 }
3272 } else {
3273 Register index_reg = as_Register(index);
3274 if (disp == 0) {
3275 __ lea(rscratch1, Address(base, index_reg, Address::lsl(scale)));
3276 __ ldaxr(dst_reg, rscratch1);
3277 } else {
3278 __ lea(rscratch1, Address(base, disp));
3279 __ lea(rscratch1, Address(rscratch1, index_reg, Address::lsl(scale)));
3280 __ ldaxr(dst_reg, rscratch1);
3281 }
3282 }
3283 %}
3284
3285 enc_class aarch64_enc_stlxr(iRegLNoSp src, memory8 mem) %{
3286 C2_MacroAssembler _masm(&cbuf);
3287 Register src_reg = as_Register($src$$reg);
3288 Register base = as_Register($mem$$base);
3289 int index = $mem$$index;
3290 int scale = $mem$$scale;
3291 int disp = $mem$$disp;
3292 if (index == -1) {
3293 if (disp != 0) {
3294 __ lea(rscratch2, Address(base, disp));
3295 __ stlxr(rscratch1, src_reg, rscratch2);
3296 } else {
3297 // TODO
3298 // should we ever get anything other than this case?
3299 __ stlxr(rscratch1, src_reg, base);
3300 }
3301 } else {
3302 Register index_reg = as_Register(index);
3303 if (disp == 0) {
3304 __ lea(rscratch2, Address(base, index_reg, Address::lsl(scale)));
3305 __ stlxr(rscratch1, src_reg, rscratch2);
3306 } else {
3307 __ lea(rscratch2, Address(base, disp));
3308 __ lea(rscratch2, Address(rscratch2, index_reg, Address::lsl(scale)));
3309 __ stlxr(rscratch1, src_reg, rscratch2);
3310 }
3311 }
3312 __ cmpw(rscratch1, zr);
3313 %}
3314
3315 enc_class aarch64_enc_cmpxchg(memory mem, iRegLNoSp oldval, iRegLNoSp newval) %{
3316 C2_MacroAssembler _masm(&cbuf);
3317 guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
3318 __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
3319 Assembler::xword, /*acquire*/ false, /*release*/ true,
3320 /*weak*/ false, noreg);
3321 %}
3322
3323 enc_class aarch64_enc_cmpxchgw(memory mem, iRegINoSp oldval, iRegINoSp newval) %{
3324 C2_MacroAssembler _masm(&cbuf);
3325 guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
3326 __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
3327 Assembler::word, /*acquire*/ false, /*release*/ true,
3328 /*weak*/ false, noreg);
3329 %}
3330
3331 enc_class aarch64_enc_cmpxchgs(memory mem, iRegINoSp oldval, iRegINoSp newval) %{
3332 C2_MacroAssembler _masm(&cbuf);
3333 guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
3334 __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
3335 Assembler::halfword, /*acquire*/ false, /*release*/ true,
3336 /*weak*/ false, noreg);
3337 %}
3338
3339 enc_class aarch64_enc_cmpxchgb(memory mem, iRegINoSp oldval, iRegINoSp newval) %{
3340 C2_MacroAssembler _masm(&cbuf);
3341 guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
3342 __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
3343 Assembler::byte, /*acquire*/ false, /*release*/ true,
3344 /*weak*/ false, noreg);
3345 %}
3346
3347
3348 // The only difference between aarch64_enc_cmpxchg and
3349 // aarch64_enc_cmpxchg_acq is that we use load-acquire in the
3350 // CompareAndSwap sequence to serve as a barrier on acquiring a
3351 // lock.
3352 enc_class aarch64_enc_cmpxchg_acq(memory mem, iRegLNoSp oldval, iRegLNoSp newval) %{
3353 C2_MacroAssembler _masm(&cbuf);
3354 guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
3355 __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
3356 Assembler::xword, /*acquire*/ true, /*release*/ true,
3357 /*weak*/ false, noreg);
3358 %}
3359
3360 enc_class aarch64_enc_cmpxchgw_acq(memory mem, iRegINoSp oldval, iRegINoSp newval) %{
3361 C2_MacroAssembler _masm(&cbuf);
3362 guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
3363 __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
3364 Assembler::word, /*acquire*/ true, /*release*/ true,
3365 /*weak*/ false, noreg);
3366 %}
3367
3368 enc_class aarch64_enc_cmpxchgs_acq(memory mem, iRegINoSp oldval, iRegINoSp newval) %{
3369 C2_MacroAssembler _masm(&cbuf);
3370 guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
3371 __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
3372 Assembler::halfword, /*acquire*/ true, /*release*/ true,
3373 /*weak*/ false, noreg);
3374 %}
3375
3376 enc_class aarch64_enc_cmpxchgb_acq(memory mem, iRegINoSp oldval, iRegINoSp newval) %{
3377 C2_MacroAssembler _masm(&cbuf);
3378 guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
3379 __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
3380 Assembler::byte, /*acquire*/ true, /*release*/ true,
3381 /*weak*/ false, noreg);
3382 %}
3383
3384 // auxiliary used for CompareAndSwapX to set result register
3385 enc_class aarch64_enc_cset_eq(iRegINoSp res) %{
3386 C2_MacroAssembler _masm(&cbuf);
3387 Register res_reg = as_Register($res$$reg);
3388 __ cset(res_reg, Assembler::EQ);
3389 %}
3390
3391 // prefetch encodings
3392
3393 enc_class aarch64_enc_prefetchw(memory mem) %{
3394 C2_MacroAssembler _masm(&cbuf);
3395 Register base = as_Register($mem$$base);
3396 int index = $mem$$index;
3397 int scale = $mem$$scale;
3398 int disp = $mem$$disp;
3399 if (index == -1) {
3400 __ prfm(Address(base, disp), PSTL1KEEP);
3401 } else {
3402 Register index_reg = as_Register(index);
3403 if (disp == 0) {
3404 __ prfm(Address(base, index_reg, Address::lsl(scale)), PSTL1KEEP);
3405 } else {
3406 __ lea(rscratch1, Address(base, disp));
3407 __ prfm(Address(rscratch1, index_reg, Address::lsl(scale)), PSTL1KEEP);
3408 }
3409 }
3410 %}
3411
3412 /// mov envcodings
3413
3414 enc_class aarch64_enc_movw_imm(iRegI dst, immI src) %{
3415 C2_MacroAssembler _masm(&cbuf);
3416 uint32_t con = (uint32_t)$src$$constant;
3417 Register dst_reg = as_Register($dst$$reg);
3418 if (con == 0) {
3419 __ movw(dst_reg, zr);
3420 } else {
3421 __ movw(dst_reg, con);
3422 }
3423 %}
3424
3425 enc_class aarch64_enc_mov_imm(iRegL dst, immL src) %{
3426 C2_MacroAssembler _masm(&cbuf);
3427 Register dst_reg = as_Register($dst$$reg);
3428 uint64_t con = (uint64_t)$src$$constant;
3429 if (con == 0) {
3430 __ mov(dst_reg, zr);
3431 } else {
3432 __ mov(dst_reg, con);
3433 }
3434 %}
3435
3436 enc_class aarch64_enc_mov_p(iRegP dst, immP src) %{
3437 C2_MacroAssembler _masm(&cbuf);
3438 Register dst_reg = as_Register($dst$$reg);
3439 address con = (address)$src$$constant;
3440 if (con == NULL || con == (address)1) {
3441 ShouldNotReachHere();
3442 } else {
3443 relocInfo::relocType rtype = $src->constant_reloc();
3444 if (rtype == relocInfo::oop_type) {
3445 __ movoop(dst_reg, (jobject)con, /*immediate*/true);
3446 } else if (rtype == relocInfo::metadata_type) {
3447 __ mov_metadata(dst_reg, (Metadata*)con);
3448 } else {
3449 assert(rtype == relocInfo::none, "unexpected reloc type");
3450 if (con < (address)(uintptr_t)os::vm_page_size()) {
3451 __ mov(dst_reg, con);
3452 } else {
3453 uint64_t offset;
3454 __ adrp(dst_reg, con, offset);
3455 __ add(dst_reg, dst_reg, offset);
3456 }
3457 }
3458 }
3459 %}
3460
3461 enc_class aarch64_enc_mov_p0(iRegP dst, immP0 src) %{
3462 C2_MacroAssembler _masm(&cbuf);
3463 Register dst_reg = as_Register($dst$$reg);
3464 __ mov(dst_reg, zr);
3465 %}
3466
3467 enc_class aarch64_enc_mov_p1(iRegP dst, immP_1 src) %{
3468 C2_MacroAssembler _masm(&cbuf);
3469 Register dst_reg = as_Register($dst$$reg);
3470 __ mov(dst_reg, (uint64_t)1);
3471 %}
3472
3473 enc_class aarch64_enc_mov_byte_map_base(iRegP dst, immByteMapBase src) %{
3474 C2_MacroAssembler _masm(&cbuf);
3475 __ load_byte_map_base($dst$$Register);
3476 %}
3477
3478 enc_class aarch64_enc_mov_n(iRegN dst, immN src) %{
3479 C2_MacroAssembler _masm(&cbuf);
3480 Register dst_reg = as_Register($dst$$reg);
3481 address con = (address)$src$$constant;
3482 if (con == NULL) {
3483 ShouldNotReachHere();
3484 } else {
3485 relocInfo::relocType rtype = $src->constant_reloc();
3486 assert(rtype == relocInfo::oop_type, "unexpected reloc type");
3487 __ set_narrow_oop(dst_reg, (jobject)con);
3488 }
3489 %}
3490
3491 enc_class aarch64_enc_mov_n0(iRegN dst, immN0 src) %{
3492 C2_MacroAssembler _masm(&cbuf);
3493 Register dst_reg = as_Register($dst$$reg);
3494 __ mov(dst_reg, zr);
3495 %}
3496
3497 enc_class aarch64_enc_mov_nk(iRegN dst, immNKlass src) %{
3498 C2_MacroAssembler _masm(&cbuf);
3499 Register dst_reg = as_Register($dst$$reg);
3500 address con = (address)$src$$constant;
3501 if (con == NULL) {
3502 ShouldNotReachHere();
3503 } else {
3504 relocInfo::relocType rtype = $src->constant_reloc();
3505 assert(rtype == relocInfo::metadata_type, "unexpected reloc type");
3506 __ set_narrow_klass(dst_reg, (Klass *)con);
3507 }
3508 %}
3509
3510 // arithmetic encodings
3511
3512 enc_class aarch64_enc_addsubw_imm(iRegI dst, iRegI src1, immIAddSub src2) %{
3513 C2_MacroAssembler _masm(&cbuf);
3514 Register dst_reg = as_Register($dst$$reg);
3515 Register src_reg = as_Register($src1$$reg);
3516 int32_t con = (int32_t)$src2$$constant;
3517 // add has primary == 0, subtract has primary == 1
3518 if ($primary) { con = -con; }
3519 if (con < 0) {
3520 __ subw(dst_reg, src_reg, -con);
3521 } else {
3522 __ addw(dst_reg, src_reg, con);
3523 }
3524 %}
3525
3526 enc_class aarch64_enc_addsub_imm(iRegL dst, iRegL src1, immLAddSub src2) %{
3527 C2_MacroAssembler _masm(&cbuf);
3528 Register dst_reg = as_Register($dst$$reg);
3529 Register src_reg = as_Register($src1$$reg);
3530 int32_t con = (int32_t)$src2$$constant;
3531 // add has primary == 0, subtract has primary == 1
3532 if ($primary) { con = -con; }
3533 if (con < 0) {
3534 __ sub(dst_reg, src_reg, -con);
3535 } else {
3536 __ add(dst_reg, src_reg, con);
3537 }
3538 %}
3539
3540 enc_class aarch64_enc_divw(iRegI dst, iRegI src1, iRegI src2) %{
3541 C2_MacroAssembler _masm(&cbuf);
3542 Register dst_reg = as_Register($dst$$reg);
3543 Register src1_reg = as_Register($src1$$reg);
3544 Register src2_reg = as_Register($src2$$reg);
3545 __ corrected_idivl(dst_reg, src1_reg, src2_reg, false, rscratch1);
3546 %}
3547
3548 enc_class aarch64_enc_div(iRegI dst, iRegI src1, iRegI src2) %{
3549 C2_MacroAssembler _masm(&cbuf);
3550 Register dst_reg = as_Register($dst$$reg);
3551 Register src1_reg = as_Register($src1$$reg);
3552 Register src2_reg = as_Register($src2$$reg);
3553 __ corrected_idivq(dst_reg, src1_reg, src2_reg, false, rscratch1);
3554 %}
3555
3556 enc_class aarch64_enc_modw(iRegI dst, iRegI src1, iRegI src2) %{
3557 C2_MacroAssembler _masm(&cbuf);
3558 Register dst_reg = as_Register($dst$$reg);
3559 Register src1_reg = as_Register($src1$$reg);
3560 Register src2_reg = as_Register($src2$$reg);
3561 __ corrected_idivl(dst_reg, src1_reg, src2_reg, true, rscratch1);
3562 %}
3563
3564 enc_class aarch64_enc_mod(iRegI dst, iRegI src1, iRegI src2) %{
3565 C2_MacroAssembler _masm(&cbuf);
3566 Register dst_reg = as_Register($dst$$reg);
3567 Register src1_reg = as_Register($src1$$reg);
3568 Register src2_reg = as_Register($src2$$reg);
3569 __ corrected_idivq(dst_reg, src1_reg, src2_reg, true, rscratch1);
3570 %}
3571
3572 // compare instruction encodings
3573
3574 enc_class aarch64_enc_cmpw(iRegI src1, iRegI src2) %{
3575 C2_MacroAssembler _masm(&cbuf);
3576 Register reg1 = as_Register($src1$$reg);
3577 Register reg2 = as_Register($src2$$reg);
3578 __ cmpw(reg1, reg2);
3579 %}
3580
3581 enc_class aarch64_enc_cmpw_imm_addsub(iRegI src1, immIAddSub src2) %{
3582 C2_MacroAssembler _masm(&cbuf);
3583 Register reg = as_Register($src1$$reg);
3584 int32_t val = $src2$$constant;
3585 if (val >= 0) {
3586 __ subsw(zr, reg, val);
3587 } else {
3588 __ addsw(zr, reg, -val);
3589 }
3590 %}
3591
3592 enc_class aarch64_enc_cmpw_imm(iRegI src1, immI src2) %{
3593 C2_MacroAssembler _masm(&cbuf);
3594 Register reg1 = as_Register($src1$$reg);
3595 uint32_t val = (uint32_t)$src2$$constant;
3596 __ movw(rscratch1, val);
3597 __ cmpw(reg1, rscratch1);
3598 %}
3599
3600 enc_class aarch64_enc_cmp(iRegL src1, iRegL src2) %{
3601 C2_MacroAssembler _masm(&cbuf);
3602 Register reg1 = as_Register($src1$$reg);
3603 Register reg2 = as_Register($src2$$reg);
3604 __ cmp(reg1, reg2);
3605 %}
3606
3607 enc_class aarch64_enc_cmp_imm_addsub(iRegL src1, immL12 src2) %{
3608 C2_MacroAssembler _masm(&cbuf);
3609 Register reg = as_Register($src1$$reg);
3610 int64_t val = $src2$$constant;
3611 if (val >= 0) {
3612 __ subs(zr, reg, val);
3613 } else if (val != -val) {
3614 __ adds(zr, reg, -val);
3615 } else {
3616 // aargh, Long.MIN_VALUE is a special case
3617 __ orr(rscratch1, zr, (uint64_t)val);
3618 __ subs(zr, reg, rscratch1);
3619 }
3620 %}
3621
3622 enc_class aarch64_enc_cmp_imm(iRegL src1, immL src2) %{
3623 C2_MacroAssembler _masm(&cbuf);
3624 Register reg1 = as_Register($src1$$reg);
3625 uint64_t val = (uint64_t)$src2$$constant;
3626 __ mov(rscratch1, val);
3627 __ cmp(reg1, rscratch1);
3628 %}
3629
3630 enc_class aarch64_enc_cmpp(iRegP src1, iRegP src2) %{
3631 C2_MacroAssembler _masm(&cbuf);
3632 Register reg1 = as_Register($src1$$reg);
3633 Register reg2 = as_Register($src2$$reg);
3634 __ cmp(reg1, reg2);
3635 %}
3636
3637 enc_class aarch64_enc_cmpn(iRegN src1, iRegN src2) %{
3638 C2_MacroAssembler _masm(&cbuf);
3639 Register reg1 = as_Register($src1$$reg);
3640 Register reg2 = as_Register($src2$$reg);
3641 __ cmpw(reg1, reg2);
3642 %}
3643
3644 enc_class aarch64_enc_testp(iRegP src) %{
3645 C2_MacroAssembler _masm(&cbuf);
3646 Register reg = as_Register($src$$reg);
3647 __ cmp(reg, zr);
3648 %}
3649
3650 enc_class aarch64_enc_testn(iRegN src) %{
3651 C2_MacroAssembler _masm(&cbuf);
3652 Register reg = as_Register($src$$reg);
3653 __ cmpw(reg, zr);
3654 %}
3655
3656 enc_class aarch64_enc_b(label lbl) %{
3657 C2_MacroAssembler _masm(&cbuf);
3658 Label *L = $lbl$$label;
3659 __ b(*L);
3660 %}
3661
3662 enc_class aarch64_enc_br_con(cmpOp cmp, label lbl) %{
3663 C2_MacroAssembler _masm(&cbuf);
3664 Label *L = $lbl$$label;
3665 __ br ((Assembler::Condition)$cmp$$cmpcode, *L);
3666 %}
3667
3668 enc_class aarch64_enc_br_conU(cmpOpU cmp, label lbl) %{
3669 C2_MacroAssembler _masm(&cbuf);
3670 Label *L = $lbl$$label;
3671 __ br ((Assembler::Condition)$cmp$$cmpcode, *L);
3672 %}
3673
3674 enc_class aarch64_enc_partial_subtype_check(iRegP sub, iRegP super, iRegP temp, iRegP result)
3675 %{
3676 Register sub_reg = as_Register($sub$$reg);
3677 Register super_reg = as_Register($super$$reg);
3678 Register temp_reg = as_Register($temp$$reg);
3679 Register result_reg = as_Register($result$$reg);
3680
3681 Label miss;
3682 C2_MacroAssembler _masm(&cbuf);
3683 __ check_klass_subtype_slow_path(sub_reg, super_reg, temp_reg, result_reg,
3684 NULL, &miss,
3685 /*set_cond_codes:*/ true);
3686 if ($primary) {
3687 __ mov(result_reg, zr);
3688 }
3689 __ bind(miss);
3690 %}
3691
3692 enc_class aarch64_enc_java_static_call(method meth) %{
3693 C2_MacroAssembler _masm(&cbuf);
3694
3695 address addr = (address)$meth$$method;
3696 address call;
3697 if (!_method) {
3698 // A call to a runtime wrapper, e.g. new, new_typeArray_Java, uncommon_trap.
3699 call = __ trampoline_call(Address(addr, relocInfo::runtime_call_type), &cbuf);
3700 if (call == NULL) {
3701 ciEnv::current()->record_failure("CodeCache is full");
3702 return;
3703 }
3704 } else {
3705 int method_index = resolved_method_index(cbuf);
3706 RelocationHolder rspec = _optimized_virtual ? opt_virtual_call_Relocation::spec(method_index)
3707 : static_call_Relocation::spec(method_index);
3708 call = __ trampoline_call(Address(addr, rspec), &cbuf);
3709 if (call == NULL) {
3710 ciEnv::current()->record_failure("CodeCache is full");
3711 return;
3712 }
3713 // Emit stub for static call
3714 address stub = CompiledStaticCall::emit_to_interp_stub(cbuf);
3715 if (stub == NULL) {
3716 ciEnv::current()->record_failure("CodeCache is full");
3717 return;
3718 }
3719 }
3720
3721 // Only non uncommon_trap calls need to reinitialize ptrue.
3722 if (Compile::current()->max_vector_size() > 0 && uncommon_trap_request() == 0) {
3723 __ reinitialize_ptrue();
3724 }
3725 %}
3726
3727 enc_class aarch64_enc_java_dynamic_call(method meth) %{
3728 C2_MacroAssembler _masm(&cbuf);
3729 int method_index = resolved_method_index(cbuf);
3730 address call = __ ic_call((address)$meth$$method, method_index);
3731 if (call == NULL) {
3732 ciEnv::current()->record_failure("CodeCache is full");
3733 return;
3734 } else if (Compile::current()->max_vector_size() > 0) {
3735 __ reinitialize_ptrue();
3736 }
3737 %}
3738
3739 enc_class aarch64_enc_call_epilog() %{
3740 C2_MacroAssembler _masm(&cbuf);
3741 if (VerifyStackAtCalls) {
3742 // Check that stack depth is unchanged: find majik cookie on stack
3743 __ call_Unimplemented();
3744 }
3745 %}
3746
3747 enc_class aarch64_enc_java_to_runtime(method meth) %{
3748 C2_MacroAssembler _masm(&cbuf);
3749
3750 // some calls to generated routines (arraycopy code) are scheduled
3751 // by C2 as runtime calls. if so we can call them using a br (they
3752 // will be in a reachable segment) otherwise we have to use a blr
3753 // which loads the absolute address into a register.
3754 address entry = (address)$meth$$method;
3755 CodeBlob *cb = CodeCache::find_blob(entry);
3756 if (cb) {
3757 address call = __ trampoline_call(Address(entry, relocInfo::runtime_call_type));
3758 if (call == NULL) {
3759 ciEnv::current()->record_failure("CodeCache is full");
3760 return;
3761 }
3762 } else {
3763 Label retaddr;
3764 __ adr(rscratch2, retaddr);
3765 __ lea(rscratch1, RuntimeAddress(entry));
3766 // Leave a breadcrumb for JavaFrameAnchor::capture_last_Java_pc()
3767 __ stp(zr, rscratch2, Address(__ pre(sp, -2 * wordSize)));
3768 __ blr(rscratch1);
3769 __ bind(retaddr);
3770 __ add(sp, sp, 2 * wordSize);
3771 }
3772 if (Compile::current()->max_vector_size() > 0) {
3773 __ reinitialize_ptrue();
3774 }
3775 %}
3776
3777 enc_class aarch64_enc_rethrow() %{
3778 C2_MacroAssembler _masm(&cbuf);
3779 __ far_jump(RuntimeAddress(OptoRuntime::rethrow_stub()));
3780 %}
3781
3782 enc_class aarch64_enc_ret() %{
3783 C2_MacroAssembler _masm(&cbuf);
3784 #ifdef ASSERT
3785 if (Compile::current()->max_vector_size() > 0) {
3786 __ verify_ptrue();
3787 }
3788 #endif
3789 __ ret(lr);
3790 %}
3791
3792 enc_class aarch64_enc_tail_call(iRegP jump_target) %{
3793 C2_MacroAssembler _masm(&cbuf);
3794 Register target_reg = as_Register($jump_target$$reg);
3795 __ br(target_reg);
3796 %}
3797
3798 enc_class aarch64_enc_tail_jmp(iRegP jump_target) %{
3799 C2_MacroAssembler _masm(&cbuf);
3800 Register target_reg = as_Register($jump_target$$reg);
3801 // exception oop should be in r0
3802 // ret addr has been popped into lr
3803 // callee expects it in r3
3804 __ mov(r3, lr);
3805 __ br(target_reg);
3806 %}
3807
3808 enc_class aarch64_enc_fast_lock(iRegP object, iRegP box, iRegP tmp, iRegP tmp2) %{
3809 C2_MacroAssembler _masm(&cbuf);
3810 Register oop = as_Register($object$$reg);
3811 Register box = as_Register($box$$reg);
3812 Register disp_hdr = as_Register($tmp$$reg);
3813 Register tmp = as_Register($tmp2$$reg);
3814 Label cont;
3815 Label object_has_monitor;
3816 Label cas_failed;
3817
3818 assert_different_registers(oop, box, tmp, disp_hdr);
3819
3820 // Load markWord from object into displaced_header.
3821 __ ldr(disp_hdr, Address(oop, oopDesc::mark_offset_in_bytes()));
3822
3823 if (DiagnoseSyncOnValueBasedClasses != 0) {
3824 __ load_klass(tmp, oop);
3825 __ ldrw(tmp, Address(tmp, Klass::access_flags_offset()));
3826 __ tstw(tmp, JVM_ACC_IS_VALUE_BASED_CLASS);
3827 __ br(Assembler::NE, cont);
3828 }
3829
3830 // Check for existing monitor
3831 __ tbnz(disp_hdr, exact_log2(markWord::monitor_value), object_has_monitor);
3832
3833 // Set tmp to be (markWord of object | UNLOCK_VALUE).
3834 __ orr(tmp, disp_hdr, markWord::unlocked_value);
3835
3836 // Initialize the box. (Must happen before we update the object mark!)
3837 __ str(tmp, Address(box, BasicLock::displaced_header_offset_in_bytes()));
3838
3839 // Compare object markWord with an unlocked value (tmp) and if
3840 // equal exchange the stack address of our box with object markWord.
3841 // On failure disp_hdr contains the possibly locked markWord.
3842 __ cmpxchg(oop, tmp, box, Assembler::xword, /*acquire*/ true,
3843 /*release*/ true, /*weak*/ false, disp_hdr);
3844 __ br(Assembler::EQ, cont);
3845
3846 assert(oopDesc::mark_offset_in_bytes() == 0, "offset of _mark is not 0");
3847
3848 // If the compare-and-exchange succeeded, then we found an unlocked
3849 // object, will have now locked it will continue at label cont
3850
3851 __ bind(cas_failed);
3852 // We did not see an unlocked object so try the fast recursive case.
3853
3854 // Check if the owner is self by comparing the value in the
3855 // markWord of object (disp_hdr) with the stack pointer.
3856 __ mov(rscratch1, sp);
3857 __ sub(disp_hdr, disp_hdr, rscratch1);
3858 __ mov(tmp, (address) (~(os::vm_page_size()-1) | markWord::lock_mask_in_place));
3859 // If condition is true we are cont and hence we can store 0 as the
3860 // displaced header in the box, which indicates that it is a recursive lock.
3861 __ ands(tmp/*==0?*/, disp_hdr, tmp); // Sets flags for result
3862 __ str(tmp/*==0, perhaps*/, Address(box, BasicLock::displaced_header_offset_in_bytes()));
3863
3864 __ b(cont);
3865
3866 // Handle existing monitor.
3867 __ bind(object_has_monitor);
3868
3869 // The object's monitor m is unlocked iff m->owner == NULL,
3870 // otherwise m->owner may contain a thread or a stack address.
3871 //
3872 // Try to CAS m->owner from NULL to current thread.
3873 __ add(tmp, disp_hdr, (ObjectMonitor::owner_offset_in_bytes()-markWord::monitor_value));
3874 __ cmpxchg(tmp, zr, rthread, Assembler::xword, /*acquire*/ true,
3875 /*release*/ true, /*weak*/ false, noreg); // Sets flags for result
3876
3877 // Store a non-null value into the box to avoid looking like a re-entrant
3878 // lock. The fast-path monitor unlock code checks for
3879 // markWord::monitor_value so use markWord::unused_mark which has the
3880 // relevant bit set, and also matches ObjectSynchronizer::enter.
3881 __ mov(tmp, (address)markWord::unused_mark().value());
3882 __ str(tmp, Address(box, BasicLock::displaced_header_offset_in_bytes()));
3883
3884 __ bind(cont);
3885 // flag == EQ indicates success
3886 // flag == NE indicates failure
3887 %}
3888
3889 enc_class aarch64_enc_fast_unlock(iRegP object, iRegP box, iRegP tmp, iRegP tmp2) %{
3890 C2_MacroAssembler _masm(&cbuf);
3891 Register oop = as_Register($object$$reg);
3892 Register box = as_Register($box$$reg);
3893 Register disp_hdr = as_Register($tmp$$reg);
3894 Register tmp = as_Register($tmp2$$reg);
3895 Label cont;
3896 Label object_has_monitor;
3897
3898 assert_different_registers(oop, box, tmp, disp_hdr);
3899
3900 // Find the lock address and load the displaced header from the stack.
3901 __ ldr(disp_hdr, Address(box, BasicLock::displaced_header_offset_in_bytes()));
3902
3903 // If the displaced header is 0, we have a recursive unlock.
3904 __ cmp(disp_hdr, zr);
3905 __ br(Assembler::EQ, cont);
3906
3907 // Handle existing monitor.
3908 __ ldr(tmp, Address(oop, oopDesc::mark_offset_in_bytes()));
3909 __ tbnz(disp_hdr, exact_log2(markWord::monitor_value), object_has_monitor);
3910
3911 // Check if it is still a light weight lock, this is is true if we
3912 // see the stack address of the basicLock in the markWord of the
3913 // object.
3914
3915 __ cmpxchg(oop, box, disp_hdr, Assembler::xword, /*acquire*/ false,
3916 /*release*/ true, /*weak*/ false, tmp);
3917 __ b(cont);
3918
3919 assert(oopDesc::mark_offset_in_bytes() == 0, "offset of _mark is not 0");
3920
3921 // Handle existing monitor.
3922 __ bind(object_has_monitor);
3923 STATIC_ASSERT(markWord::monitor_value <= INT_MAX);
3924 __ add(tmp, tmp, -(int)markWord::monitor_value); // monitor
3925 __ ldr(rscratch1, Address(tmp, ObjectMonitor::owner_offset_in_bytes()));
3926 __ ldr(disp_hdr, Address(tmp, ObjectMonitor::recursions_offset_in_bytes()));
3927 __ eor(rscratch1, rscratch1, rthread); // Will be 0 if we are the owner.
3928 __ orr(rscratch1, rscratch1, disp_hdr); // Will be 0 if there are 0 recursions
3929 __ cmp(rscratch1, zr); // Sets flags for result
3930 __ br(Assembler::NE, cont);
3931
3932 __ ldr(rscratch1, Address(tmp, ObjectMonitor::EntryList_offset_in_bytes()));
3933 __ ldr(disp_hdr, Address(tmp, ObjectMonitor::cxq_offset_in_bytes()));
3934 __ orr(rscratch1, rscratch1, disp_hdr); // Will be 0 if both are 0.
3935 __ cmp(rscratch1, zr); // Sets flags for result
3936 __ cbnz(rscratch1, cont);
3937 // need a release store here
3938 __ lea(tmp, Address(tmp, ObjectMonitor::owner_offset_in_bytes()));
3939 __ stlr(zr, tmp); // set unowned
3940
3941 __ bind(cont);
3942 // flag == EQ indicates success
3943 // flag == NE indicates failure
3944 %}
3945
3946 %}
3947
3948 //----------FRAME--------------------------------------------------------------
3949 // Definition of frame structure and management information.
3950 //
3951 // S T A C K L A Y O U T Allocators stack-slot number
3952 // | (to get allocators register number
3953 // G Owned by | | v add OptoReg::stack0())
3954 // r CALLER | |
3955 // o | +--------+ pad to even-align allocators stack-slot
3956 // w V | pad0 | numbers; owned by CALLER
3957 // t -----------+--------+----> Matcher::_in_arg_limit, unaligned
3958 // h ^ | in | 5
3959 // | | args | 4 Holes in incoming args owned by SELF
3960 // | | | | 3
3961 // | | +--------+
3962 // V | | old out| Empty on Intel, window on Sparc
3963 // | old |preserve| Must be even aligned.
3964 // | SP-+--------+----> Matcher::_old_SP, even aligned
3965 // | | in | 3 area for Intel ret address
3966 // Owned by |preserve| Empty on Sparc.
3967 // SELF +--------+
3968 // | | pad2 | 2 pad to align old SP
3969 // | +--------+ 1
3970 // | | locks | 0
3971 // | +--------+----> OptoReg::stack0(), even aligned
3972 // | | pad1 | 11 pad to align new SP
3973 // | +--------+
3974 // | | | 10
3975 // | | spills | 9 spills
3976 // V | | 8 (pad0 slot for callee)
3977 // -----------+--------+----> Matcher::_out_arg_limit, unaligned
3978 // ^ | out | 7
3979 // | | args | 6 Holes in outgoing args owned by CALLEE
3980 // Owned by +--------+
3981 // CALLEE | new out| 6 Empty on Intel, window on Sparc
3982 // | new |preserve| Must be even-aligned.
3983 // | SP-+--------+----> Matcher::_new_SP, even aligned
3984 // | | |
3985 //
3986 // Note 1: Only region 8-11 is determined by the allocator. Region 0-5 is
3987 // known from SELF's arguments and the Java calling convention.
3988 // Region 6-7 is determined per call site.
3989 // Note 2: If the calling convention leaves holes in the incoming argument
3990 // area, those holes are owned by SELF. Holes in the outgoing area
3991 // are owned by the CALLEE. Holes should not be nessecary in the
3992 // incoming area, as the Java calling convention is completely under
3993 // the control of the AD file. Doubles can be sorted and packed to
3994 // avoid holes. Holes in the outgoing arguments may be nessecary for
3995 // varargs C calling conventions.
3996 // Note 3: Region 0-3 is even aligned, with pad2 as needed. Region 3-5 is
3997 // even aligned with pad0 as needed.
3998 // Region 6 is even aligned. Region 6-7 is NOT even aligned;
3999 // (the latter is true on Intel but is it false on AArch64?)
4000 // region 6-11 is even aligned; it may be padded out more so that
4001 // the region from SP to FP meets the minimum stack alignment.
4002 // Note 4: For I2C adapters, the incoming FP may not meet the minimum stack
4003 // alignment. Region 11, pad1, may be dynamically extended so that
4004 // SP meets the minimum alignment.
4005
4006 frame %{
4007 // These three registers define part of the calling convention
4008 // between compiled code and the interpreter.
4009
4010 // Inline Cache Register or Method for I2C.
4011 inline_cache_reg(R12);
4012
4013 // Number of stack slots consumed by locking an object
4014 sync_stack_slots(2);
4015
4016 // Compiled code's Frame Pointer
4017 frame_pointer(R31);
4018
4019 // Interpreter stores its frame pointer in a register which is
4020 // stored to the stack by I2CAdaptors.
4021 // I2CAdaptors convert from interpreted java to compiled java.
4022 interpreter_frame_pointer(R29);
4023
4024 // Stack alignment requirement
4025 stack_alignment(StackAlignmentInBytes); // Alignment size in bytes (128-bit -> 16 bytes)
4026
4027 // Number of outgoing stack slots killed above the out_preserve_stack_slots
4028 // for calls to C. Supports the var-args backing area for register parms.
4029 varargs_C_out_slots_killed(frame::arg_reg_save_area_bytes/BytesPerInt);
4030
4031 // The after-PROLOG location of the return address. Location of
4032 // return address specifies a type (REG or STACK) and a number
4033 // representing the register number (i.e. - use a register name) or
4034 // stack slot.
4035 // Ret Addr is on stack in slot 0 if no locks or verification or alignment.
4036 // Otherwise, it is above the locks and verification slot and alignment word
4037 // TODO this may well be correct but need to check why that - 2 is there
4038 // ppc port uses 0 but we definitely need to allow for fixed_slots
4039 // which folds in the space used for monitors
4040 return_addr(STACK - 2 +
4041 align_up((Compile::current()->in_preserve_stack_slots() +
4042 Compile::current()->fixed_slots()),
4043 stack_alignment_in_slots()));
4044
4045 // Location of compiled Java return values. Same as C for now.
4046 return_value
4047 %{
4048 // TODO do we allow ideal_reg == Op_RegN???
4049 assert(ideal_reg >= Op_RegI && ideal_reg <= Op_RegL,
4050 "only return normal values");
4051
4052 static const int lo[Op_RegL + 1] = { // enum name
4053 0, // Op_Node
4054 0, // Op_Set
4055 R0_num, // Op_RegN
4056 R0_num, // Op_RegI
4057 R0_num, // Op_RegP
4058 V0_num, // Op_RegF
4059 V0_num, // Op_RegD
4060 R0_num // Op_RegL
4061 };
4062
4063 static const int hi[Op_RegL + 1] = { // enum name
4064 0, // Op_Node
4065 0, // Op_Set
4066 OptoReg::Bad, // Op_RegN
4067 OptoReg::Bad, // Op_RegI
4068 R0_H_num, // Op_RegP
4069 OptoReg::Bad, // Op_RegF
4070 V0_H_num, // Op_RegD
4071 R0_H_num // Op_RegL
4072 };
4073
4074 return OptoRegPair(hi[ideal_reg], lo[ideal_reg]);
4075 %}
4076 %}
4077
4078 //----------ATTRIBUTES---------------------------------------------------------
4079 //----------Operand Attributes-------------------------------------------------
4080 op_attrib op_cost(1); // Required cost attribute
4081
4082 //----------Instruction Attributes---------------------------------------------
4083 ins_attrib ins_cost(INSN_COST); // Required cost attribute
4084 ins_attrib ins_size(32); // Required size attribute (in bits)
4085 ins_attrib ins_short_branch(0); // Required flag: is this instruction
4086 // a non-matching short branch variant
4087 // of some long branch?
4088 ins_attrib ins_alignment(4); // Required alignment attribute (must
4089 // be a power of 2) specifies the
4090 // alignment that some part of the
4091 // instruction (not necessarily the
4092 // start) requires. If > 1, a
4093 // compute_padding() function must be
4094 // provided for the instruction
4095
4096 //----------OPERANDS-----------------------------------------------------------
4097 // Operand definitions must precede instruction definitions for correct parsing
4098 // in the ADLC because operands constitute user defined types which are used in
4099 // instruction definitions.
4100
4101 //----------Simple Operands----------------------------------------------------
4102
4103 // Integer operands 32 bit
4104 // 32 bit immediate
4105 operand immI()
4106 %{
4107 match(ConI);
4108
4109 op_cost(0);
4110 format %{ %}
4111 interface(CONST_INTER);
4112 %}
4113
4114 // 32 bit zero
4115 operand immI0()
4116 %{
4117 predicate(n->get_int() == 0);
4118 match(ConI);
4119
4120 op_cost(0);
4121 format %{ %}
4122 interface(CONST_INTER);
4123 %}
4124
4125 // 32 bit unit increment
4126 operand immI_1()
4127 %{
4128 predicate(n->get_int() == 1);
4129 match(ConI);
4130
4131 op_cost(0);
4132 format %{ %}
4133 interface(CONST_INTER);
4134 %}
4135
4136 // 32 bit unit decrement
4137 operand immI_M1()
4138 %{
4139 predicate(n->get_int() == -1);
4140 match(ConI);
4141
4142 op_cost(0);
4143 format %{ %}
4144 interface(CONST_INTER);
4145 %}
4146
4147 // Shift values for add/sub extension shift
4148 operand immIExt()
4149 %{
4150 predicate(0 <= n->get_int() && (n->get_int() <= 4));
4151 match(ConI);
4152
4153 op_cost(0);
4154 format %{ %}
4155 interface(CONST_INTER);
4156 %}
4157
4158 operand immI_gt_1()
4159 %{
4160 predicate(n->get_int() > 1);
4161 match(ConI);
4162
4163 op_cost(0);
4164 format %{ %}
4165 interface(CONST_INTER);
4166 %}
4167
4168 operand immI_le_4()
4169 %{
4170 predicate(n->get_int() <= 4);
4171 match(ConI);
4172
4173 op_cost(0);
4174 format %{ %}
4175 interface(CONST_INTER);
4176 %}
4177
4178 operand immI_31()
4179 %{
4180 predicate(n->get_int() == 31);
4181 match(ConI);
4182
4183 op_cost(0);
4184 format %{ %}
4185 interface(CONST_INTER);
4186 %}
4187
4188 operand immI_2()
4189 %{
4190 predicate(n->get_int() == 2);
4191 match(ConI);
4192
4193 op_cost(0);
4194 format %{ %}
4195 interface(CONST_INTER);
4196 %}
4197
4198 operand immI_4()
4199 %{
4200 predicate(n->get_int() == 4);
4201 match(ConI);
4202
4203 op_cost(0);
4204 format %{ %}
4205 interface(CONST_INTER);
4206 %}
4207
4208 operand immI_8()
4209 %{
4210 predicate(n->get_int() == 8);
4211 match(ConI);
4212
4213 op_cost(0);
4214 format %{ %}
4215 interface(CONST_INTER);
4216 %}
4217
4218 operand immI_16()
4219 %{
4220 predicate(n->get_int() == 16);
4221 match(ConI);
4222
4223 op_cost(0);
4224 format %{ %}
4225 interface(CONST_INTER);
4226 %}
4227
4228 operand immI_24()
4229 %{
4230 predicate(n->get_int() == 24);
4231 match(ConI);
4232
4233 op_cost(0);
4234 format %{ %}
4235 interface(CONST_INTER);
4236 %}
4237
4238 operand immI_32()
4239 %{
4240 predicate(n->get_int() == 32);
4241 match(ConI);
4242
4243 op_cost(0);
4244 format %{ %}
4245 interface(CONST_INTER);
4246 %}
4247
4248 operand immI_48()
4249 %{
4250 predicate(n->get_int() == 48);
4251 match(ConI);
4252
4253 op_cost(0);
4254 format %{ %}
4255 interface(CONST_INTER);
4256 %}
4257
4258 operand immI_56()
4259 %{
4260 predicate(n->get_int() == 56);
4261 match(ConI);
4262
4263 op_cost(0);
4264 format %{ %}
4265 interface(CONST_INTER);
4266 %}
4267
4268 operand immI_63()
4269 %{
4270 predicate(n->get_int() == 63);
4271 match(ConI);
4272
4273 op_cost(0);
4274 format %{ %}
4275 interface(CONST_INTER);
4276 %}
4277
4278 operand immI_64()
4279 %{
4280 predicate(n->get_int() == 64);
4281 match(ConI);
4282
4283 op_cost(0);
4284 format %{ %}
4285 interface(CONST_INTER);
4286 %}
4287
4288 operand immI_255()
4289 %{
4290 predicate(n->get_int() == 255);
4291 match(ConI);
4292
4293 op_cost(0);
4294 format %{ %}
4295 interface(CONST_INTER);
4296 %}
4297
4298 operand immI_65535()
4299 %{
4300 predicate(n->get_int() == 65535);
4301 match(ConI);
4302
4303 op_cost(0);
4304 format %{ %}
4305 interface(CONST_INTER);
4306 %}
4307
4308 operand immL_255()
4309 %{
4310 predicate(n->get_long() == 255L);
4311 match(ConL);
4312
4313 op_cost(0);
4314 format %{ %}
4315 interface(CONST_INTER);
4316 %}
4317
4318 operand immL_65535()
4319 %{
4320 predicate(n->get_long() == 65535L);
4321 match(ConL);
4322
4323 op_cost(0);
4324 format %{ %}
4325 interface(CONST_INTER);
4326 %}
4327
4328 operand immL_4294967295()
4329 %{
4330 predicate(n->get_long() == 4294967295L);
4331 match(ConL);
4332
4333 op_cost(0);
4334 format %{ %}
4335 interface(CONST_INTER);
4336 %}
4337
4338 operand immL_bitmask()
4339 %{
4340 predicate((n->get_long() != 0)
4341 && ((n->get_long() & 0xc000000000000000l) == 0)
4342 && is_power_of_2(n->get_long() + 1));
4343 match(ConL);
4344
4345 op_cost(0);
4346 format %{ %}
4347 interface(CONST_INTER);
4348 %}
4349
4350 operand immI_bitmask()
4351 %{
4352 predicate((n->get_int() != 0)
4353 && ((n->get_int() & 0xc0000000) == 0)
4354 && is_power_of_2(n->get_int() + 1));
4355 match(ConI);
4356
4357 op_cost(0);
4358 format %{ %}
4359 interface(CONST_INTER);
4360 %}
4361
4362 operand immL_positive_bitmaskI()
4363 %{
4364 predicate((n->get_long() != 0)
4365 && ((julong)n->get_long() < 0x80000000ULL)
4366 && is_power_of_2(n->get_long() + 1));
4367 match(ConL);
4368
4369 op_cost(0);
4370 format %{ %}
4371 interface(CONST_INTER);
4372 %}
4373
4374 // Scale values for scaled offset addressing modes (up to long but not quad)
4375 operand immIScale()
4376 %{
4377 predicate(0 <= n->get_int() && (n->get_int() <= 3));
4378 match(ConI);
4379
4380 op_cost(0);
4381 format %{ %}
4382 interface(CONST_INTER);
4383 %}
4384
4385 // 26 bit signed offset -- for pc-relative branches
4386 operand immI26()
4387 %{
4388 predicate(((-(1 << 25)) <= n->get_int()) && (n->get_int() < (1 << 25)));
4389 match(ConI);
4390
4391 op_cost(0);
4392 format %{ %}
4393 interface(CONST_INTER);
4394 %}
4395
4396 // 19 bit signed offset -- for pc-relative loads
4397 operand immI19()
4398 %{
4399 predicate(((-(1 << 18)) <= n->get_int()) && (n->get_int() < (1 << 18)));
4400 match(ConI);
4401
4402 op_cost(0);
4403 format %{ %}
4404 interface(CONST_INTER);
4405 %}
4406
4407 // 12 bit unsigned offset -- for base plus immediate loads
4408 operand immIU12()
4409 %{
4410 predicate((0 <= n->get_int()) && (n->get_int() < (1 << 12)));
4411 match(ConI);
4412
4413 op_cost(0);
4414 format %{ %}
4415 interface(CONST_INTER);
4416 %}
4417
4418 operand immLU12()
4419 %{
4420 predicate((0 <= n->get_long()) && (n->get_long() < (1 << 12)));
4421 match(ConL);
4422
4423 op_cost(0);
4424 format %{ %}
4425 interface(CONST_INTER);
4426 %}
4427
4428 // Offset for scaled or unscaled immediate loads and stores
4429 operand immIOffset()
4430 %{
4431 predicate(Address::offset_ok_for_immed(n->get_int(), 0));
4432 match(ConI);
4433
4434 op_cost(0);
4435 format %{ %}
4436 interface(CONST_INTER);
4437 %}
4438
4439 operand immIOffset1()
4440 %{
4441 predicate(Address::offset_ok_for_immed(n->get_int(), 0));
4442 match(ConI);
4443
4444 op_cost(0);
4445 format %{ %}
4446 interface(CONST_INTER);
4447 %}
4448
4449 operand immIOffset2()
4450 %{
4451 predicate(Address::offset_ok_for_immed(n->get_int(), 1));
4452 match(ConI);
4453
4454 op_cost(0);
4455 format %{ %}
4456 interface(CONST_INTER);
4457 %}
4458
4459 operand immIOffset4()
4460 %{
4461 predicate(Address::offset_ok_for_immed(n->get_int(), 2));
4462 match(ConI);
4463
4464 op_cost(0);
4465 format %{ %}
4466 interface(CONST_INTER);
4467 %}
4468
4469 operand immIOffset8()
4470 %{
4471 predicate(Address::offset_ok_for_immed(n->get_int(), 3));
4472 match(ConI);
4473
4474 op_cost(0);
4475 format %{ %}
4476 interface(CONST_INTER);
4477 %}
4478
4479 operand immIOffset16()
4480 %{
4481 predicate(Address::offset_ok_for_immed(n->get_int(), 4));
4482 match(ConI);
4483
4484 op_cost(0);
4485 format %{ %}
4486 interface(CONST_INTER);
4487 %}
4488
4489 operand immLoffset()
4490 %{
4491 predicate(Address::offset_ok_for_immed(n->get_long(), 0));
4492 match(ConL);
4493
4494 op_cost(0);
4495 format %{ %}
4496 interface(CONST_INTER);
4497 %}
4498
4499 operand immLoffset1()
4500 %{
4501 predicate(Address::offset_ok_for_immed(n->get_long(), 0));
4502 match(ConL);
4503
4504 op_cost(0);
4505 format %{ %}
4506 interface(CONST_INTER);
4507 %}
4508
4509 operand immLoffset2()
4510 %{
4511 predicate(Address::offset_ok_for_immed(n->get_long(), 1));
4512 match(ConL);
4513
4514 op_cost(0);
4515 format %{ %}
4516 interface(CONST_INTER);
4517 %}
4518
4519 operand immLoffset4()
4520 %{
4521 predicate(Address::offset_ok_for_immed(n->get_long(), 2));
4522 match(ConL);
4523
4524 op_cost(0);
4525 format %{ %}
4526 interface(CONST_INTER);
4527 %}
4528
4529 operand immLoffset8()
4530 %{
4531 predicate(Address::offset_ok_for_immed(n->get_long(), 3));
4532 match(ConL);
4533
4534 op_cost(0);
4535 format %{ %}
4536 interface(CONST_INTER);
4537 %}
4538
4539 operand immLoffset16()
4540 %{
4541 predicate(Address::offset_ok_for_immed(n->get_long(), 4));
4542 match(ConL);
4543
4544 op_cost(0);
4545 format %{ %}
4546 interface(CONST_INTER);
4547 %}
4548
4549 // 8 bit signed value.
4550 operand immI8()
4551 %{
4552 predicate(n->get_int() <= 127 && n->get_int() >= -128);
4553 match(ConI);
4554
4555 op_cost(0);
4556 format %{ %}
4557 interface(CONST_INTER);
4558 %}
4559
4560 // 8 bit signed value (simm8), or #simm8 LSL 8.
4561 operand immI8_shift8()
4562 %{
4563 predicate((n->get_int() <= 127 && n->get_int() >= -128) ||
4564 (n->get_int() <= 32512 && n->get_int() >= -32768 && (n->get_int() & 0xff) == 0));
4565 match(ConI);
4566
4567 op_cost(0);
4568 format %{ %}
4569 interface(CONST_INTER);
4570 %}
4571
4572 // 8 bit signed value (simm8), or #simm8 LSL 8.
4573 operand immL8_shift8()
4574 %{
4575 predicate((n->get_long() <= 127 && n->get_long() >= -128) ||
4576 (n->get_long() <= 32512 && n->get_long() >= -32768 && (n->get_long() & 0xff) == 0));
4577 match(ConL);
4578
4579 op_cost(0);
4580 format %{ %}
4581 interface(CONST_INTER);
4582 %}
4583
4584 // 32 bit integer valid for add sub immediate
4585 operand immIAddSub()
4586 %{
4587 predicate(Assembler::operand_valid_for_add_sub_immediate((int64_t)n->get_int()));
4588 match(ConI);
4589 op_cost(0);
4590 format %{ %}
4591 interface(CONST_INTER);
4592 %}
4593
4594 // 32 bit unsigned integer valid for logical immediate
4595 // TODO -- check this is right when e.g the mask is 0x80000000
4596 operand immILog()
4597 %{
4598 predicate(Assembler::operand_valid_for_logical_immediate(/*is32*/true, (uint64_t)n->get_int()));
4599 match(ConI);
4600
4601 op_cost(0);
4602 format %{ %}
4603 interface(CONST_INTER);
4604 %}
4605
4606 // Integer operands 64 bit
4607 // 64 bit immediate
4608 operand immL()
4609 %{
4610 match(ConL);
4611
4612 op_cost(0);
4613 format %{ %}
4614 interface(CONST_INTER);
4615 %}
4616
4617 // 64 bit zero
4618 operand immL0()
4619 %{
4620 predicate(n->get_long() == 0);
4621 match(ConL);
4622
4623 op_cost(0);
4624 format %{ %}
4625 interface(CONST_INTER);
4626 %}
4627
4628 // 64 bit unit increment
4629 operand immL_1()
4630 %{
4631 predicate(n->get_long() == 1);
4632 match(ConL);
4633
4634 op_cost(0);
4635 format %{ %}
4636 interface(CONST_INTER);
4637 %}
4638
4639 // 64 bit unit decrement
4640 operand immL_M1()
4641 %{
4642 predicate(n->get_long() == -1);
4643 match(ConL);
4644
4645 op_cost(0);
4646 format %{ %}
4647 interface(CONST_INTER);
4648 %}
4649
4650 // 32 bit offset of pc in thread anchor
4651
4652 operand immL_pc_off()
4653 %{
4654 predicate(n->get_long() == in_bytes(JavaThread::frame_anchor_offset()) +
4655 in_bytes(JavaFrameAnchor::last_Java_pc_offset()));
4656 match(ConL);
4657
4658 op_cost(0);
4659 format %{ %}
4660 interface(CONST_INTER);
4661 %}
4662
4663 // 64 bit integer valid for add sub immediate
4664 operand immLAddSub()
4665 %{
4666 predicate(Assembler::operand_valid_for_add_sub_immediate(n->get_long()));
4667 match(ConL);
4668 op_cost(0);
4669 format %{ %}
4670 interface(CONST_INTER);
4671 %}
4672
4673 // 64 bit integer valid for logical immediate
4674 operand immLLog()
4675 %{
4676 predicate(Assembler::operand_valid_for_logical_immediate(/*is32*/false, (uint64_t)n->get_long()));
4677 match(ConL);
4678 op_cost(0);
4679 format %{ %}
4680 interface(CONST_INTER);
4681 %}
4682
4683 // Long Immediate: low 32-bit mask
4684 operand immL_32bits()
4685 %{
4686 predicate(n->get_long() == 0xFFFFFFFFL);
4687 match(ConL);
4688 op_cost(0);
4689 format %{ %}
4690 interface(CONST_INTER);
4691 %}
4692
4693 // Pointer operands
4694 // Pointer Immediate
4695 operand immP()
4696 %{
4697 match(ConP);
4698
4699 op_cost(0);
4700 format %{ %}
4701 interface(CONST_INTER);
4702 %}
4703
4704 // NULL Pointer Immediate
4705 operand immP0()
4706 %{
4707 predicate(n->get_ptr() == 0);
4708 match(ConP);
4709
4710 op_cost(0);
4711 format %{ %}
4712 interface(CONST_INTER);
4713 %}
4714
4715 // Pointer Immediate One
4716 // this is used in object initialization (initial object header)
4717 operand immP_1()
4718 %{
4719 predicate(n->get_ptr() == 1);
4720 match(ConP);
4721
4722 op_cost(0);
4723 format %{ %}
4724 interface(CONST_INTER);
4725 %}
4726
4727 // Card Table Byte Map Base
4728 operand immByteMapBase()
4729 %{
4730 // Get base of card map
4731 predicate(BarrierSet::barrier_set()->is_a(BarrierSet::CardTableBarrierSet) &&
4732 (CardTable::CardValue*)n->get_ptr() == ((CardTableBarrierSet*)(BarrierSet::barrier_set()))->card_table()->byte_map_base());
4733 match(ConP);
4734
4735 op_cost(0);
4736 format %{ %}
4737 interface(CONST_INTER);
4738 %}
4739
4740 // Pointer Immediate Minus One
4741 // this is used when we want to write the current PC to the thread anchor
4742 operand immP_M1()
4743 %{
4744 predicate(n->get_ptr() == -1);
4745 match(ConP);
4746
4747 op_cost(0);
4748 format %{ %}
4749 interface(CONST_INTER);
4750 %}
4751
4752 // Pointer Immediate Minus Two
4753 // this is used when we want to write the current PC to the thread anchor
4754 operand immP_M2()
4755 %{
4756 predicate(n->get_ptr() == -2);
4757 match(ConP);
4758
4759 op_cost(0);
4760 format %{ %}
4761 interface(CONST_INTER);
4762 %}
4763
4764 // Float and Double operands
4765 // Double Immediate
4766 operand immD()
4767 %{
4768 match(ConD);
4769 op_cost(0);
4770 format %{ %}
4771 interface(CONST_INTER);
4772 %}
4773
4774 // Double Immediate: +0.0d
4775 operand immD0()
4776 %{
4777 predicate(jlong_cast(n->getd()) == 0);
4778 match(ConD);
4779
4780 op_cost(0);
4781 format %{ %}
4782 interface(CONST_INTER);
4783 %}
4784
4785 // constant 'double +0.0'.
4786 operand immDPacked()
4787 %{
4788 predicate(Assembler::operand_valid_for_float_immediate(n->getd()));
4789 match(ConD);
4790 op_cost(0);
4791 format %{ %}
4792 interface(CONST_INTER);
4793 %}
4794
4795 // Float Immediate
4796 operand immF()
4797 %{
4798 match(ConF);
4799 op_cost(0);
4800 format %{ %}
4801 interface(CONST_INTER);
4802 %}
4803
4804 // Float Immediate: +0.0f.
4805 operand immF0()
4806 %{
4807 predicate(jint_cast(n->getf()) == 0);
4808 match(ConF);
4809
4810 op_cost(0);
4811 format %{ %}
4812 interface(CONST_INTER);
4813 %}
4814
4815 //
4816 operand immFPacked()
4817 %{
4818 predicate(Assembler::operand_valid_for_float_immediate((double)n->getf()));
4819 match(ConF);
4820 op_cost(0);
4821 format %{ %}
4822 interface(CONST_INTER);
4823 %}
4824
4825 // Narrow pointer operands
4826 // Narrow Pointer Immediate
4827 operand immN()
4828 %{
4829 match(ConN);
4830
4831 op_cost(0);
4832 format %{ %}
4833 interface(CONST_INTER);
4834 %}
4835
4836 // Narrow NULL Pointer Immediate
4837 operand immN0()
4838 %{
4839 predicate(n->get_narrowcon() == 0);
4840 match(ConN);
4841
4842 op_cost(0);
4843 format %{ %}
4844 interface(CONST_INTER);
4845 %}
4846
4847 operand immNKlass()
4848 %{
4849 match(ConNKlass);
4850
4851 op_cost(0);
4852 format %{ %}
4853 interface(CONST_INTER);
4854 %}
4855
4856 // Integer 32 bit Register Operands
4857 // Integer 32 bitRegister (excludes SP)
4858 operand iRegI()
4859 %{
4860 constraint(ALLOC_IN_RC(any_reg32));
4861 match(RegI);
4862 match(iRegINoSp);
4863 op_cost(0);
4864 format %{ %}
4865 interface(REG_INTER);
4866 %}
4867
4868 // Integer 32 bit Register not Special
4869 operand iRegINoSp()
4870 %{
4871 constraint(ALLOC_IN_RC(no_special_reg32));
4872 match(RegI);
4873 op_cost(0);
4874 format %{ %}
4875 interface(REG_INTER);
4876 %}
4877
4878 // Integer 64 bit Register Operands
4879 // Integer 64 bit Register (includes SP)
4880 operand iRegL()
4881 %{
4882 constraint(ALLOC_IN_RC(any_reg));
4883 match(RegL);
4884 match(iRegLNoSp);
4885 op_cost(0);
4886 format %{ %}
4887 interface(REG_INTER);
4888 %}
4889
4890 // Integer 64 bit Register not Special
4891 operand iRegLNoSp()
4892 %{
4893 constraint(ALLOC_IN_RC(no_special_reg));
4894 match(RegL);
4895 match(iRegL_R0);
4896 format %{ %}
4897 interface(REG_INTER);
4898 %}
4899
4900 // Pointer Register Operands
4901 // Pointer Register
4902 operand iRegP()
4903 %{
4904 constraint(ALLOC_IN_RC(ptr_reg));
4905 match(RegP);
4906 match(iRegPNoSp);
4907 match(iRegP_R0);
4908 //match(iRegP_R2);
4909 //match(iRegP_R4);
4910 //match(iRegP_R5);
4911 match(thread_RegP);
4912 op_cost(0);
4913 format %{ %}
4914 interface(REG_INTER);
4915 %}
4916
4917 // Pointer 64 bit Register not Special
4918 operand iRegPNoSp()
4919 %{
4920 constraint(ALLOC_IN_RC(no_special_ptr_reg));
4921 match(RegP);
4922 // match(iRegP);
4923 // match(iRegP_R0);
4924 // match(iRegP_R2);
4925 // match(iRegP_R4);
4926 // match(iRegP_R5);
4927 // match(thread_RegP);
4928 op_cost(0);
4929 format %{ %}
4930 interface(REG_INTER);
4931 %}
4932
4933 // Pointer 64 bit Register R0 only
4934 operand iRegP_R0()
4935 %{
4936 constraint(ALLOC_IN_RC(r0_reg));
4937 match(RegP);
4938 // match(iRegP);
4939 match(iRegPNoSp);
4940 op_cost(0);
4941 format %{ %}
4942 interface(REG_INTER);
4943 %}
4944
4945 // Pointer 64 bit Register R1 only
4946 operand iRegP_R1()
4947 %{
4948 constraint(ALLOC_IN_RC(r1_reg));
4949 match(RegP);
4950 // match(iRegP);
4951 match(iRegPNoSp);
4952 op_cost(0);
4953 format %{ %}
4954 interface(REG_INTER);
4955 %}
4956
4957 // Pointer 64 bit Register R2 only
4958 operand iRegP_R2()
4959 %{
4960 constraint(ALLOC_IN_RC(r2_reg));
4961 match(RegP);
4962 // match(iRegP);
4963 match(iRegPNoSp);
4964 op_cost(0);
4965 format %{ %}
4966 interface(REG_INTER);
4967 %}
4968
4969 // Pointer 64 bit Register R3 only
4970 operand iRegP_R3()
4971 %{
4972 constraint(ALLOC_IN_RC(r3_reg));
4973 match(RegP);
4974 // match(iRegP);
4975 match(iRegPNoSp);
4976 op_cost(0);
4977 format %{ %}
4978 interface(REG_INTER);
4979 %}
4980
4981 // Pointer 64 bit Register R4 only
4982 operand iRegP_R4()
4983 %{
4984 constraint(ALLOC_IN_RC(r4_reg));
4985 match(RegP);
4986 // match(iRegP);
4987 match(iRegPNoSp);
4988 op_cost(0);
4989 format %{ %}
4990 interface(REG_INTER);
4991 %}
4992
4993 // Pointer 64 bit Register R5 only
4994 operand iRegP_R5()
4995 %{
4996 constraint(ALLOC_IN_RC(r5_reg));
4997 match(RegP);
4998 // match(iRegP);
4999 match(iRegPNoSp);
5000 op_cost(0);
5001 format %{ %}
5002 interface(REG_INTER);
5003 %}
5004
5005 // Pointer 64 bit Register R10 only
5006 operand iRegP_R10()
5007 %{
5008 constraint(ALLOC_IN_RC(r10_reg));
5009 match(RegP);
5010 // match(iRegP);
5011 match(iRegPNoSp);
5012 op_cost(0);
5013 format %{ %}
5014 interface(REG_INTER);
5015 %}
5016
5017 // Long 64 bit Register R0 only
5018 operand iRegL_R0()
5019 %{
5020 constraint(ALLOC_IN_RC(r0_reg));
5021 match(RegL);
5022 match(iRegLNoSp);
5023 op_cost(0);
5024 format %{ %}
5025 interface(REG_INTER);
5026 %}
5027
5028 // Long 64 bit Register R2 only
5029 operand iRegL_R2()
5030 %{
5031 constraint(ALLOC_IN_RC(r2_reg));
5032 match(RegL);
5033 match(iRegLNoSp);
5034 op_cost(0);
5035 format %{ %}
5036 interface(REG_INTER);
5037 %}
5038
5039 // Long 64 bit Register R3 only
5040 operand iRegL_R3()
5041 %{
5042 constraint(ALLOC_IN_RC(r3_reg));
5043 match(RegL);
5044 match(iRegLNoSp);
5045 op_cost(0);
5046 format %{ %}
5047 interface(REG_INTER);
5048 %}
5049
5050 // Long 64 bit Register R11 only
5051 operand iRegL_R11()
5052 %{
5053 constraint(ALLOC_IN_RC(r11_reg));
5054 match(RegL);
5055 match(iRegLNoSp);
5056 op_cost(0);
5057 format %{ %}
5058 interface(REG_INTER);
5059 %}
5060
5061 // Pointer 64 bit Register FP only
5062 operand iRegP_FP()
5063 %{
5064 constraint(ALLOC_IN_RC(fp_reg));
5065 match(RegP);
5066 // match(iRegP);
5067 op_cost(0);
5068 format %{ %}
5069 interface(REG_INTER);
5070 %}
5071
5072 // Register R0 only
5073 operand iRegI_R0()
5074 %{
5075 constraint(ALLOC_IN_RC(int_r0_reg));
5076 match(RegI);
5077 match(iRegINoSp);
5078 op_cost(0);
5079 format %{ %}
5080 interface(REG_INTER);
5081 %}
5082
5083 // Register R2 only
5084 operand iRegI_R2()
5085 %{
5086 constraint(ALLOC_IN_RC(int_r2_reg));
5087 match(RegI);
5088 match(iRegINoSp);
5089 op_cost(0);
5090 format %{ %}
5091 interface(REG_INTER);
5092 %}
5093
5094 // Register R3 only
5095 operand iRegI_R3()
5096 %{
5097 constraint(ALLOC_IN_RC(int_r3_reg));
5098 match(RegI);
5099 match(iRegINoSp);
5100 op_cost(0);
5101 format %{ %}
5102 interface(REG_INTER);
5103 %}
5104
5105
5106 // Register R4 only
5107 operand iRegI_R4()
5108 %{
5109 constraint(ALLOC_IN_RC(int_r4_reg));
5110 match(RegI);
5111 match(iRegINoSp);
5112 op_cost(0);
5113 format %{ %}
5114 interface(REG_INTER);
5115 %}
5116
5117
5118 // Pointer Register Operands
5119 // Narrow Pointer Register
5120 operand iRegN()
5121 %{
5122 constraint(ALLOC_IN_RC(any_reg32));
5123 match(RegN);
5124 match(iRegNNoSp);
5125 op_cost(0);
5126 format %{ %}
5127 interface(REG_INTER);
5128 %}
5129
5130 operand iRegN_R0()
5131 %{
5132 constraint(ALLOC_IN_RC(r0_reg));
5133 match(iRegN);
5134 op_cost(0);
5135 format %{ %}
5136 interface(REG_INTER);
5137 %}
5138
5139 operand iRegN_R2()
5140 %{
5141 constraint(ALLOC_IN_RC(r2_reg));
5142 match(iRegN);
5143 op_cost(0);
5144 format %{ %}
5145 interface(REG_INTER);
5146 %}
5147
5148 operand iRegN_R3()
5149 %{
5150 constraint(ALLOC_IN_RC(r3_reg));
5151 match(iRegN);
5152 op_cost(0);
5153 format %{ %}
5154 interface(REG_INTER);
5155 %}
5156
5157 // Integer 64 bit Register not Special
5158 operand iRegNNoSp()
5159 %{
5160 constraint(ALLOC_IN_RC(no_special_reg32));
5161 match(RegN);
5162 op_cost(0);
5163 format %{ %}
5164 interface(REG_INTER);
5165 %}
5166
5167 // heap base register -- used for encoding immN0
5168
5169 operand iRegIHeapbase()
5170 %{
5171 constraint(ALLOC_IN_RC(heapbase_reg));
5172 match(RegI);
5173 op_cost(0);
5174 format %{ %}
5175 interface(REG_INTER);
5176 %}
5177
5178 // Float Register
5179 // Float register operands
5180 operand vRegF()
5181 %{
5182 constraint(ALLOC_IN_RC(float_reg));
5183 match(RegF);
5184
5185 op_cost(0);
5186 format %{ %}
5187 interface(REG_INTER);
5188 %}
5189
5190 // Double Register
5191 // Double register operands
5192 operand vRegD()
5193 %{
5194 constraint(ALLOC_IN_RC(double_reg));
5195 match(RegD);
5196
5197 op_cost(0);
5198 format %{ %}
5199 interface(REG_INTER);
5200 %}
5201
5202 // Generic vector class. This will be used for
5203 // all vector operands, including NEON and SVE,
5204 // but currently only used for SVE VecA.
5205 operand vReg()
5206 %{
5207 constraint(ALLOC_IN_RC(vectora_reg));
5208 match(VecA);
5209 op_cost(0);
5210 format %{ %}
5211 interface(REG_INTER);
5212 %}
5213
5214 operand vecD()
5215 %{
5216 constraint(ALLOC_IN_RC(vectord_reg));
5217 match(VecD);
5218
5219 op_cost(0);
5220 format %{ %}
5221 interface(REG_INTER);
5222 %}
5223
5224 operand vecX()
5225 %{
5226 constraint(ALLOC_IN_RC(vectorx_reg));
5227 match(VecX);
5228
5229 op_cost(0);
5230 format %{ %}
5231 interface(REG_INTER);
5232 %}
5233
5234 operand vRegD_V0()
5235 %{
5236 constraint(ALLOC_IN_RC(v0_reg));
5237 match(RegD);
5238 op_cost(0);
5239 format %{ %}
5240 interface(REG_INTER);
5241 %}
5242
5243 operand vRegD_V1()
5244 %{
5245 constraint(ALLOC_IN_RC(v1_reg));
5246 match(RegD);
5247 op_cost(0);
5248 format %{ %}
5249 interface(REG_INTER);
5250 %}
5251
5252 operand vRegD_V2()
5253 %{
5254 constraint(ALLOC_IN_RC(v2_reg));
5255 match(RegD);
5256 op_cost(0);
5257 format %{ %}
5258 interface(REG_INTER);
5259 %}
5260
5261 operand vRegD_V3()
5262 %{
5263 constraint(ALLOC_IN_RC(v3_reg));
5264 match(RegD);
5265 op_cost(0);
5266 format %{ %}
5267 interface(REG_INTER);
5268 %}
5269
5270 operand vRegD_V4()
5271 %{
5272 constraint(ALLOC_IN_RC(v4_reg));
5273 match(RegD);
5274 op_cost(0);
5275 format %{ %}
5276 interface(REG_INTER);
5277 %}
5278
5279 operand vRegD_V5()
5280 %{
5281 constraint(ALLOC_IN_RC(v5_reg));
5282 match(RegD);
5283 op_cost(0);
5284 format %{ %}
5285 interface(REG_INTER);
5286 %}
5287
5288 operand vRegD_V6()
5289 %{
5290 constraint(ALLOC_IN_RC(v6_reg));
5291 match(RegD);
5292 op_cost(0);
5293 format %{ %}
5294 interface(REG_INTER);
5295 %}
5296
5297 operand vRegD_V7()
5298 %{
5299 constraint(ALLOC_IN_RC(v7_reg));
5300 match(RegD);
5301 op_cost(0);
5302 format %{ %}
5303 interface(REG_INTER);
5304 %}
5305
5306 operand vRegD_V8()
5307 %{
5308 constraint(ALLOC_IN_RC(v8_reg));
5309 match(RegD);
5310 op_cost(0);
5311 format %{ %}
5312 interface(REG_INTER);
5313 %}
5314
5315 operand vRegD_V9()
5316 %{
5317 constraint(ALLOC_IN_RC(v9_reg));
5318 match(RegD);
5319 op_cost(0);
5320 format %{ %}
5321 interface(REG_INTER);
5322 %}
5323
5324 operand vRegD_V10()
5325 %{
5326 constraint(ALLOC_IN_RC(v10_reg));
5327 match(RegD);
5328 op_cost(0);
5329 format %{ %}
5330 interface(REG_INTER);
5331 %}
5332
5333 operand vRegD_V11()
5334 %{
5335 constraint(ALLOC_IN_RC(v11_reg));
5336 match(RegD);
5337 op_cost(0);
5338 format %{ %}
5339 interface(REG_INTER);
5340 %}
5341
5342 operand vRegD_V12()
5343 %{
5344 constraint(ALLOC_IN_RC(v12_reg));
5345 match(RegD);
5346 op_cost(0);
5347 format %{ %}
5348 interface(REG_INTER);
5349 %}
5350
5351 operand vRegD_V13()
5352 %{
5353 constraint(ALLOC_IN_RC(v13_reg));
5354 match(RegD);
5355 op_cost(0);
5356 format %{ %}
5357 interface(REG_INTER);
5358 %}
5359
5360 operand vRegD_V14()
5361 %{
5362 constraint(ALLOC_IN_RC(v14_reg));
5363 match(RegD);
5364 op_cost(0);
5365 format %{ %}
5366 interface(REG_INTER);
5367 %}
5368
5369 operand vRegD_V15()
5370 %{
5371 constraint(ALLOC_IN_RC(v15_reg));
5372 match(RegD);
5373 op_cost(0);
5374 format %{ %}
5375 interface(REG_INTER);
5376 %}
5377
5378 operand vRegD_V16()
5379 %{
5380 constraint(ALLOC_IN_RC(v16_reg));
5381 match(RegD);
5382 op_cost(0);
5383 format %{ %}
5384 interface(REG_INTER);
5385 %}
5386
5387 operand vRegD_V17()
5388 %{
5389 constraint(ALLOC_IN_RC(v17_reg));
5390 match(RegD);
5391 op_cost(0);
5392 format %{ %}
5393 interface(REG_INTER);
5394 %}
5395
5396 operand vRegD_V18()
5397 %{
5398 constraint(ALLOC_IN_RC(v18_reg));
5399 match(RegD);
5400 op_cost(0);
5401 format %{ %}
5402 interface(REG_INTER);
5403 %}
5404
5405 operand vRegD_V19()
5406 %{
5407 constraint(ALLOC_IN_RC(v19_reg));
5408 match(RegD);
5409 op_cost(0);
5410 format %{ %}
5411 interface(REG_INTER);
5412 %}
5413
5414 operand vRegD_V20()
5415 %{
5416 constraint(ALLOC_IN_RC(v20_reg));
5417 match(RegD);
5418 op_cost(0);
5419 format %{ %}
5420 interface(REG_INTER);
5421 %}
5422
5423 operand vRegD_V21()
5424 %{
5425 constraint(ALLOC_IN_RC(v21_reg));
5426 match(RegD);
5427 op_cost(0);
5428 format %{ %}
5429 interface(REG_INTER);
5430 %}
5431
5432 operand vRegD_V22()
5433 %{
5434 constraint(ALLOC_IN_RC(v22_reg));
5435 match(RegD);
5436 op_cost(0);
5437 format %{ %}
5438 interface(REG_INTER);
5439 %}
5440
5441 operand vRegD_V23()
5442 %{
5443 constraint(ALLOC_IN_RC(v23_reg));
5444 match(RegD);
5445 op_cost(0);
5446 format %{ %}
5447 interface(REG_INTER);
5448 %}
5449
5450 operand vRegD_V24()
5451 %{
5452 constraint(ALLOC_IN_RC(v24_reg));
5453 match(RegD);
5454 op_cost(0);
5455 format %{ %}
5456 interface(REG_INTER);
5457 %}
5458
5459 operand vRegD_V25()
5460 %{
5461 constraint(ALLOC_IN_RC(v25_reg));
5462 match(RegD);
5463 op_cost(0);
5464 format %{ %}
5465 interface(REG_INTER);
5466 %}
5467
5468 operand vRegD_V26()
5469 %{
5470 constraint(ALLOC_IN_RC(v26_reg));
5471 match(RegD);
5472 op_cost(0);
5473 format %{ %}
5474 interface(REG_INTER);
5475 %}
5476
5477 operand vRegD_V27()
5478 %{
5479 constraint(ALLOC_IN_RC(v27_reg));
5480 match(RegD);
5481 op_cost(0);
5482 format %{ %}
5483 interface(REG_INTER);
5484 %}
5485
5486 operand vRegD_V28()
5487 %{
5488 constraint(ALLOC_IN_RC(v28_reg));
5489 match(RegD);
5490 op_cost(0);
5491 format %{ %}
5492 interface(REG_INTER);
5493 %}
5494
5495 operand vRegD_V29()
5496 %{
5497 constraint(ALLOC_IN_RC(v29_reg));
5498 match(RegD);
5499 op_cost(0);
5500 format %{ %}
5501 interface(REG_INTER);
5502 %}
5503
5504 operand vRegD_V30()
5505 %{
5506 constraint(ALLOC_IN_RC(v30_reg));
5507 match(RegD);
5508 op_cost(0);
5509 format %{ %}
5510 interface(REG_INTER);
5511 %}
5512
5513 operand vRegD_V31()
5514 %{
5515 constraint(ALLOC_IN_RC(v31_reg));
5516 match(RegD);
5517 op_cost(0);
5518 format %{ %}
5519 interface(REG_INTER);
5520 %}
5521
5522 operand pReg()
5523 %{
5524 constraint(ALLOC_IN_RC(pr_reg));
5525 match(RegVectMask);
5526 op_cost(0);
5527 format %{ %}
5528 interface(REG_INTER);
5529 %}
5530
5531 operand pRegGov()
5532 %{
5533 constraint(ALLOC_IN_RC(gov_pr));
5534 match(RegVectMask);
5535 op_cost(0);
5536 format %{ %}
5537 interface(REG_INTER);
5538 %}
5539
5540 // Flags register, used as output of signed compare instructions
5541
5542 // note that on AArch64 we also use this register as the output for
5543 // for floating point compare instructions (CmpF CmpD). this ensures
5544 // that ordered inequality tests use GT, GE, LT or LE none of which
5545 // pass through cases where the result is unordered i.e. one or both
5546 // inputs to the compare is a NaN. this means that the ideal code can
5547 // replace e.g. a GT with an LE and not end up capturing the NaN case
5548 // (where the comparison should always fail). EQ and NE tests are
5549 // always generated in ideal code so that unordered folds into the NE
5550 // case, matching the behaviour of AArch64 NE.
5551 //
5552 // This differs from x86 where the outputs of FP compares use a
5553 // special FP flags registers and where compares based on this
5554 // register are distinguished into ordered inequalities (cmpOpUCF) and
5555 // EQ/NEQ tests (cmpOpUCF2). x86 has to special case the latter tests
5556 // to explicitly handle the unordered case in branches. x86 also has
5557 // to include extra CMoveX rules to accept a cmpOpUCF input.
5558
5559 operand rFlagsReg()
5560 %{
5561 constraint(ALLOC_IN_RC(int_flags));
5562 match(RegFlags);
5563
5564 op_cost(0);
5565 format %{ "RFLAGS" %}
5566 interface(REG_INTER);
5567 %}
5568
5569 // Flags register, used as output of unsigned compare instructions
5570 operand rFlagsRegU()
5571 %{
5572 constraint(ALLOC_IN_RC(int_flags));
5573 match(RegFlags);
5574
5575 op_cost(0);
5576 format %{ "RFLAGSU" %}
5577 interface(REG_INTER);
5578 %}
5579
5580 // Special Registers
5581
5582 // Method Register
5583 operand inline_cache_RegP(iRegP reg)
5584 %{
5585 constraint(ALLOC_IN_RC(method_reg)); // inline_cache_reg
5586 match(reg);
5587 match(iRegPNoSp);
5588 op_cost(0);
5589 format %{ %}
5590 interface(REG_INTER);
5591 %}
5592
5593 // Thread Register
5594 operand thread_RegP(iRegP reg)
5595 %{
5596 constraint(ALLOC_IN_RC(thread_reg)); // link_reg
5597 match(reg);
5598 op_cost(0);
5599 format %{ %}
5600 interface(REG_INTER);
5601 %}
5602
5603 operand lr_RegP(iRegP reg)
5604 %{
5605 constraint(ALLOC_IN_RC(lr_reg)); // link_reg
5606 match(reg);
5607 op_cost(0);
5608 format %{ %}
5609 interface(REG_INTER);
5610 %}
5611
5612 //----------Memory Operands----------------------------------------------------
5613
5614 operand indirect(iRegP reg)
5615 %{
5616 constraint(ALLOC_IN_RC(ptr_reg));
5617 match(reg);
5618 op_cost(0);
5619 format %{ "[$reg]" %}
5620 interface(MEMORY_INTER) %{
5621 base($reg);
5622 index(0xffffffff);
5623 scale(0x0);
5624 disp(0x0);
5625 %}
5626 %}
5627
5628 operand indIndexScaledI2L(iRegP reg, iRegI ireg, immIScale scale)
5629 %{
5630 constraint(ALLOC_IN_RC(ptr_reg));
5631 predicate(size_fits_all_mem_uses(n->as_AddP(), n->in(AddPNode::Offset)->in(2)->get_int()));
5632 match(AddP reg (LShiftL (ConvI2L ireg) scale));
5633 op_cost(0);
5634 format %{ "$reg, $ireg sxtw($scale), 0, I2L" %}
5635 interface(MEMORY_INTER) %{
5636 base($reg);
5637 index($ireg);
5638 scale($scale);
5639 disp(0x0);
5640 %}
5641 %}
5642
5643 operand indIndexScaled(iRegP reg, iRegL lreg, immIScale scale)
5644 %{
5645 constraint(ALLOC_IN_RC(ptr_reg));
5646 predicate(size_fits_all_mem_uses(n->as_AddP(), n->in(AddPNode::Offset)->in(2)->get_int()));
5647 match(AddP reg (LShiftL lreg scale));
5648 op_cost(0);
5649 format %{ "$reg, $lreg lsl($scale)" %}
5650 interface(MEMORY_INTER) %{
5651 base($reg);
5652 index($lreg);
5653 scale($scale);
5654 disp(0x0);
5655 %}
5656 %}
5657
5658 operand indIndexI2L(iRegP reg, iRegI ireg)
5659 %{
5660 constraint(ALLOC_IN_RC(ptr_reg));
5661 match(AddP reg (ConvI2L ireg));
5662 op_cost(0);
5663 format %{ "$reg, $ireg, 0, I2L" %}
5664 interface(MEMORY_INTER) %{
5665 base($reg);
5666 index($ireg);
5667 scale(0x0);
5668 disp(0x0);
5669 %}
5670 %}
5671
5672 operand indIndex(iRegP reg, iRegL lreg)
5673 %{
5674 constraint(ALLOC_IN_RC(ptr_reg));
5675 match(AddP reg lreg);
5676 op_cost(0);
5677 format %{ "$reg, $lreg" %}
5678 interface(MEMORY_INTER) %{
5679 base($reg);
5680 index($lreg);
5681 scale(0x0);
5682 disp(0x0);
5683 %}
5684 %}
5685
5686 operand indOffI(iRegP reg, immIOffset off)
5687 %{
5688 constraint(ALLOC_IN_RC(ptr_reg));
5689 match(AddP reg off);
5690 op_cost(0);
5691 format %{ "[$reg, $off]" %}
5692 interface(MEMORY_INTER) %{
5693 base($reg);
5694 index(0xffffffff);
5695 scale(0x0);
5696 disp($off);
5697 %}
5698 %}
5699
5700 operand indOffI1(iRegP reg, immIOffset1 off)
5701 %{
5702 constraint(ALLOC_IN_RC(ptr_reg));
5703 match(AddP reg off);
5704 op_cost(0);
5705 format %{ "[$reg, $off]" %}
5706 interface(MEMORY_INTER) %{
5707 base($reg);
5708 index(0xffffffff);
5709 scale(0x0);
5710 disp($off);
5711 %}
5712 %}
5713
5714 operand indOffI2(iRegP reg, immIOffset2 off)
5715 %{
5716 constraint(ALLOC_IN_RC(ptr_reg));
5717 match(AddP reg off);
5718 op_cost(0);
5719 format %{ "[$reg, $off]" %}
5720 interface(MEMORY_INTER) %{
5721 base($reg);
5722 index(0xffffffff);
5723 scale(0x0);
5724 disp($off);
5725 %}
5726 %}
5727
5728 operand indOffI4(iRegP reg, immIOffset4 off)
5729 %{
5730 constraint(ALLOC_IN_RC(ptr_reg));
5731 match(AddP reg off);
5732 op_cost(0);
5733 format %{ "[$reg, $off]" %}
5734 interface(MEMORY_INTER) %{
5735 base($reg);
5736 index(0xffffffff);
5737 scale(0x0);
5738 disp($off);
5739 %}
5740 %}
5741
5742 operand indOffI8(iRegP reg, immIOffset8 off)
5743 %{
5744 constraint(ALLOC_IN_RC(ptr_reg));
5745 match(AddP reg off);
5746 op_cost(0);
5747 format %{ "[$reg, $off]" %}
5748 interface(MEMORY_INTER) %{
5749 base($reg);
5750 index(0xffffffff);
5751 scale(0x0);
5752 disp($off);
5753 %}
5754 %}
5755
5756 operand indOffI16(iRegP reg, immIOffset16 off)
5757 %{
5758 constraint(ALLOC_IN_RC(ptr_reg));
5759 match(AddP reg off);
5760 op_cost(0);
5761 format %{ "[$reg, $off]" %}
5762 interface(MEMORY_INTER) %{
5763 base($reg);
5764 index(0xffffffff);
5765 scale(0x0);
5766 disp($off);
5767 %}
5768 %}
5769
5770 operand indOffL(iRegP reg, immLoffset off)
5771 %{
5772 constraint(ALLOC_IN_RC(ptr_reg));
5773 match(AddP reg off);
5774 op_cost(0);
5775 format %{ "[$reg, $off]" %}
5776 interface(MEMORY_INTER) %{
5777 base($reg);
5778 index(0xffffffff);
5779 scale(0x0);
5780 disp($off);
5781 %}
5782 %}
5783
5784 operand indOffL1(iRegP reg, immLoffset1 off)
5785 %{
5786 constraint(ALLOC_IN_RC(ptr_reg));
5787 match(AddP reg off);
5788 op_cost(0);
5789 format %{ "[$reg, $off]" %}
5790 interface(MEMORY_INTER) %{
5791 base($reg);
5792 index(0xffffffff);
5793 scale(0x0);
5794 disp($off);
5795 %}
5796 %}
5797
5798 operand indOffL2(iRegP reg, immLoffset2 off)
5799 %{
5800 constraint(ALLOC_IN_RC(ptr_reg));
5801 match(AddP reg off);
5802 op_cost(0);
5803 format %{ "[$reg, $off]" %}
5804 interface(MEMORY_INTER) %{
5805 base($reg);
5806 index(0xffffffff);
5807 scale(0x0);
5808 disp($off);
5809 %}
5810 %}
5811
5812 operand indOffL4(iRegP reg, immLoffset4 off)
5813 %{
5814 constraint(ALLOC_IN_RC(ptr_reg));
5815 match(AddP reg off);
5816 op_cost(0);
5817 format %{ "[$reg, $off]" %}
5818 interface(MEMORY_INTER) %{
5819 base($reg);
5820 index(0xffffffff);
5821 scale(0x0);
5822 disp($off);
5823 %}
5824 %}
5825
5826 operand indOffL8(iRegP reg, immLoffset8 off)
5827 %{
5828 constraint(ALLOC_IN_RC(ptr_reg));
5829 match(AddP reg off);
5830 op_cost(0);
5831 format %{ "[$reg, $off]" %}
5832 interface(MEMORY_INTER) %{
5833 base($reg);
5834 index(0xffffffff);
5835 scale(0x0);
5836 disp($off);
5837 %}
5838 %}
5839
5840 operand indOffL16(iRegP reg, immLoffset16 off)
5841 %{
5842 constraint(ALLOC_IN_RC(ptr_reg));
5843 match(AddP reg off);
5844 op_cost(0);
5845 format %{ "[$reg, $off]" %}
5846 interface(MEMORY_INTER) %{
5847 base($reg);
5848 index(0xffffffff);
5849 scale(0x0);
5850 disp($off);
5851 %}
5852 %}
5853
5854 operand indirectN(iRegN reg)
5855 %{
5856 predicate(CompressedOops::shift() == 0);
5857 constraint(ALLOC_IN_RC(ptr_reg));
5858 match(DecodeN reg);
5859 op_cost(0);
5860 format %{ "[$reg]\t# narrow" %}
5861 interface(MEMORY_INTER) %{
5862 base($reg);
5863 index(0xffffffff);
5864 scale(0x0);
5865 disp(0x0);
5866 %}
5867 %}
5868
5869 operand indIndexScaledI2LN(iRegN reg, iRegI ireg, immIScale scale)
5870 %{
5871 predicate(CompressedOops::shift() == 0 && size_fits_all_mem_uses(n->as_AddP(), n->in(AddPNode::Offset)->in(2)->get_int()));
5872 constraint(ALLOC_IN_RC(ptr_reg));
5873 match(AddP (DecodeN reg) (LShiftL (ConvI2L ireg) scale));
5874 op_cost(0);
5875 format %{ "$reg, $ireg sxtw($scale), 0, I2L\t# narrow" %}
5876 interface(MEMORY_INTER) %{
5877 base($reg);
5878 index($ireg);
5879 scale($scale);
5880 disp(0x0);
5881 %}
5882 %}
5883
5884 operand indIndexScaledN(iRegN reg, iRegL lreg, immIScale scale)
5885 %{
5886 predicate(CompressedOops::shift() == 0 && size_fits_all_mem_uses(n->as_AddP(), n->in(AddPNode::Offset)->in(2)->get_int()));
5887 constraint(ALLOC_IN_RC(ptr_reg));
5888 match(AddP (DecodeN reg) (LShiftL lreg scale));
5889 op_cost(0);
5890 format %{ "$reg, $lreg lsl($scale)\t# narrow" %}
5891 interface(MEMORY_INTER) %{
5892 base($reg);
5893 index($lreg);
5894 scale($scale);
5895 disp(0x0);
5896 %}
5897 %}
5898
5899 operand indIndexI2LN(iRegN reg, iRegI ireg)
5900 %{
5901 predicate(CompressedOops::shift() == 0);
5902 constraint(ALLOC_IN_RC(ptr_reg));
5903 match(AddP (DecodeN reg) (ConvI2L ireg));
5904 op_cost(0);
5905 format %{ "$reg, $ireg, 0, I2L\t# narrow" %}
5906 interface(MEMORY_INTER) %{
5907 base($reg);
5908 index($ireg);
5909 scale(0x0);
5910 disp(0x0);
5911 %}
5912 %}
5913
5914 operand indIndexN(iRegN reg, iRegL lreg)
5915 %{
5916 predicate(CompressedOops::shift() == 0);
5917 constraint(ALLOC_IN_RC(ptr_reg));
5918 match(AddP (DecodeN reg) lreg);
5919 op_cost(0);
5920 format %{ "$reg, $lreg\t# narrow" %}
5921 interface(MEMORY_INTER) %{
5922 base($reg);
5923 index($lreg);
5924 scale(0x0);
5925 disp(0x0);
5926 %}
5927 %}
5928
5929 operand indOffIN(iRegN reg, immIOffset off)
5930 %{
5931 predicate(CompressedOops::shift() == 0);
5932 constraint(ALLOC_IN_RC(ptr_reg));
5933 match(AddP (DecodeN reg) off);
5934 op_cost(0);
5935 format %{ "[$reg, $off]\t# narrow" %}
5936 interface(MEMORY_INTER) %{
5937 base($reg);
5938 index(0xffffffff);
5939 scale(0x0);
5940 disp($off);
5941 %}
5942 %}
5943
5944 operand indOffLN(iRegN reg, immLoffset off)
5945 %{
5946 predicate(CompressedOops::shift() == 0);
5947 constraint(ALLOC_IN_RC(ptr_reg));
5948 match(AddP (DecodeN reg) off);
5949 op_cost(0);
5950 format %{ "[$reg, $off]\t# narrow" %}
5951 interface(MEMORY_INTER) %{
5952 base($reg);
5953 index(0xffffffff);
5954 scale(0x0);
5955 disp($off);
5956 %}
5957 %}
5958
5959
5960
5961 // AArch64 opto stubs need to write to the pc slot in the thread anchor
5962 operand thread_anchor_pc(thread_RegP reg, immL_pc_off off)
5963 %{
5964 constraint(ALLOC_IN_RC(ptr_reg));
5965 match(AddP reg off);
5966 op_cost(0);
5967 format %{ "[$reg, $off]" %}
5968 interface(MEMORY_INTER) %{
5969 base($reg);
5970 index(0xffffffff);
5971 scale(0x0);
5972 disp($off);
5973 %}
5974 %}
5975
5976 //----------Special Memory Operands--------------------------------------------
5977 // Stack Slot Operand - This operand is used for loading and storing temporary
5978 // values on the stack where a match requires a value to
5979 // flow through memory.
5980 operand stackSlotP(sRegP reg)
5981 %{
5982 constraint(ALLOC_IN_RC(stack_slots));
5983 op_cost(100);
5984 // No match rule because this operand is only generated in matching
5985 // match(RegP);
5986 format %{ "[$reg]" %}
5987 interface(MEMORY_INTER) %{
5988 base(0x1e); // RSP
5989 index(0x0); // No Index
5990 scale(0x0); // No Scale
5991 disp($reg); // Stack Offset
5992 %}
5993 %}
5994
5995 operand stackSlotI(sRegI reg)
5996 %{
5997 constraint(ALLOC_IN_RC(stack_slots));
5998 // No match rule because this operand is only generated in matching
5999 // match(RegI);
6000 format %{ "[$reg]" %}
6001 interface(MEMORY_INTER) %{
6002 base(0x1e); // RSP
6003 index(0x0); // No Index
6004 scale(0x0); // No Scale
6005 disp($reg); // Stack Offset
6006 %}
6007 %}
6008
6009 operand stackSlotF(sRegF reg)
6010 %{
6011 constraint(ALLOC_IN_RC(stack_slots));
6012 // No match rule because this operand is only generated in matching
6013 // match(RegF);
6014 format %{ "[$reg]" %}
6015 interface(MEMORY_INTER) %{
6016 base(0x1e); // RSP
6017 index(0x0); // No Index
6018 scale(0x0); // No Scale
6019 disp($reg); // Stack Offset
6020 %}
6021 %}
6022
6023 operand stackSlotD(sRegD reg)
6024 %{
6025 constraint(ALLOC_IN_RC(stack_slots));
6026 // No match rule because this operand is only generated in matching
6027 // match(RegD);
6028 format %{ "[$reg]" %}
6029 interface(MEMORY_INTER) %{
6030 base(0x1e); // RSP
6031 index(0x0); // No Index
6032 scale(0x0); // No Scale
6033 disp($reg); // Stack Offset
6034 %}
6035 %}
6036
6037 operand stackSlotL(sRegL reg)
6038 %{
6039 constraint(ALLOC_IN_RC(stack_slots));
6040 // No match rule because this operand is only generated in matching
6041 // match(RegL);
6042 format %{ "[$reg]" %}
6043 interface(MEMORY_INTER) %{
6044 base(0x1e); // RSP
6045 index(0x0); // No Index
6046 scale(0x0); // No Scale
6047 disp($reg); // Stack Offset
6048 %}
6049 %}
6050
6051 // Operands for expressing Control Flow
6052 // NOTE: Label is a predefined operand which should not be redefined in
6053 // the AD file. It is generically handled within the ADLC.
6054
6055 //----------Conditional Branch Operands----------------------------------------
6056 // Comparison Op - This is the operation of the comparison, and is limited to
6057 // the following set of codes:
6058 // L (<), LE (<=), G (>), GE (>=), E (==), NE (!=)
6059 //
6060 // Other attributes of the comparison, such as unsignedness, are specified
6061 // by the comparison instruction that sets a condition code flags register.
6062 // That result is represented by a flags operand whose subtype is appropriate
6063 // to the unsignedness (etc.) of the comparison.
6064 //
6065 // Later, the instruction which matches both the Comparison Op (a Bool) and
6066 // the flags (produced by the Cmp) specifies the coding of the comparison op
6067 // by matching a specific subtype of Bool operand below, such as cmpOpU.
6068
6069 // used for signed integral comparisons and fp comparisons
6070
6071 operand cmpOp()
6072 %{
6073 match(Bool);
6074
6075 format %{ "" %}
6076 interface(COND_INTER) %{
6077 equal(0x0, "eq");
6078 not_equal(0x1, "ne");
6079 less(0xb, "lt");
6080 greater_equal(0xa, "ge");
6081 less_equal(0xd, "le");
6082 greater(0xc, "gt");
6083 overflow(0x6, "vs");
6084 no_overflow(0x7, "vc");
6085 %}
6086 %}
6087
6088 // used for unsigned integral comparisons
6089
6090 operand cmpOpU()
6091 %{
6092 match(Bool);
6093
6094 format %{ "" %}
6095 interface(COND_INTER) %{
6096 equal(0x0, "eq");
6097 not_equal(0x1, "ne");
6098 less(0x3, "lo");
6099 greater_equal(0x2, "hs");
6100 less_equal(0x9, "ls");
6101 greater(0x8, "hi");
6102 overflow(0x6, "vs");
6103 no_overflow(0x7, "vc");
6104 %}
6105 %}
6106
6107 // used for certain integral comparisons which can be
6108 // converted to cbxx or tbxx instructions
6109
6110 operand cmpOpEqNe()
6111 %{
6112 match(Bool);
6113 op_cost(0);
6114 predicate(n->as_Bool()->_test._test == BoolTest::ne
6115 || n->as_Bool()->_test._test == BoolTest::eq);
6116
6117 format %{ "" %}
6118 interface(COND_INTER) %{
6119 equal(0x0, "eq");
6120 not_equal(0x1, "ne");
6121 less(0xb, "lt");
6122 greater_equal(0xa, "ge");
6123 less_equal(0xd, "le");
6124 greater(0xc, "gt");
6125 overflow(0x6, "vs");
6126 no_overflow(0x7, "vc");
6127 %}
6128 %}
6129
6130 // used for certain integral comparisons which can be
6131 // converted to cbxx or tbxx instructions
6132
6133 operand cmpOpLtGe()
6134 %{
6135 match(Bool);
6136 op_cost(0);
6137
6138 predicate(n->as_Bool()->_test._test == BoolTest::lt
6139 || n->as_Bool()->_test._test == BoolTest::ge);
6140
6141 format %{ "" %}
6142 interface(COND_INTER) %{
6143 equal(0x0, "eq");
6144 not_equal(0x1, "ne");
6145 less(0xb, "lt");
6146 greater_equal(0xa, "ge");
6147 less_equal(0xd, "le");
6148 greater(0xc, "gt");
6149 overflow(0x6, "vs");
6150 no_overflow(0x7, "vc");
6151 %}
6152 %}
6153
6154 // used for certain unsigned integral comparisons which can be
6155 // converted to cbxx or tbxx instructions
6156
6157 operand cmpOpUEqNeLtGe()
6158 %{
6159 match(Bool);
6160 op_cost(0);
6161
6162 predicate(n->as_Bool()->_test._test == BoolTest::eq
6163 || n->as_Bool()->_test._test == BoolTest::ne
6164 || n->as_Bool()->_test._test == BoolTest::lt
6165 || n->as_Bool()->_test._test == BoolTest::ge);
6166
6167 format %{ "" %}
6168 interface(COND_INTER) %{
6169 equal(0x0, "eq");
6170 not_equal(0x1, "ne");
6171 less(0xb, "lt");
6172 greater_equal(0xa, "ge");
6173 less_equal(0xd, "le");
6174 greater(0xc, "gt");
6175 overflow(0x6, "vs");
6176 no_overflow(0x7, "vc");
6177 %}
6178 %}
6179
6180 // Special operand allowing long args to int ops to be truncated for free
6181
6182 operand iRegL2I(iRegL reg) %{
6183
6184 op_cost(0);
6185
6186 match(ConvL2I reg);
6187
6188 format %{ "l2i($reg)" %}
6189
6190 interface(REG_INTER)
6191 %}
6192
6193 opclass vmem2(indirect, indIndex, indOffI2, indOffL2);
6194 opclass vmem4(indirect, indIndex, indOffI4, indOffL4);
6195 opclass vmem8(indirect, indIndex, indOffI8, indOffL8);
6196 opclass vmem16(indirect, indIndex, indOffI16, indOffL16);
6197
6198 //----------OPERAND CLASSES----------------------------------------------------
6199 // Operand Classes are groups of operands that are used as to simplify
6200 // instruction definitions by not requiring the AD writer to specify
6201 // separate instructions for every form of operand when the
6202 // instruction accepts multiple operand types with the same basic
6203 // encoding and format. The classic case of this is memory operands.
6204
6205 // memory is used to define read/write location for load/store
6206 // instruction defs. we can turn a memory op into an Address
6207
6208 opclass memory1(indirect, indIndexScaled, indIndexScaledI2L, indIndexI2L, indIndex, indOffI1, indOffL1,
6209 indirectN, indIndexScaledN, indIndexScaledI2LN, indIndexI2LN, indIndexN);
6210
6211 opclass memory2(indirect, indIndexScaled, indIndexScaledI2L, indIndexI2L, indIndex, indOffI2, indOffL2,
6212 indirectN, indIndexScaledN, indIndexScaledI2LN, indIndexI2LN, indIndexN);
6213
6214 opclass memory4(indirect, indIndexScaled, indIndexScaledI2L, indIndexI2L, indIndex, indOffI4, indOffL4,
6215 indirectN, indIndexScaledN, indIndexScaledI2LN, indIndexI2LN, indIndexN, indOffIN, indOffLN);
6216
6217 opclass memory8(indirect, indIndexScaled, indIndexScaledI2L, indIndexI2L, indIndex, indOffI8, indOffL8,
6218 indirectN, indIndexScaledN, indIndexScaledI2LN, indIndexI2LN, indIndexN, indOffIN, indOffLN);
6219
6220 // All of the memory operands. For the pipeline description.
6221 opclass memory(indirect, indIndexScaled, indIndexScaledI2L, indIndexI2L, indIndex,
6222 indOffI1, indOffL1, indOffI2, indOffL2, indOffI4, indOffL4, indOffI8, indOffL8,
6223 indirectN, indIndexScaledN, indIndexScaledI2LN, indIndexI2LN, indIndexN, indOffIN, indOffLN);
6224
6225
6226 // iRegIorL2I is used for src inputs in rules for 32 bit int (I)
6227 // operations. it allows the src to be either an iRegI or a (ConvL2I
6228 // iRegL). in the latter case the l2i normally planted for a ConvL2I
6229 // can be elided because the 32-bit instruction will just employ the
6230 // lower 32 bits anyway.
6231 //
6232 // n.b. this does not elide all L2I conversions. if the truncated
6233 // value is consumed by more than one operation then the ConvL2I
6234 // cannot be bundled into the consuming nodes so an l2i gets planted
6235 // (actually a movw $dst $src) and the downstream instructions consume
6236 // the result of the l2i as an iRegI input. That's a shame since the
6237 // movw is actually redundant but its not too costly.
6238
6239 opclass iRegIorL2I(iRegI, iRegL2I);
6240
6241 //----------PIPELINE-----------------------------------------------------------
6242 // Rules which define the behavior of the target architectures pipeline.
6243
6244 // For specific pipelines, eg A53, define the stages of that pipeline
6245 //pipe_desc(ISS, EX1, EX2, WR);
6246 #define ISS S0
6247 #define EX1 S1
6248 #define EX2 S2
6249 #define WR S3
6250
6251 // Integer ALU reg operation
6252 pipeline %{
6253
6254 attributes %{
6255 // ARM instructions are of fixed length
6256 fixed_size_instructions; // Fixed size instructions TODO does
6257 max_instructions_per_bundle = 4; // A53 = 2, A57 = 4
6258 // ARM instructions come in 32-bit word units
6259 instruction_unit_size = 4; // An instruction is 4 bytes long
6260 instruction_fetch_unit_size = 64; // The processor fetches one line
6261 instruction_fetch_units = 1; // of 64 bytes
6262
6263 // List of nop instructions
6264 nops( MachNop );
6265 %}
6266
6267 // We don't use an actual pipeline model so don't care about resources
6268 // or description. we do use pipeline classes to introduce fixed
6269 // latencies
6270
6271 //----------RESOURCES----------------------------------------------------------
6272 // Resources are the functional units available to the machine
6273
6274 resources( INS0, INS1, INS01 = INS0 | INS1,
6275 ALU0, ALU1, ALU = ALU0 | ALU1,
6276 MAC,
6277 DIV,
6278 BRANCH,
6279 LDST,
6280 NEON_FP);
6281
6282 //----------PIPELINE DESCRIPTION-----------------------------------------------
6283 // Pipeline Description specifies the stages in the machine's pipeline
6284
6285 // Define the pipeline as a generic 6 stage pipeline
6286 pipe_desc(S0, S1, S2, S3, S4, S5);
6287
6288 //----------PIPELINE CLASSES---------------------------------------------------
6289 // Pipeline Classes describe the stages in which input and output are
6290 // referenced by the hardware pipeline.
6291
6292 pipe_class fp_dop_reg_reg_s(vRegF dst, vRegF src1, vRegF src2)
6293 %{
6294 single_instruction;
6295 src1 : S1(read);
6296 src2 : S2(read);
6297 dst : S5(write);
6298 INS01 : ISS;
6299 NEON_FP : S5;
6300 %}
6301
6302 pipe_class fp_dop_reg_reg_d(vRegD dst, vRegD src1, vRegD src2)
6303 %{
6304 single_instruction;
6305 src1 : S1(read);
6306 src2 : S2(read);
6307 dst : S5(write);
6308 INS01 : ISS;
6309 NEON_FP : S5;
6310 %}
6311
6312 pipe_class fp_uop_s(vRegF dst, vRegF src)
6313 %{
6314 single_instruction;
6315 src : S1(read);
6316 dst : S5(write);
6317 INS01 : ISS;
6318 NEON_FP : S5;
6319 %}
6320
6321 pipe_class fp_uop_d(vRegD dst, vRegD src)
6322 %{
6323 single_instruction;
6324 src : S1(read);
6325 dst : S5(write);
6326 INS01 : ISS;
6327 NEON_FP : S5;
6328 %}
6329
6330 pipe_class fp_d2f(vRegF dst, vRegD src)
6331 %{
6332 single_instruction;
6333 src : S1(read);
6334 dst : S5(write);
6335 INS01 : ISS;
6336 NEON_FP : S5;
6337 %}
6338
6339 pipe_class fp_f2d(vRegD dst, vRegF src)
6340 %{
6341 single_instruction;
6342 src : S1(read);
6343 dst : S5(write);
6344 INS01 : ISS;
6345 NEON_FP : S5;
6346 %}
6347
6348 pipe_class fp_f2i(iRegINoSp dst, vRegF src)
6349 %{
6350 single_instruction;
6351 src : S1(read);
6352 dst : S5(write);
6353 INS01 : ISS;
6354 NEON_FP : S5;
6355 %}
6356
6357 pipe_class fp_f2l(iRegLNoSp dst, vRegF src)
6358 %{
6359 single_instruction;
6360 src : S1(read);
6361 dst : S5(write);
6362 INS01 : ISS;
6363 NEON_FP : S5;
6364 %}
6365
6366 pipe_class fp_i2f(vRegF dst, iRegIorL2I src)
6367 %{
6368 single_instruction;
6369 src : S1(read);
6370 dst : S5(write);
6371 INS01 : ISS;
6372 NEON_FP : S5;
6373 %}
6374
6375 pipe_class fp_l2f(vRegF dst, iRegL src)
6376 %{
6377 single_instruction;
6378 src : S1(read);
6379 dst : S5(write);
6380 INS01 : ISS;
6381 NEON_FP : S5;
6382 %}
6383
6384 pipe_class fp_d2i(iRegINoSp dst, vRegD src)
6385 %{
6386 single_instruction;
6387 src : S1(read);
6388 dst : S5(write);
6389 INS01 : ISS;
6390 NEON_FP : S5;
6391 %}
6392
6393 pipe_class fp_d2l(iRegLNoSp dst, vRegD src)
6394 %{
6395 single_instruction;
6396 src : S1(read);
6397 dst : S5(write);
6398 INS01 : ISS;
6399 NEON_FP : S5;
6400 %}
6401
6402 pipe_class fp_i2d(vRegD dst, iRegIorL2I src)
6403 %{
6404 single_instruction;
6405 src : S1(read);
6406 dst : S5(write);
6407 INS01 : ISS;
6408 NEON_FP : S5;
6409 %}
6410
6411 pipe_class fp_l2d(vRegD dst, iRegIorL2I src)
6412 %{
6413 single_instruction;
6414 src : S1(read);
6415 dst : S5(write);
6416 INS01 : ISS;
6417 NEON_FP : S5;
6418 %}
6419
6420 pipe_class fp_div_s(vRegF dst, vRegF src1, vRegF src2)
6421 %{
6422 single_instruction;
6423 src1 : S1(read);
6424 src2 : S2(read);
6425 dst : S5(write);
6426 INS0 : ISS;
6427 NEON_FP : S5;
6428 %}
6429
6430 pipe_class fp_div_d(vRegD dst, vRegD src1, vRegD src2)
6431 %{
6432 single_instruction;
6433 src1 : S1(read);
6434 src2 : S2(read);
6435 dst : S5(write);
6436 INS0 : ISS;
6437 NEON_FP : S5;
6438 %}
6439
6440 pipe_class fp_cond_reg_reg_s(vRegF dst, vRegF src1, vRegF src2, rFlagsReg cr)
6441 %{
6442 single_instruction;
6443 cr : S1(read);
6444 src1 : S1(read);
6445 src2 : S1(read);
6446 dst : S3(write);
6447 INS01 : ISS;
6448 NEON_FP : S3;
6449 %}
6450
6451 pipe_class fp_cond_reg_reg_d(vRegD dst, vRegD src1, vRegD src2, rFlagsReg cr)
6452 %{
6453 single_instruction;
6454 cr : S1(read);
6455 src1 : S1(read);
6456 src2 : S1(read);
6457 dst : S3(write);
6458 INS01 : ISS;
6459 NEON_FP : S3;
6460 %}
6461
6462 pipe_class fp_imm_s(vRegF dst)
6463 %{
6464 single_instruction;
6465 dst : S3(write);
6466 INS01 : ISS;
6467 NEON_FP : S3;
6468 %}
6469
6470 pipe_class fp_imm_d(vRegD dst)
6471 %{
6472 single_instruction;
6473 dst : S3(write);
6474 INS01 : ISS;
6475 NEON_FP : S3;
6476 %}
6477
6478 pipe_class fp_load_constant_s(vRegF dst)
6479 %{
6480 single_instruction;
6481 dst : S4(write);
6482 INS01 : ISS;
6483 NEON_FP : S4;
6484 %}
6485
6486 pipe_class fp_load_constant_d(vRegD dst)
6487 %{
6488 single_instruction;
6489 dst : S4(write);
6490 INS01 : ISS;
6491 NEON_FP : S4;
6492 %}
6493
6494 pipe_class vmul64(vecD dst, vecD src1, vecD src2)
6495 %{
6496 single_instruction;
6497 dst : S5(write);
6498 src1 : S1(read);
6499 src2 : S1(read);
6500 INS01 : ISS;
6501 NEON_FP : S5;
6502 %}
6503
6504 pipe_class vmul128(vecX dst, vecX src1, vecX src2)
6505 %{
6506 single_instruction;
6507 dst : S5(write);
6508 src1 : S1(read);
6509 src2 : S1(read);
6510 INS0 : ISS;
6511 NEON_FP : S5;
6512 %}
6513
6514 pipe_class vmla64(vecD dst, vecD src1, vecD src2)
6515 %{
6516 single_instruction;
6517 dst : S5(write);
6518 src1 : S1(read);
6519 src2 : S1(read);
6520 dst : S1(read);
6521 INS01 : ISS;
6522 NEON_FP : S5;
6523 %}
6524
6525 pipe_class vmla128(vecX dst, vecX src1, vecX src2)
6526 %{
6527 single_instruction;
6528 dst : S5(write);
6529 src1 : S1(read);
6530 src2 : S1(read);
6531 dst : S1(read);
6532 INS0 : ISS;
6533 NEON_FP : S5;
6534 %}
6535
6536 pipe_class vdop64(vecD dst, vecD src1, vecD src2)
6537 %{
6538 single_instruction;
6539 dst : S4(write);
6540 src1 : S2(read);
6541 src2 : S2(read);
6542 INS01 : ISS;
6543 NEON_FP : S4;
6544 %}
6545
6546 pipe_class vdop128(vecX dst, vecX src1, vecX src2)
6547 %{
6548 single_instruction;
6549 dst : S4(write);
6550 src1 : S2(read);
6551 src2 : S2(read);
6552 INS0 : ISS;
6553 NEON_FP : S4;
6554 %}
6555
6556 pipe_class vlogical64(vecD dst, vecD src1, vecD src2)
6557 %{
6558 single_instruction;
6559 dst : S3(write);
6560 src1 : S2(read);
6561 src2 : S2(read);
6562 INS01 : ISS;
6563 NEON_FP : S3;
6564 %}
6565
6566 pipe_class vlogical128(vecX dst, vecX src1, vecX src2)
6567 %{
6568 single_instruction;
6569 dst : S3(write);
6570 src1 : S2(read);
6571 src2 : S2(read);
6572 INS0 : ISS;
6573 NEON_FP : S3;
6574 %}
6575
6576 pipe_class vshift64(vecD dst, vecD src, vecX shift)
6577 %{
6578 single_instruction;
6579 dst : S3(write);
6580 src : S1(read);
6581 shift : S1(read);
6582 INS01 : ISS;
6583 NEON_FP : S3;
6584 %}
6585
6586 pipe_class vshift128(vecX dst, vecX src, vecX shift)
6587 %{
6588 single_instruction;
6589 dst : S3(write);
6590 src : S1(read);
6591 shift : S1(read);
6592 INS0 : ISS;
6593 NEON_FP : S3;
6594 %}
6595
6596 pipe_class vshift64_imm(vecD dst, vecD src, immI shift)
6597 %{
6598 single_instruction;
6599 dst : S3(write);
6600 src : S1(read);
6601 INS01 : ISS;
6602 NEON_FP : S3;
6603 %}
6604
6605 pipe_class vshift128_imm(vecX dst, vecX src, immI shift)
6606 %{
6607 single_instruction;
6608 dst : S3(write);
6609 src : S1(read);
6610 INS0 : ISS;
6611 NEON_FP : S3;
6612 %}
6613
6614 pipe_class vdop_fp64(vecD dst, vecD src1, vecD src2)
6615 %{
6616 single_instruction;
6617 dst : S5(write);
6618 src1 : S1(read);
6619 src2 : S1(read);
6620 INS01 : ISS;
6621 NEON_FP : S5;
6622 %}
6623
6624 pipe_class vdop_fp128(vecX dst, vecX src1, vecX src2)
6625 %{
6626 single_instruction;
6627 dst : S5(write);
6628 src1 : S1(read);
6629 src2 : S1(read);
6630 INS0 : ISS;
6631 NEON_FP : S5;
6632 %}
6633
6634 pipe_class vmuldiv_fp64(vecD dst, vecD src1, vecD src2)
6635 %{
6636 single_instruction;
6637 dst : S5(write);
6638 src1 : S1(read);
6639 src2 : S1(read);
6640 INS0 : ISS;
6641 NEON_FP : S5;
6642 %}
6643
6644 pipe_class vmuldiv_fp128(vecX dst, vecX src1, vecX src2)
6645 %{
6646 single_instruction;
6647 dst : S5(write);
6648 src1 : S1(read);
6649 src2 : S1(read);
6650 INS0 : ISS;
6651 NEON_FP : S5;
6652 %}
6653
6654 pipe_class vsqrt_fp128(vecX dst, vecX src)
6655 %{
6656 single_instruction;
6657 dst : S5(write);
6658 src : S1(read);
6659 INS0 : ISS;
6660 NEON_FP : S5;
6661 %}
6662
6663 pipe_class vunop_fp64(vecD dst, vecD src)
6664 %{
6665 single_instruction;
6666 dst : S5(write);
6667 src : S1(read);
6668 INS01 : ISS;
6669 NEON_FP : S5;
6670 %}
6671
6672 pipe_class vunop_fp128(vecX dst, vecX src)
6673 %{
6674 single_instruction;
6675 dst : S5(write);
6676 src : S1(read);
6677 INS0 : ISS;
6678 NEON_FP : S5;
6679 %}
6680
6681 pipe_class vdup_reg_reg64(vecD dst, iRegI src)
6682 %{
6683 single_instruction;
6684 dst : S3(write);
6685 src : S1(read);
6686 INS01 : ISS;
6687 NEON_FP : S3;
6688 %}
6689
6690 pipe_class vdup_reg_reg128(vecX dst, iRegI src)
6691 %{
6692 single_instruction;
6693 dst : S3(write);
6694 src : S1(read);
6695 INS01 : ISS;
6696 NEON_FP : S3;
6697 %}
6698
6699 pipe_class vdup_reg_freg64(vecD dst, vRegF src)
6700 %{
6701 single_instruction;
6702 dst : S3(write);
6703 src : S1(read);
6704 INS01 : ISS;
6705 NEON_FP : S3;
6706 %}
6707
6708 pipe_class vdup_reg_freg128(vecX dst, vRegF src)
6709 %{
6710 single_instruction;
6711 dst : S3(write);
6712 src : S1(read);
6713 INS01 : ISS;
6714 NEON_FP : S3;
6715 %}
6716
6717 pipe_class vdup_reg_dreg128(vecX dst, vRegD src)
6718 %{
6719 single_instruction;
6720 dst : S3(write);
6721 src : S1(read);
6722 INS01 : ISS;
6723 NEON_FP : S3;
6724 %}
6725
6726 pipe_class vmovi_reg_imm64(vecD dst)
6727 %{
6728 single_instruction;
6729 dst : S3(write);
6730 INS01 : ISS;
6731 NEON_FP : S3;
6732 %}
6733
6734 pipe_class vmovi_reg_imm128(vecX dst)
6735 %{
6736 single_instruction;
6737 dst : S3(write);
6738 INS0 : ISS;
6739 NEON_FP : S3;
6740 %}
6741
6742 pipe_class vload_reg_mem64(vecD dst, vmem8 mem)
6743 %{
6744 single_instruction;
6745 dst : S5(write);
6746 mem : ISS(read);
6747 INS01 : ISS;
6748 NEON_FP : S3;
6749 %}
6750
6751 pipe_class vload_reg_mem128(vecX dst, vmem16 mem)
6752 %{
6753 single_instruction;
6754 dst : S5(write);
6755 mem : ISS(read);
6756 INS01 : ISS;
6757 NEON_FP : S3;
6758 %}
6759
6760 pipe_class vstore_reg_mem64(vecD src, vmem8 mem)
6761 %{
6762 single_instruction;
6763 mem : ISS(read);
6764 src : S2(read);
6765 INS01 : ISS;
6766 NEON_FP : S3;
6767 %}
6768
6769 pipe_class vstore_reg_mem128(vecD src, vmem16 mem)
6770 %{
6771 single_instruction;
6772 mem : ISS(read);
6773 src : S2(read);
6774 INS01 : ISS;
6775 NEON_FP : S3;
6776 %}
6777
6778 //------- Integer ALU operations --------------------------
6779
6780 // Integer ALU reg-reg operation
6781 // Operands needed in EX1, result generated in EX2
6782 // Eg. ADD x0, x1, x2
6783 pipe_class ialu_reg_reg(iRegI dst, iRegI src1, iRegI src2)
6784 %{
6785 single_instruction;
6786 dst : EX2(write);
6787 src1 : EX1(read);
6788 src2 : EX1(read);
6789 INS01 : ISS; // Dual issue as instruction 0 or 1
6790 ALU : EX2;
6791 %}
6792
6793 // Integer ALU reg-reg operation with constant shift
6794 // Shifted register must be available in LATE_ISS instead of EX1
6795 // Eg. ADD x0, x1, x2, LSL #2
6796 pipe_class ialu_reg_reg_shift(iRegI dst, iRegI src1, iRegI src2, immI shift)
6797 %{
6798 single_instruction;
6799 dst : EX2(write);
6800 src1 : EX1(read);
6801 src2 : ISS(read);
6802 INS01 : ISS;
6803 ALU : EX2;
6804 %}
6805
6806 // Integer ALU reg operation with constant shift
6807 // Eg. LSL x0, x1, #shift
6808 pipe_class ialu_reg_shift(iRegI dst, iRegI src1)
6809 %{
6810 single_instruction;
6811 dst : EX2(write);
6812 src1 : ISS(read);
6813 INS01 : ISS;
6814 ALU : EX2;
6815 %}
6816
6817 // Integer ALU reg-reg operation with variable shift
6818 // Both operands must be available in LATE_ISS instead of EX1
6819 // Result is available in EX1 instead of EX2
6820 // Eg. LSLV x0, x1, x2
6821 pipe_class ialu_reg_reg_vshift(iRegI dst, iRegI src1, iRegI src2)
6822 %{
6823 single_instruction;
6824 dst : EX1(write);
6825 src1 : ISS(read);
6826 src2 : ISS(read);
6827 INS01 : ISS;
6828 ALU : EX1;
6829 %}
6830
6831 // Integer ALU reg-reg operation with extract
6832 // As for _vshift above, but result generated in EX2
6833 // Eg. EXTR x0, x1, x2, #N
6834 pipe_class ialu_reg_reg_extr(iRegI dst, iRegI src1, iRegI src2)
6835 %{
6836 single_instruction;
6837 dst : EX2(write);
6838 src1 : ISS(read);
6839 src2 : ISS(read);
6840 INS1 : ISS; // Can only dual issue as Instruction 1
6841 ALU : EX1;
6842 %}
6843
6844 // Integer ALU reg operation
6845 // Eg. NEG x0, x1
6846 pipe_class ialu_reg(iRegI dst, iRegI src)
6847 %{
6848 single_instruction;
6849 dst : EX2(write);
6850 src : EX1(read);
6851 INS01 : ISS;
6852 ALU : EX2;
6853 %}
6854
6855 // Integer ALU reg mmediate operation
6856 // Eg. ADD x0, x1, #N
6857 pipe_class ialu_reg_imm(iRegI dst, iRegI src1)
6858 %{
6859 single_instruction;
6860 dst : EX2(write);
6861 src1 : EX1(read);
6862 INS01 : ISS;
6863 ALU : EX2;
6864 %}
6865
6866 // Integer ALU immediate operation (no source operands)
6867 // Eg. MOV x0, #N
6868 pipe_class ialu_imm(iRegI dst)
6869 %{
6870 single_instruction;
6871 dst : EX1(write);
6872 INS01 : ISS;
6873 ALU : EX1;
6874 %}
6875
6876 //------- Compare operation -------------------------------
6877
6878 // Compare reg-reg
6879 // Eg. CMP x0, x1
6880 pipe_class icmp_reg_reg(rFlagsReg cr, iRegI op1, iRegI op2)
6881 %{
6882 single_instruction;
6883 // fixed_latency(16);
6884 cr : EX2(write);
6885 op1 : EX1(read);
6886 op2 : EX1(read);
6887 INS01 : ISS;
6888 ALU : EX2;
6889 %}
6890
6891 // Compare reg-reg
6892 // Eg. CMP x0, #N
6893 pipe_class icmp_reg_imm(rFlagsReg cr, iRegI op1)
6894 %{
6895 single_instruction;
6896 // fixed_latency(16);
6897 cr : EX2(write);
6898 op1 : EX1(read);
6899 INS01 : ISS;
6900 ALU : EX2;
6901 %}
6902
6903 //------- Conditional instructions ------------------------
6904
6905 // Conditional no operands
6906 // Eg. CSINC x0, zr, zr, <cond>
6907 pipe_class icond_none(iRegI dst, rFlagsReg cr)
6908 %{
6909 single_instruction;
6910 cr : EX1(read);
6911 dst : EX2(write);
6912 INS01 : ISS;
6913 ALU : EX2;
6914 %}
6915
6916 // Conditional 2 operand
6917 // EG. CSEL X0, X1, X2, <cond>
6918 pipe_class icond_reg_reg(iRegI dst, iRegI src1, iRegI src2, rFlagsReg cr)
6919 %{
6920 single_instruction;
6921 cr : EX1(read);
6922 src1 : EX1(read);
6923 src2 : EX1(read);
6924 dst : EX2(write);
6925 INS01 : ISS;
6926 ALU : EX2;
6927 %}
6928
6929 // Conditional 2 operand
6930 // EG. CSEL X0, X1, X2, <cond>
6931 pipe_class icond_reg(iRegI dst, iRegI src, rFlagsReg cr)
6932 %{
6933 single_instruction;
6934 cr : EX1(read);
6935 src : EX1(read);
6936 dst : EX2(write);
6937 INS01 : ISS;
6938 ALU : EX2;
6939 %}
6940
6941 //------- Multiply pipeline operations --------------------
6942
6943 // Multiply reg-reg
6944 // Eg. MUL w0, w1, w2
6945 pipe_class imul_reg_reg(iRegI dst, iRegI src1, iRegI src2)
6946 %{
6947 single_instruction;
6948 dst : WR(write);
6949 src1 : ISS(read);
6950 src2 : ISS(read);
6951 INS01 : ISS;
6952 MAC : WR;
6953 %}
6954
6955 // Multiply accumulate
6956 // Eg. MADD w0, w1, w2, w3
6957 pipe_class imac_reg_reg(iRegI dst, iRegI src1, iRegI src2, iRegI src3)
6958 %{
6959 single_instruction;
6960 dst : WR(write);
6961 src1 : ISS(read);
6962 src2 : ISS(read);
6963 src3 : ISS(read);
6964 INS01 : ISS;
6965 MAC : WR;
6966 %}
6967
6968 // Eg. MUL w0, w1, w2
6969 pipe_class lmul_reg_reg(iRegI dst, iRegI src1, iRegI src2)
6970 %{
6971 single_instruction;
6972 fixed_latency(3); // Maximum latency for 64 bit mul
6973 dst : WR(write);
6974 src1 : ISS(read);
6975 src2 : ISS(read);
6976 INS01 : ISS;
6977 MAC : WR;
6978 %}
6979
6980 // Multiply accumulate
6981 // Eg. MADD w0, w1, w2, w3
6982 pipe_class lmac_reg_reg(iRegI dst, iRegI src1, iRegI src2, iRegI src3)
6983 %{
6984 single_instruction;
6985 fixed_latency(3); // Maximum latency for 64 bit mul
6986 dst : WR(write);
6987 src1 : ISS(read);
6988 src2 : ISS(read);
6989 src3 : ISS(read);
6990 INS01 : ISS;
6991 MAC : WR;
6992 %}
6993
6994 //------- Divide pipeline operations --------------------
6995
6996 // Eg. SDIV w0, w1, w2
6997 pipe_class idiv_reg_reg(iRegI dst, iRegI src1, iRegI src2)
6998 %{
6999 single_instruction;
7000 fixed_latency(8); // Maximum latency for 32 bit divide
7001 dst : WR(write);
7002 src1 : ISS(read);
7003 src2 : ISS(read);
7004 INS0 : ISS; // Can only dual issue as instruction 0
7005 DIV : WR;
7006 %}
7007
7008 // Eg. SDIV x0, x1, x2
7009 pipe_class ldiv_reg_reg(iRegI dst, iRegI src1, iRegI src2)
7010 %{
7011 single_instruction;
7012 fixed_latency(16); // Maximum latency for 64 bit divide
7013 dst : WR(write);
7014 src1 : ISS(read);
7015 src2 : ISS(read);
7016 INS0 : ISS; // Can only dual issue as instruction 0
7017 DIV : WR;
7018 %}
7019
7020 //------- Load pipeline operations ------------------------
7021
7022 // Load - prefetch
7023 // Eg. PFRM <mem>
7024 pipe_class iload_prefetch(memory mem)
7025 %{
7026 single_instruction;
7027 mem : ISS(read);
7028 INS01 : ISS;
7029 LDST : WR;
7030 %}
7031
7032 // Load - reg, mem
7033 // Eg. LDR x0, <mem>
7034 pipe_class iload_reg_mem(iRegI dst, memory mem)
7035 %{
7036 single_instruction;
7037 dst : WR(write);
7038 mem : ISS(read);
7039 INS01 : ISS;
7040 LDST : WR;
7041 %}
7042
7043 // Load - reg, reg
7044 // Eg. LDR x0, [sp, x1]
7045 pipe_class iload_reg_reg(iRegI dst, iRegI src)
7046 %{
7047 single_instruction;
7048 dst : WR(write);
7049 src : ISS(read);
7050 INS01 : ISS;
7051 LDST : WR;
7052 %}
7053
7054 //------- Store pipeline operations -----------------------
7055
7056 // Store - zr, mem
7057 // Eg. STR zr, <mem>
7058 pipe_class istore_mem(memory mem)
7059 %{
7060 single_instruction;
7061 mem : ISS(read);
7062 INS01 : ISS;
7063 LDST : WR;
7064 %}
7065
7066 // Store - reg, mem
7067 // Eg. STR x0, <mem>
7068 pipe_class istore_reg_mem(iRegI src, memory mem)
7069 %{
7070 single_instruction;
7071 mem : ISS(read);
7072 src : EX2(read);
7073 INS01 : ISS;
7074 LDST : WR;
7075 %}
7076
7077 // Store - reg, reg
7078 // Eg. STR x0, [sp, x1]
7079 pipe_class istore_reg_reg(iRegI dst, iRegI src)
7080 %{
7081 single_instruction;
7082 dst : ISS(read);
7083 src : EX2(read);
7084 INS01 : ISS;
7085 LDST : WR;
7086 %}
7087
7088 //------- Store pipeline operations -----------------------
7089
7090 // Branch
7091 pipe_class pipe_branch()
7092 %{
7093 single_instruction;
7094 INS01 : ISS;
7095 BRANCH : EX1;
7096 %}
7097
7098 // Conditional branch
7099 pipe_class pipe_branch_cond(rFlagsReg cr)
7100 %{
7101 single_instruction;
7102 cr : EX1(read);
7103 INS01 : ISS;
7104 BRANCH : EX1;
7105 %}
7106
7107 // Compare & Branch
7108 // EG. CBZ/CBNZ
7109 pipe_class pipe_cmp_branch(iRegI op1)
7110 %{
7111 single_instruction;
7112 op1 : EX1(read);
7113 INS01 : ISS;
7114 BRANCH : EX1;
7115 %}
7116
7117 //------- Synchronisation operations ----------------------
7118
7119 // Any operation requiring serialization.
7120 // EG. DMB/Atomic Ops/Load Acquire/Str Release
7121 pipe_class pipe_serial()
7122 %{
7123 single_instruction;
7124 force_serialization;
7125 fixed_latency(16);
7126 INS01 : ISS(2); // Cannot dual issue with any other instruction
7127 LDST : WR;
7128 %}
7129
7130 // Generic big/slow expanded idiom - also serialized
7131 pipe_class pipe_slow()
7132 %{
7133 instruction_count(10);
7134 multiple_bundles;
7135 force_serialization;
7136 fixed_latency(16);
7137 INS01 : ISS(2); // Cannot dual issue with any other instruction
7138 LDST : WR;
7139 %}
7140
7141 // Empty pipeline class
7142 pipe_class pipe_class_empty()
7143 %{
7144 single_instruction;
7145 fixed_latency(0);
7146 %}
7147
7148 // Default pipeline class.
7149 pipe_class pipe_class_default()
7150 %{
7151 single_instruction;
7152 fixed_latency(2);
7153 %}
7154
7155 // Pipeline class for compares.
7156 pipe_class pipe_class_compare()
7157 %{
7158 single_instruction;
7159 fixed_latency(16);
7160 %}
7161
7162 // Pipeline class for memory operations.
7163 pipe_class pipe_class_memory()
7164 %{
7165 single_instruction;
7166 fixed_latency(16);
7167 %}
7168
7169 // Pipeline class for call.
7170 pipe_class pipe_class_call()
7171 %{
7172 single_instruction;
7173 fixed_latency(100);
7174 %}
7175
7176 // Define the class for the Nop node.
7177 define %{
7178 MachNop = pipe_class_empty;
7179 %}
7180
7181 %}
7182 //----------INSTRUCTIONS-------------------------------------------------------
7183 //
7184 // match -- States which machine-independent subtree may be replaced
7185 // by this instruction.
7186 // ins_cost -- The estimated cost of this instruction is used by instruction
7187 // selection to identify a minimum cost tree of machine
7188 // instructions that matches a tree of machine-independent
7189 // instructions.
7190 // format -- A string providing the disassembly for this instruction.
7191 // The value of an instruction's operand may be inserted
7192 // by referring to it with a '$' prefix.
7193 // opcode -- Three instruction opcodes may be provided. These are referred
7194 // to within an encode class as $primary, $secondary, and $tertiary
7195 // rrspectively. The primary opcode is commonly used to
7196 // indicate the type of machine instruction, while secondary
7197 // and tertiary are often used for prefix options or addressing
7198 // modes.
7199 // ins_encode -- A list of encode classes with parameters. The encode class
7200 // name must have been defined in an 'enc_class' specification
7201 // in the encode section of the architecture description.
7202
7203 // ============================================================================
7204 // Memory (Load/Store) Instructions
7205
7206 // Load Instructions
7207
7208 // Load Byte (8 bit signed)
7209 instruct loadB(iRegINoSp dst, memory1 mem)
7210 %{
7211 match(Set dst (LoadB mem));
7212 predicate(!needs_acquiring_load(n));
7213
7214 ins_cost(4 * INSN_COST);
7215 format %{ "ldrsbw $dst, $mem\t# byte" %}
7216
7217 ins_encode(aarch64_enc_ldrsbw(dst, mem));
7218
7219 ins_pipe(iload_reg_mem);
7220 %}
7221
7222 // Load Byte (8 bit signed) into long
7223 instruct loadB2L(iRegLNoSp dst, memory1 mem)
7224 %{
7225 match(Set dst (ConvI2L (LoadB mem)));
7226 predicate(!needs_acquiring_load(n->in(1)));
7227
7228 ins_cost(4 * INSN_COST);
7229 format %{ "ldrsb $dst, $mem\t# byte" %}
7230
7231 ins_encode(aarch64_enc_ldrsb(dst, mem));
7232
7233 ins_pipe(iload_reg_mem);
7234 %}
7235
7236 // Load Byte (8 bit unsigned)
7237 instruct loadUB(iRegINoSp dst, memory1 mem)
7238 %{
7239 match(Set dst (LoadUB mem));
7240 predicate(!needs_acquiring_load(n));
7241
7242 ins_cost(4 * INSN_COST);
7243 format %{ "ldrbw $dst, $mem\t# byte" %}
7244
7245 ins_encode(aarch64_enc_ldrb(dst, mem));
7246
7247 ins_pipe(iload_reg_mem);
7248 %}
7249
7250 // Load Byte (8 bit unsigned) into long
7251 instruct loadUB2L(iRegLNoSp dst, memory1 mem)
7252 %{
7253 match(Set dst (ConvI2L (LoadUB mem)));
7254 predicate(!needs_acquiring_load(n->in(1)));
7255
7256 ins_cost(4 * INSN_COST);
7257 format %{ "ldrb $dst, $mem\t# byte" %}
7258
7259 ins_encode(aarch64_enc_ldrb(dst, mem));
7260
7261 ins_pipe(iload_reg_mem);
7262 %}
7263
7264 // Load Short (16 bit signed)
7265 instruct loadS(iRegINoSp dst, memory2 mem)
7266 %{
7267 match(Set dst (LoadS mem));
7268 predicate(!needs_acquiring_load(n));
7269
7270 ins_cost(4 * INSN_COST);
7271 format %{ "ldrshw $dst, $mem\t# short" %}
7272
7273 ins_encode(aarch64_enc_ldrshw(dst, mem));
7274
7275 ins_pipe(iload_reg_mem);
7276 %}
7277
7278 // Load Short (16 bit signed) into long
7279 instruct loadS2L(iRegLNoSp dst, memory2 mem)
7280 %{
7281 match(Set dst (ConvI2L (LoadS mem)));
7282 predicate(!needs_acquiring_load(n->in(1)));
7283
7284 ins_cost(4 * INSN_COST);
7285 format %{ "ldrsh $dst, $mem\t# short" %}
7286
7287 ins_encode(aarch64_enc_ldrsh(dst, mem));
7288
7289 ins_pipe(iload_reg_mem);
7290 %}
7291
7292 // Load Char (16 bit unsigned)
7293 instruct loadUS(iRegINoSp dst, memory2 mem)
7294 %{
7295 match(Set dst (LoadUS mem));
7296 predicate(!needs_acquiring_load(n));
7297
7298 ins_cost(4 * INSN_COST);
7299 format %{ "ldrh $dst, $mem\t# short" %}
7300
7301 ins_encode(aarch64_enc_ldrh(dst, mem));
7302
7303 ins_pipe(iload_reg_mem);
7304 %}
7305
7306 // Load Short/Char (16 bit unsigned) into long
7307 instruct loadUS2L(iRegLNoSp dst, memory2 mem)
7308 %{
7309 match(Set dst (ConvI2L (LoadUS mem)));
7310 predicate(!needs_acquiring_load(n->in(1)));
7311
7312 ins_cost(4 * INSN_COST);
7313 format %{ "ldrh $dst, $mem\t# short" %}
7314
7315 ins_encode(aarch64_enc_ldrh(dst, mem));
7316
7317 ins_pipe(iload_reg_mem);
7318 %}
7319
7320 // Load Integer (32 bit signed)
7321 instruct loadI(iRegINoSp dst, memory4 mem)
7322 %{
7323 match(Set dst (LoadI mem));
7324 predicate(!needs_acquiring_load(n));
7325
7326 ins_cost(4 * INSN_COST);
7327 format %{ "ldrw $dst, $mem\t# int" %}
7328
7329 ins_encode(aarch64_enc_ldrw(dst, mem));
7330
7331 ins_pipe(iload_reg_mem);
7332 %}
7333
7334 // Load Integer (32 bit signed) into long
7335 instruct loadI2L(iRegLNoSp dst, memory4 mem)
7336 %{
7337 match(Set dst (ConvI2L (LoadI mem)));
7338 predicate(!needs_acquiring_load(n->in(1)));
7339
7340 ins_cost(4 * INSN_COST);
7341 format %{ "ldrsw $dst, $mem\t# int" %}
7342
7343 ins_encode(aarch64_enc_ldrsw(dst, mem));
7344
7345 ins_pipe(iload_reg_mem);
7346 %}
7347
7348 // Load Integer (32 bit unsigned) into long
7349 instruct loadUI2L(iRegLNoSp dst, memory4 mem, immL_32bits mask)
7350 %{
7351 match(Set dst (AndL (ConvI2L (LoadI mem)) mask));
7352 predicate(!needs_acquiring_load(n->in(1)->in(1)->as_Load()));
7353
7354 ins_cost(4 * INSN_COST);
7355 format %{ "ldrw $dst, $mem\t# int" %}
7356
7357 ins_encode(aarch64_enc_ldrw(dst, mem));
7358
7359 ins_pipe(iload_reg_mem);
7360 %}
7361
7362 // Load Long (64 bit signed)
7363 instruct loadL(iRegLNoSp dst, memory8 mem)
7364 %{
7365 match(Set dst (LoadL mem));
7366 predicate(!needs_acquiring_load(n));
7367
7368 ins_cost(4 * INSN_COST);
7369 format %{ "ldr $dst, $mem\t# int" %}
7370
7371 ins_encode(aarch64_enc_ldr(dst, mem));
7372
7373 ins_pipe(iload_reg_mem);
7374 %}
7375
7376 // Load Range
7377 instruct loadRange(iRegINoSp dst, memory4 mem)
7378 %{
7379 match(Set dst (LoadRange mem));
7380
7381 ins_cost(4 * INSN_COST);
7382 format %{ "ldrw $dst, $mem\t# range" %}
7383
7384 ins_encode(aarch64_enc_ldrw(dst, mem));
7385
7386 ins_pipe(iload_reg_mem);
7387 %}
7388
7389 // Load Pointer
7390 instruct loadP(iRegPNoSp dst, memory8 mem)
7391 %{
7392 match(Set dst (LoadP mem));
7393 predicate(!needs_acquiring_load(n) && (n->as_Load()->barrier_data() == 0));
7394
7395 ins_cost(4 * INSN_COST);
7396 format %{ "ldr $dst, $mem\t# ptr" %}
7397
7398 ins_encode(aarch64_enc_ldr(dst, mem));
7399
7400 ins_pipe(iload_reg_mem);
7401 %}
7402
7403 // Load Compressed Pointer
7404 instruct loadN(iRegNNoSp dst, memory4 mem)
7405 %{
7406 match(Set dst (LoadN mem));
7407 predicate(!needs_acquiring_load(n));
7408
7409 ins_cost(4 * INSN_COST);
7410 format %{ "ldrw $dst, $mem\t# compressed ptr" %}
7411
7412 ins_encode(aarch64_enc_ldrw(dst, mem));
7413
7414 ins_pipe(iload_reg_mem);
7415 %}
7416
7417 // Load Klass Pointer
7418 instruct loadKlass(iRegPNoSp dst, memory8 mem)
7419 %{
7420 match(Set dst (LoadKlass mem));
7421 predicate(!needs_acquiring_load(n));
7422
7423 ins_cost(4 * INSN_COST);
7424 format %{ "ldr $dst, $mem\t# class" %}
7425
7426 ins_encode(aarch64_enc_ldr(dst, mem));
7427
7428 ins_pipe(iload_reg_mem);
7429 %}
7430
7431 // Load Narrow Klass Pointer
7432 instruct loadNKlass(iRegNNoSp dst, memory4 mem)
7433 %{
7434 match(Set dst (LoadNKlass mem));
7435 predicate(!needs_acquiring_load(n));
7436
7437 ins_cost(4 * INSN_COST);
7438 format %{ "ldrw $dst, $mem\t# compressed class ptr" %}
7439
7440 ins_encode(aarch64_enc_ldrw(dst, mem));
7441
7442 ins_pipe(iload_reg_mem);
7443 %}
7444
7445 // Load Float
7446 instruct loadF(vRegF dst, memory4 mem)
7447 %{
7448 match(Set dst (LoadF mem));
7449 predicate(!needs_acquiring_load(n));
7450
7451 ins_cost(4 * INSN_COST);
7452 format %{ "ldrs $dst, $mem\t# float" %}
7453
7454 ins_encode( aarch64_enc_ldrs(dst, mem) );
7455
7456 ins_pipe(pipe_class_memory);
7457 %}
7458
7459 // Load Double
7460 instruct loadD(vRegD dst, memory8 mem)
7461 %{
7462 match(Set dst (LoadD mem));
7463 predicate(!needs_acquiring_load(n));
7464
7465 ins_cost(4 * INSN_COST);
7466 format %{ "ldrd $dst, $mem\t# double" %}
7467
7468 ins_encode( aarch64_enc_ldrd(dst, mem) );
7469
7470 ins_pipe(pipe_class_memory);
7471 %}
7472
7473
7474 // Load Int Constant
7475 instruct loadConI(iRegINoSp dst, immI src)
7476 %{
7477 match(Set dst src);
7478
7479 ins_cost(INSN_COST);
7480 format %{ "mov $dst, $src\t# int" %}
7481
7482 ins_encode( aarch64_enc_movw_imm(dst, src) );
7483
7484 ins_pipe(ialu_imm);
7485 %}
7486
7487 // Load Long Constant
7488 instruct loadConL(iRegLNoSp dst, immL src)
7489 %{
7490 match(Set dst src);
7491
7492 ins_cost(INSN_COST);
7493 format %{ "mov $dst, $src\t# long" %}
7494
7495 ins_encode( aarch64_enc_mov_imm(dst, src) );
7496
7497 ins_pipe(ialu_imm);
7498 %}
7499
7500 // Load Pointer Constant
7501
7502 instruct loadConP(iRegPNoSp dst, immP con)
7503 %{
7504 match(Set dst con);
7505
7506 ins_cost(INSN_COST * 4);
7507 format %{
7508 "mov $dst, $con\t# ptr\n\t"
7509 %}
7510
7511 ins_encode(aarch64_enc_mov_p(dst, con));
7512
7513 ins_pipe(ialu_imm);
7514 %}
7515
7516 // Load Null Pointer Constant
7517
7518 instruct loadConP0(iRegPNoSp dst, immP0 con)
7519 %{
7520 match(Set dst con);
7521
7522 ins_cost(INSN_COST);
7523 format %{ "mov $dst, $con\t# NULL ptr" %}
7524
7525 ins_encode(aarch64_enc_mov_p0(dst, con));
7526
7527 ins_pipe(ialu_imm);
7528 %}
7529
7530 // Load Pointer Constant One
7531
7532 instruct loadConP1(iRegPNoSp dst, immP_1 con)
7533 %{
7534 match(Set dst con);
7535
7536 ins_cost(INSN_COST);
7537 format %{ "mov $dst, $con\t# NULL ptr" %}
7538
7539 ins_encode(aarch64_enc_mov_p1(dst, con));
7540
7541 ins_pipe(ialu_imm);
7542 %}
7543
7544 // Load Byte Map Base Constant
7545
7546 instruct loadByteMapBase(iRegPNoSp dst, immByteMapBase con)
7547 %{
7548 match(Set dst con);
7549
7550 ins_cost(INSN_COST);
7551 format %{ "adr $dst, $con\t# Byte Map Base" %}
7552
7553 ins_encode(aarch64_enc_mov_byte_map_base(dst, con));
7554
7555 ins_pipe(ialu_imm);
7556 %}
7557
7558 // Load Narrow Pointer Constant
7559
7560 instruct loadConN(iRegNNoSp dst, immN con)
7561 %{
7562 match(Set dst con);
7563
7564 ins_cost(INSN_COST * 4);
7565 format %{ "mov $dst, $con\t# compressed ptr" %}
7566
7567 ins_encode(aarch64_enc_mov_n(dst, con));
7568
7569 ins_pipe(ialu_imm);
7570 %}
7571
7572 // Load Narrow Null Pointer Constant
7573
7574 instruct loadConN0(iRegNNoSp dst, immN0 con)
7575 %{
7576 match(Set dst con);
7577
7578 ins_cost(INSN_COST);
7579 format %{ "mov $dst, $con\t# compressed NULL ptr" %}
7580
7581 ins_encode(aarch64_enc_mov_n0(dst, con));
7582
7583 ins_pipe(ialu_imm);
7584 %}
7585
7586 // Load Narrow Klass Constant
7587
7588 instruct loadConNKlass(iRegNNoSp dst, immNKlass con)
7589 %{
7590 match(Set dst con);
7591
7592 ins_cost(INSN_COST);
7593 format %{ "mov $dst, $con\t# compressed klass ptr" %}
7594
7595 ins_encode(aarch64_enc_mov_nk(dst, con));
7596
7597 ins_pipe(ialu_imm);
7598 %}
7599
7600 // Load Packed Float Constant
7601
7602 instruct loadConF_packed(vRegF dst, immFPacked con) %{
7603 match(Set dst con);
7604 ins_cost(INSN_COST * 4);
7605 format %{ "fmovs $dst, $con"%}
7606 ins_encode %{
7607 __ fmovs(as_FloatRegister($dst$$reg), (double)$con$$constant);
7608 %}
7609
7610 ins_pipe(fp_imm_s);
7611 %}
7612
7613 // Load Float Constant
7614
7615 instruct loadConF(vRegF dst, immF con) %{
7616 match(Set dst con);
7617
7618 ins_cost(INSN_COST * 4);
7619
7620 format %{
7621 "ldrs $dst, [$constantaddress]\t# load from constant table: float=$con\n\t"
7622 %}
7623
7624 ins_encode %{
7625 __ ldrs(as_FloatRegister($dst$$reg), $constantaddress($con));
7626 %}
7627
7628 ins_pipe(fp_load_constant_s);
7629 %}
7630
7631 // Load Packed Double Constant
7632
7633 instruct loadConD_packed(vRegD dst, immDPacked con) %{
7634 match(Set dst con);
7635 ins_cost(INSN_COST);
7636 format %{ "fmovd $dst, $con"%}
7637 ins_encode %{
7638 __ fmovd(as_FloatRegister($dst$$reg), $con$$constant);
7639 %}
7640
7641 ins_pipe(fp_imm_d);
7642 %}
7643
7644 // Load Double Constant
7645
7646 instruct loadConD(vRegD dst, immD con) %{
7647 match(Set dst con);
7648
7649 ins_cost(INSN_COST * 5);
7650 format %{
7651 "ldrd $dst, [$constantaddress]\t# load from constant table: float=$con\n\t"
7652 %}
7653
7654 ins_encode %{
7655 __ ldrd(as_FloatRegister($dst$$reg), $constantaddress($con));
7656 %}
7657
7658 ins_pipe(fp_load_constant_d);
7659 %}
7660
7661 // Store Instructions
7662
7663 // Store CMS card-mark Immediate
7664 instruct storeimmCM0(immI0 zero, memory1 mem)
7665 %{
7666 match(Set mem (StoreCM mem zero));
7667
7668 ins_cost(INSN_COST);
7669 format %{ "storestore (elided)\n\t"
7670 "strb zr, $mem\t# byte" %}
7671
7672 ins_encode(aarch64_enc_strb0(mem));
7673
7674 ins_pipe(istore_mem);
7675 %}
7676
7677 // Store CMS card-mark Immediate with intervening StoreStore
7678 // needed when using CMS with no conditional card marking
7679 instruct storeimmCM0_ordered(immI0 zero, memory1 mem)
7680 %{
7681 match(Set mem (StoreCM mem zero));
7682
7683 ins_cost(INSN_COST * 2);
7684 format %{ "storestore\n\t"
7685 "dmb ishst"
7686 "\n\tstrb zr, $mem\t# byte" %}
7687
7688 ins_encode(aarch64_enc_strb0_ordered(mem));
7689
7690 ins_pipe(istore_mem);
7691 %}
7692
7693 // Store Byte
7694 instruct storeB(iRegIorL2I src, memory1 mem)
7695 %{
7696 match(Set mem (StoreB mem src));
7697 predicate(!needs_releasing_store(n));
7698
7699 ins_cost(INSN_COST);
7700 format %{ "strb $src, $mem\t# byte" %}
7701
7702 ins_encode(aarch64_enc_strb(src, mem));
7703
7704 ins_pipe(istore_reg_mem);
7705 %}
7706
7707
7708 instruct storeimmB0(immI0 zero, memory1 mem)
7709 %{
7710 match(Set mem (StoreB mem zero));
7711 predicate(!needs_releasing_store(n));
7712
7713 ins_cost(INSN_COST);
7714 format %{ "strb rscractch2, $mem\t# byte" %}
7715
7716 ins_encode(aarch64_enc_strb0(mem));
7717
7718 ins_pipe(istore_mem);
7719 %}
7720
7721 // Store Char/Short
7722 instruct storeC(iRegIorL2I src, memory2 mem)
7723 %{
7724 match(Set mem (StoreC mem src));
7725 predicate(!needs_releasing_store(n));
7726
7727 ins_cost(INSN_COST);
7728 format %{ "strh $src, $mem\t# short" %}
7729
7730 ins_encode(aarch64_enc_strh(src, mem));
7731
7732 ins_pipe(istore_reg_mem);
7733 %}
7734
7735 instruct storeimmC0(immI0 zero, memory2 mem)
7736 %{
7737 match(Set mem (StoreC mem zero));
7738 predicate(!needs_releasing_store(n));
7739
7740 ins_cost(INSN_COST);
7741 format %{ "strh zr, $mem\t# short" %}
7742
7743 ins_encode(aarch64_enc_strh0(mem));
7744
7745 ins_pipe(istore_mem);
7746 %}
7747
7748 // Store Integer
7749
7750 instruct storeI(iRegIorL2I src, memory4 mem)
7751 %{
7752 match(Set mem(StoreI mem src));
7753 predicate(!needs_releasing_store(n));
7754
7755 ins_cost(INSN_COST);
7756 format %{ "strw $src, $mem\t# int" %}
7757
7758 ins_encode(aarch64_enc_strw(src, mem));
7759
7760 ins_pipe(istore_reg_mem);
7761 %}
7762
7763 instruct storeimmI0(immI0 zero, memory4 mem)
7764 %{
7765 match(Set mem(StoreI mem zero));
7766 predicate(!needs_releasing_store(n));
7767
7768 ins_cost(INSN_COST);
7769 format %{ "strw zr, $mem\t# int" %}
7770
7771 ins_encode(aarch64_enc_strw0(mem));
7772
7773 ins_pipe(istore_mem);
7774 %}
7775
7776 // Store Long (64 bit signed)
7777 instruct storeL(iRegL src, memory8 mem)
7778 %{
7779 match(Set mem (StoreL mem src));
7780 predicate(!needs_releasing_store(n));
7781
7782 ins_cost(INSN_COST);
7783 format %{ "str $src, $mem\t# int" %}
7784
7785 ins_encode(aarch64_enc_str(src, mem));
7786
7787 ins_pipe(istore_reg_mem);
7788 %}
7789
7790 // Store Long (64 bit signed)
7791 instruct storeimmL0(immL0 zero, memory8 mem)
7792 %{
7793 match(Set mem (StoreL mem zero));
7794 predicate(!needs_releasing_store(n));
7795
7796 ins_cost(INSN_COST);
7797 format %{ "str zr, $mem\t# int" %}
7798
7799 ins_encode(aarch64_enc_str0(mem));
7800
7801 ins_pipe(istore_mem);
7802 %}
7803
7804 // Store Pointer
7805 instruct storeP(iRegP src, memory8 mem)
7806 %{
7807 match(Set mem (StoreP mem src));
7808 predicate(!needs_releasing_store(n));
7809
7810 ins_cost(INSN_COST);
7811 format %{ "str $src, $mem\t# ptr" %}
7812
7813 ins_encode(aarch64_enc_str(src, mem));
7814
7815 ins_pipe(istore_reg_mem);
7816 %}
7817
7818 // Store Pointer
7819 instruct storeimmP0(immP0 zero, memory8 mem)
7820 %{
7821 match(Set mem (StoreP mem zero));
7822 predicate(!needs_releasing_store(n));
7823
7824 ins_cost(INSN_COST);
7825 format %{ "str zr, $mem\t# ptr" %}
7826
7827 ins_encode(aarch64_enc_str0(mem));
7828
7829 ins_pipe(istore_mem);
7830 %}
7831
7832 // Store Compressed Pointer
7833 instruct storeN(iRegN src, memory4 mem)
7834 %{
7835 match(Set mem (StoreN mem src));
7836 predicate(!needs_releasing_store(n));
7837
7838 ins_cost(INSN_COST);
7839 format %{ "strw $src, $mem\t# compressed ptr" %}
7840
7841 ins_encode(aarch64_enc_strw(src, mem));
7842
7843 ins_pipe(istore_reg_mem);
7844 %}
7845
7846 instruct storeImmN0(immN0 zero, memory4 mem)
7847 %{
7848 match(Set mem (StoreN mem zero));
7849 predicate(!needs_releasing_store(n));
7850
7851 ins_cost(INSN_COST);
7852 format %{ "strw zr, $mem\t# compressed ptr" %}
7853
7854 ins_encode(aarch64_enc_strw0(mem));
7855
7856 ins_pipe(istore_mem);
7857 %}
7858
7859 // Store Float
7860 instruct storeF(vRegF src, memory4 mem)
7861 %{
7862 match(Set mem (StoreF mem src));
7863 predicate(!needs_releasing_store(n));
7864
7865 ins_cost(INSN_COST);
7866 format %{ "strs $src, $mem\t# float" %}
7867
7868 ins_encode( aarch64_enc_strs(src, mem) );
7869
7870 ins_pipe(pipe_class_memory);
7871 %}
7872
7873 // TODO
7874 // implement storeImmF0 and storeFImmPacked
7875
7876 // Store Double
7877 instruct storeD(vRegD src, memory8 mem)
7878 %{
7879 match(Set mem (StoreD mem src));
7880 predicate(!needs_releasing_store(n));
7881
7882 ins_cost(INSN_COST);
7883 format %{ "strd $src, $mem\t# double" %}
7884
7885 ins_encode( aarch64_enc_strd(src, mem) );
7886
7887 ins_pipe(pipe_class_memory);
7888 %}
7889
7890 // Store Compressed Klass Pointer
7891 instruct storeNKlass(iRegN src, memory4 mem)
7892 %{
7893 predicate(!needs_releasing_store(n));
7894 match(Set mem (StoreNKlass mem src));
7895
7896 ins_cost(INSN_COST);
7897 format %{ "strw $src, $mem\t# compressed klass ptr" %}
7898
7899 ins_encode(aarch64_enc_strw(src, mem));
7900
7901 ins_pipe(istore_reg_mem);
7902 %}
7903
7904 // TODO
7905 // implement storeImmD0 and storeDImmPacked
7906
7907 // prefetch instructions
7908 // Must be safe to execute with invalid address (cannot fault).
7909
7910 instruct prefetchalloc( memory8 mem ) %{
7911 match(PrefetchAllocation mem);
7912
7913 ins_cost(INSN_COST);
7914 format %{ "prfm $mem, PSTL1KEEP\t# Prefetch into level 1 cache write keep" %}
7915
7916 ins_encode( aarch64_enc_prefetchw(mem) );
7917
7918 ins_pipe(iload_prefetch);
7919 %}
7920
7921 // ---------------- volatile loads and stores ----------------
7922
7923 // Load Byte (8 bit signed)
7924 instruct loadB_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
7925 %{
7926 match(Set dst (LoadB mem));
7927
7928 ins_cost(VOLATILE_REF_COST);
7929 format %{ "ldarsb $dst, $mem\t# byte" %}
7930
7931 ins_encode(aarch64_enc_ldarsb(dst, mem));
7932
7933 ins_pipe(pipe_serial);
7934 %}
7935
7936 // Load Byte (8 bit signed) into long
7937 instruct loadB2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
7938 %{
7939 match(Set dst (ConvI2L (LoadB mem)));
7940
7941 ins_cost(VOLATILE_REF_COST);
7942 format %{ "ldarsb $dst, $mem\t# byte" %}
7943
7944 ins_encode(aarch64_enc_ldarsb(dst, mem));
7945
7946 ins_pipe(pipe_serial);
7947 %}
7948
7949 // Load Byte (8 bit unsigned)
7950 instruct loadUB_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
7951 %{
7952 match(Set dst (LoadUB mem));
7953
7954 ins_cost(VOLATILE_REF_COST);
7955 format %{ "ldarb $dst, $mem\t# byte" %}
7956
7957 ins_encode(aarch64_enc_ldarb(dst, mem));
7958
7959 ins_pipe(pipe_serial);
7960 %}
7961
7962 // Load Byte (8 bit unsigned) into long
7963 instruct loadUB2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
7964 %{
7965 match(Set dst (ConvI2L (LoadUB mem)));
7966
7967 ins_cost(VOLATILE_REF_COST);
7968 format %{ "ldarb $dst, $mem\t# byte" %}
7969
7970 ins_encode(aarch64_enc_ldarb(dst, mem));
7971
7972 ins_pipe(pipe_serial);
7973 %}
7974
7975 // Load Short (16 bit signed)
7976 instruct loadS_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
7977 %{
7978 match(Set dst (LoadS mem));
7979
7980 ins_cost(VOLATILE_REF_COST);
7981 format %{ "ldarshw $dst, $mem\t# short" %}
7982
7983 ins_encode(aarch64_enc_ldarshw(dst, mem));
7984
7985 ins_pipe(pipe_serial);
7986 %}
7987
7988 instruct loadUS_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
7989 %{
7990 match(Set dst (LoadUS mem));
7991
7992 ins_cost(VOLATILE_REF_COST);
7993 format %{ "ldarhw $dst, $mem\t# short" %}
7994
7995 ins_encode(aarch64_enc_ldarhw(dst, mem));
7996
7997 ins_pipe(pipe_serial);
7998 %}
7999
8000 // Load Short/Char (16 bit unsigned) into long
8001 instruct loadUS2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
8002 %{
8003 match(Set dst (ConvI2L (LoadUS mem)));
8004
8005 ins_cost(VOLATILE_REF_COST);
8006 format %{ "ldarh $dst, $mem\t# short" %}
8007
8008 ins_encode(aarch64_enc_ldarh(dst, mem));
8009
8010 ins_pipe(pipe_serial);
8011 %}
8012
8013 // Load Short/Char (16 bit signed) into long
8014 instruct loadS2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
8015 %{
8016 match(Set dst (ConvI2L (LoadS mem)));
8017
8018 ins_cost(VOLATILE_REF_COST);
8019 format %{ "ldarh $dst, $mem\t# short" %}
8020
8021 ins_encode(aarch64_enc_ldarsh(dst, mem));
8022
8023 ins_pipe(pipe_serial);
8024 %}
8025
8026 // Load Integer (32 bit signed)
8027 instruct loadI_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
8028 %{
8029 match(Set dst (LoadI mem));
8030
8031 ins_cost(VOLATILE_REF_COST);
8032 format %{ "ldarw $dst, $mem\t# int" %}
8033
8034 ins_encode(aarch64_enc_ldarw(dst, mem));
8035
8036 ins_pipe(pipe_serial);
8037 %}
8038
8039 // Load Integer (32 bit unsigned) into long
8040 instruct loadUI2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem, immL_32bits mask)
8041 %{
8042 match(Set dst (AndL (ConvI2L (LoadI mem)) mask));
8043
8044 ins_cost(VOLATILE_REF_COST);
8045 format %{ "ldarw $dst, $mem\t# int" %}
8046
8047 ins_encode(aarch64_enc_ldarw(dst, mem));
8048
8049 ins_pipe(pipe_serial);
8050 %}
8051
8052 // Load Long (64 bit signed)
8053 instruct loadL_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
8054 %{
8055 match(Set dst (LoadL mem));
8056
8057 ins_cost(VOLATILE_REF_COST);
8058 format %{ "ldar $dst, $mem\t# int" %}
8059
8060 ins_encode(aarch64_enc_ldar(dst, mem));
8061
8062 ins_pipe(pipe_serial);
8063 %}
8064
8065 // Load Pointer
8066 instruct loadP_volatile(iRegPNoSp dst, /* sync_memory*/indirect mem)
8067 %{
8068 match(Set dst (LoadP mem));
8069 predicate(n->as_Load()->barrier_data() == 0);
8070
8071 ins_cost(VOLATILE_REF_COST);
8072 format %{ "ldar $dst, $mem\t# ptr" %}
8073
8074 ins_encode(aarch64_enc_ldar(dst, mem));
8075
8076 ins_pipe(pipe_serial);
8077 %}
8078
8079 // Load Compressed Pointer
8080 instruct loadN_volatile(iRegNNoSp dst, /* sync_memory*/indirect mem)
8081 %{
8082 match(Set dst (LoadN mem));
8083
8084 ins_cost(VOLATILE_REF_COST);
8085 format %{ "ldarw $dst, $mem\t# compressed ptr" %}
8086
8087 ins_encode(aarch64_enc_ldarw(dst, mem));
8088
8089 ins_pipe(pipe_serial);
8090 %}
8091
8092 // Load Float
8093 instruct loadF_volatile(vRegF dst, /* sync_memory*/indirect mem)
8094 %{
8095 match(Set dst (LoadF mem));
8096
8097 ins_cost(VOLATILE_REF_COST);
8098 format %{ "ldars $dst, $mem\t# float" %}
8099
8100 ins_encode( aarch64_enc_fldars(dst, mem) );
8101
8102 ins_pipe(pipe_serial);
8103 %}
8104
8105 // Load Double
8106 instruct loadD_volatile(vRegD dst, /* sync_memory*/indirect mem)
8107 %{
8108 match(Set dst (LoadD mem));
8109
8110 ins_cost(VOLATILE_REF_COST);
8111 format %{ "ldard $dst, $mem\t# double" %}
8112
8113 ins_encode( aarch64_enc_fldard(dst, mem) );
8114
8115 ins_pipe(pipe_serial);
8116 %}
8117
8118 // Store Byte
8119 instruct storeB_volatile(iRegIorL2I src, /* sync_memory*/indirect mem)
8120 %{
8121 match(Set mem (StoreB mem src));
8122
8123 ins_cost(VOLATILE_REF_COST);
8124 format %{ "stlrb $src, $mem\t# byte" %}
8125
8126 ins_encode(aarch64_enc_stlrb(src, mem));
8127
8128 ins_pipe(pipe_class_memory);
8129 %}
8130
8131 // Store Char/Short
8132 instruct storeC_volatile(iRegIorL2I src, /* sync_memory*/indirect mem)
8133 %{
8134 match(Set mem (StoreC mem src));
8135
8136 ins_cost(VOLATILE_REF_COST);
8137 format %{ "stlrh $src, $mem\t# short" %}
8138
8139 ins_encode(aarch64_enc_stlrh(src, mem));
8140
8141 ins_pipe(pipe_class_memory);
8142 %}
8143
8144 // Store Integer
8145
8146 instruct storeI_volatile(iRegIorL2I src, /* sync_memory*/indirect mem)
8147 %{
8148 match(Set mem(StoreI mem src));
8149
8150 ins_cost(VOLATILE_REF_COST);
8151 format %{ "stlrw $src, $mem\t# int" %}
8152
8153 ins_encode(aarch64_enc_stlrw(src, mem));
8154
8155 ins_pipe(pipe_class_memory);
8156 %}
8157
8158 // Store Long (64 bit signed)
8159 instruct storeL_volatile(iRegL src, /* sync_memory*/indirect mem)
8160 %{
8161 match(Set mem (StoreL mem src));
8162
8163 ins_cost(VOLATILE_REF_COST);
8164 format %{ "stlr $src, $mem\t# int" %}
8165
8166 ins_encode(aarch64_enc_stlr(src, mem));
8167
8168 ins_pipe(pipe_class_memory);
8169 %}
8170
8171 // Store Pointer
8172 instruct storeP_volatile(iRegP src, /* sync_memory*/indirect mem)
8173 %{
8174 match(Set mem (StoreP mem src));
8175
8176 ins_cost(VOLATILE_REF_COST);
8177 format %{ "stlr $src, $mem\t# ptr" %}
8178
8179 ins_encode(aarch64_enc_stlr(src, mem));
8180
8181 ins_pipe(pipe_class_memory);
8182 %}
8183
8184 // Store Compressed Pointer
8185 instruct storeN_volatile(iRegN src, /* sync_memory*/indirect mem)
8186 %{
8187 match(Set mem (StoreN mem src));
8188
8189 ins_cost(VOLATILE_REF_COST);
8190 format %{ "stlrw $src, $mem\t# compressed ptr" %}
8191
8192 ins_encode(aarch64_enc_stlrw(src, mem));
8193
8194 ins_pipe(pipe_class_memory);
8195 %}
8196
8197 // Store Float
8198 instruct storeF_volatile(vRegF src, /* sync_memory*/indirect mem)
8199 %{
8200 match(Set mem (StoreF mem src));
8201
8202 ins_cost(VOLATILE_REF_COST);
8203 format %{ "stlrs $src, $mem\t# float" %}
8204
8205 ins_encode( aarch64_enc_fstlrs(src, mem) );
8206
8207 ins_pipe(pipe_class_memory);
8208 %}
8209
8210 // TODO
8211 // implement storeImmF0 and storeFImmPacked
8212
8213 // Store Double
8214 instruct storeD_volatile(vRegD src, /* sync_memory*/indirect mem)
8215 %{
8216 match(Set mem (StoreD mem src));
8217
8218 ins_cost(VOLATILE_REF_COST);
8219 format %{ "stlrd $src, $mem\t# double" %}
8220
8221 ins_encode( aarch64_enc_fstlrd(src, mem) );
8222
8223 ins_pipe(pipe_class_memory);
8224 %}
8225
8226 // ---------------- end of volatile loads and stores ----------------
8227
8228 instruct cacheWB(indirect addr)
8229 %{
8230 predicate(VM_Version::supports_data_cache_line_flush());
8231 match(CacheWB addr);
8232
8233 ins_cost(100);
8234 format %{"cache wb $addr" %}
8235 ins_encode %{
8236 assert($addr->index_position() < 0, "should be");
8237 assert($addr$$disp == 0, "should be");
8238 __ cache_wb(Address($addr$$base$$Register, 0));
8239 %}
8240 ins_pipe(pipe_slow); // XXX
8241 %}
8242
8243 instruct cacheWBPreSync()
8244 %{
8245 predicate(VM_Version::supports_data_cache_line_flush());
8246 match(CacheWBPreSync);
8247
8248 ins_cost(100);
8249 format %{"cache wb presync" %}
8250 ins_encode %{
8251 __ cache_wbsync(true);
8252 %}
8253 ins_pipe(pipe_slow); // XXX
8254 %}
8255
8256 instruct cacheWBPostSync()
8257 %{
8258 predicate(VM_Version::supports_data_cache_line_flush());
8259 match(CacheWBPostSync);
8260
8261 ins_cost(100);
8262 format %{"cache wb postsync" %}
8263 ins_encode %{
8264 __ cache_wbsync(false);
8265 %}
8266 ins_pipe(pipe_slow); // XXX
8267 %}
8268
8269 // ============================================================================
8270 // BSWAP Instructions
8271
8272 instruct bytes_reverse_int(iRegINoSp dst, iRegIorL2I src) %{
8273 match(Set dst (ReverseBytesI src));
8274
8275 ins_cost(INSN_COST);
8276 format %{ "revw $dst, $src" %}
8277
8278 ins_encode %{
8279 __ revw(as_Register($dst$$reg), as_Register($src$$reg));
8280 %}
8281
8282 ins_pipe(ialu_reg);
8283 %}
8284
8285 instruct bytes_reverse_long(iRegLNoSp dst, iRegL src) %{
8286 match(Set dst (ReverseBytesL src));
8287
8288 ins_cost(INSN_COST);
8289 format %{ "rev $dst, $src" %}
8290
8291 ins_encode %{
8292 __ rev(as_Register($dst$$reg), as_Register($src$$reg));
8293 %}
8294
8295 ins_pipe(ialu_reg);
8296 %}
8297
8298 instruct bytes_reverse_unsigned_short(iRegINoSp dst, iRegIorL2I src) %{
8299 match(Set dst (ReverseBytesUS src));
8300
8301 ins_cost(INSN_COST);
8302 format %{ "rev16w $dst, $src" %}
8303
8304 ins_encode %{
8305 __ rev16w(as_Register($dst$$reg), as_Register($src$$reg));
8306 %}
8307
8308 ins_pipe(ialu_reg);
8309 %}
8310
8311 instruct bytes_reverse_short(iRegINoSp dst, iRegIorL2I src) %{
8312 match(Set dst (ReverseBytesS src));
8313
8314 ins_cost(INSN_COST);
8315 format %{ "rev16w $dst, $src\n\t"
8316 "sbfmw $dst, $dst, #0, #15" %}
8317
8318 ins_encode %{
8319 __ rev16w(as_Register($dst$$reg), as_Register($src$$reg));
8320 __ sbfmw(as_Register($dst$$reg), as_Register($dst$$reg), 0U, 15U);
8321 %}
8322
8323 ins_pipe(ialu_reg);
8324 %}
8325
8326 // ============================================================================
8327 // Zero Count Instructions
8328
8329 instruct countLeadingZerosI(iRegINoSp dst, iRegIorL2I src) %{
8330 match(Set dst (CountLeadingZerosI src));
8331
8332 ins_cost(INSN_COST);
8333 format %{ "clzw $dst, $src" %}
8334 ins_encode %{
8335 __ clzw(as_Register($dst$$reg), as_Register($src$$reg));
8336 %}
8337
8338 ins_pipe(ialu_reg);
8339 %}
8340
8341 instruct countLeadingZerosL(iRegINoSp dst, iRegL src) %{
8342 match(Set dst (CountLeadingZerosL src));
8343
8344 ins_cost(INSN_COST);
8345 format %{ "clz $dst, $src" %}
8346 ins_encode %{
8347 __ clz(as_Register($dst$$reg), as_Register($src$$reg));
8348 %}
8349
8350 ins_pipe(ialu_reg);
8351 %}
8352
8353 instruct countTrailingZerosI(iRegINoSp dst, iRegIorL2I src) %{
8354 match(Set dst (CountTrailingZerosI src));
8355
8356 ins_cost(INSN_COST * 2);
8357 format %{ "rbitw $dst, $src\n\t"
8358 "clzw $dst, $dst" %}
8359 ins_encode %{
8360 __ rbitw(as_Register($dst$$reg), as_Register($src$$reg));
8361 __ clzw(as_Register($dst$$reg), as_Register($dst$$reg));
8362 %}
8363
8364 ins_pipe(ialu_reg);
8365 %}
8366
8367 instruct countTrailingZerosL(iRegINoSp dst, iRegL src) %{
8368 match(Set dst (CountTrailingZerosL src));
8369
8370 ins_cost(INSN_COST * 2);
8371 format %{ "rbit $dst, $src\n\t"
8372 "clz $dst, $dst" %}
8373 ins_encode %{
8374 __ rbit(as_Register($dst$$reg), as_Register($src$$reg));
8375 __ clz(as_Register($dst$$reg), as_Register($dst$$reg));
8376 %}
8377
8378 ins_pipe(ialu_reg);
8379 %}
8380
8381 //---------- Population Count Instructions -------------------------------------
8382 //
8383
8384 instruct popCountI(iRegINoSp dst, iRegIorL2I src, vRegF tmp) %{
8385 predicate(UsePopCountInstruction);
8386 match(Set dst (PopCountI src));
8387 effect(TEMP tmp);
8388 ins_cost(INSN_COST * 13);
8389
8390 format %{ "movw $src, $src\n\t"
8391 "mov $tmp, $src\t# vector (1D)\n\t"
8392 "cnt $tmp, $tmp\t# vector (8B)\n\t"
8393 "addv $tmp, $tmp\t# vector (8B)\n\t"
8394 "mov $dst, $tmp\t# vector (1D)" %}
8395 ins_encode %{
8396 __ movw($src$$Register, $src$$Register); // ensure top 32 bits 0
8397 __ mov($tmp$$FloatRegister, __ T1D, 0, $src$$Register);
8398 __ cnt($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
8399 __ addv($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
8400 __ mov($dst$$Register, $tmp$$FloatRegister, __ T1D, 0);
8401 %}
8402
8403 ins_pipe(pipe_class_default);
8404 %}
8405
8406 instruct popCountI_mem(iRegINoSp dst, memory4 mem, vRegF tmp) %{
8407 predicate(UsePopCountInstruction);
8408 match(Set dst (PopCountI (LoadI mem)));
8409 effect(TEMP tmp);
8410 ins_cost(INSN_COST * 13);
8411
8412 format %{ "ldrs $tmp, $mem\n\t"
8413 "cnt $tmp, $tmp\t# vector (8B)\n\t"
8414 "addv $tmp, $tmp\t# vector (8B)\n\t"
8415 "mov $dst, $tmp\t# vector (1D)" %}
8416 ins_encode %{
8417 FloatRegister tmp_reg = as_FloatRegister($tmp$$reg);
8418 loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrs, tmp_reg, $mem->opcode(),
8419 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
8420 __ cnt($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
8421 __ addv($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
8422 __ mov($dst$$Register, $tmp$$FloatRegister, __ T1D, 0);
8423 %}
8424
8425 ins_pipe(pipe_class_default);
8426 %}
8427
8428 // Note: Long.bitCount(long) returns an int.
8429 instruct popCountL(iRegINoSp dst, iRegL src, vRegD tmp) %{
8430 predicate(UsePopCountInstruction);
8431 match(Set dst (PopCountL src));
8432 effect(TEMP tmp);
8433 ins_cost(INSN_COST * 13);
8434
8435 format %{ "mov $tmp, $src\t# vector (1D)\n\t"
8436 "cnt $tmp, $tmp\t# vector (8B)\n\t"
8437 "addv $tmp, $tmp\t# vector (8B)\n\t"
8438 "mov $dst, $tmp\t# vector (1D)" %}
8439 ins_encode %{
8440 __ mov($tmp$$FloatRegister, __ T1D, 0, $src$$Register);
8441 __ cnt($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
8442 __ addv($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
8443 __ mov($dst$$Register, $tmp$$FloatRegister, __ T1D, 0);
8444 %}
8445
8446 ins_pipe(pipe_class_default);
8447 %}
8448
8449 instruct popCountL_mem(iRegINoSp dst, memory8 mem, vRegD tmp) %{
8450 predicate(UsePopCountInstruction);
8451 match(Set dst (PopCountL (LoadL mem)));
8452 effect(TEMP tmp);
8453 ins_cost(INSN_COST * 13);
8454
8455 format %{ "ldrd $tmp, $mem\n\t"
8456 "cnt $tmp, $tmp\t# vector (8B)\n\t"
8457 "addv $tmp, $tmp\t# vector (8B)\n\t"
8458 "mov $dst, $tmp\t# vector (1D)" %}
8459 ins_encode %{
8460 FloatRegister tmp_reg = as_FloatRegister($tmp$$reg);
8461 loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrd, tmp_reg, $mem->opcode(),
8462 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
8463 __ cnt($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
8464 __ addv($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
8465 __ mov($dst$$Register, $tmp$$FloatRegister, __ T1D, 0);
8466 %}
8467
8468 ins_pipe(pipe_class_default);
8469 %}
8470
8471 // ============================================================================
8472 // MemBar Instruction
8473
8474 instruct load_fence() %{
8475 match(LoadFence);
8476 ins_cost(VOLATILE_REF_COST);
8477
8478 format %{ "load_fence" %}
8479
8480 ins_encode %{
8481 __ membar(Assembler::LoadLoad|Assembler::LoadStore);
8482 %}
8483 ins_pipe(pipe_serial);
8484 %}
8485
8486 instruct unnecessary_membar_acquire() %{
8487 predicate(unnecessary_acquire(n));
8488 match(MemBarAcquire);
8489 ins_cost(0);
8490
8491 format %{ "membar_acquire (elided)" %}
8492
8493 ins_encode %{
8494 __ block_comment("membar_acquire (elided)");
8495 %}
8496
8497 ins_pipe(pipe_class_empty);
8498 %}
8499
8500 instruct membar_acquire() %{
8501 match(MemBarAcquire);
8502 ins_cost(VOLATILE_REF_COST);
8503
8504 format %{ "membar_acquire\n\t"
8505 "dmb ish" %}
8506
8507 ins_encode %{
8508 __ block_comment("membar_acquire");
8509 __ membar(Assembler::LoadLoad|Assembler::LoadStore);
8510 %}
8511
8512 ins_pipe(pipe_serial);
8513 %}
8514
8515
8516 instruct membar_acquire_lock() %{
8517 match(MemBarAcquireLock);
8518 ins_cost(VOLATILE_REF_COST);
8519
8520 format %{ "membar_acquire_lock (elided)" %}
8521
8522 ins_encode %{
8523 __ block_comment("membar_acquire_lock (elided)");
8524 %}
8525
8526 ins_pipe(pipe_serial);
8527 %}
8528
8529 instruct store_fence() %{
8530 match(StoreFence);
8531 ins_cost(VOLATILE_REF_COST);
8532
8533 format %{ "store_fence" %}
8534
8535 ins_encode %{
8536 __ membar(Assembler::LoadStore|Assembler::StoreStore);
8537 %}
8538 ins_pipe(pipe_serial);
8539 %}
8540
8541 instruct unnecessary_membar_release() %{
8542 predicate(unnecessary_release(n));
8543 match(MemBarRelease);
8544 ins_cost(0);
8545
8546 format %{ "membar_release (elided)" %}
8547
8548 ins_encode %{
8549 __ block_comment("membar_release (elided)");
8550 %}
8551 ins_pipe(pipe_serial);
8552 %}
8553
8554 instruct membar_release() %{
8555 match(MemBarRelease);
8556 ins_cost(VOLATILE_REF_COST);
8557
8558 format %{ "membar_release\n\t"
8559 "dmb ish" %}
8560
8561 ins_encode %{
8562 __ block_comment("membar_release");
8563 __ membar(Assembler::LoadStore|Assembler::StoreStore);
8564 %}
8565 ins_pipe(pipe_serial);
8566 %}
8567
8568 instruct membar_storestore() %{
8569 match(MemBarStoreStore);
8570 ins_cost(VOLATILE_REF_COST);
8571
8572 format %{ "MEMBAR-store-store" %}
8573
8574 ins_encode %{
8575 __ membar(Assembler::StoreStore);
8576 %}
8577 ins_pipe(pipe_serial);
8578 %}
8579
8580 instruct membar_release_lock() %{
8581 match(MemBarReleaseLock);
8582 ins_cost(VOLATILE_REF_COST);
8583
8584 format %{ "membar_release_lock (elided)" %}
8585
8586 ins_encode %{
8587 __ block_comment("membar_release_lock (elided)");
8588 %}
8589
8590 ins_pipe(pipe_serial);
8591 %}
8592
8593 instruct unnecessary_membar_volatile() %{
8594 predicate(unnecessary_volatile(n));
8595 match(MemBarVolatile);
8596 ins_cost(0);
8597
8598 format %{ "membar_volatile (elided)" %}
8599
8600 ins_encode %{
8601 __ block_comment("membar_volatile (elided)");
8602 %}
8603
8604 ins_pipe(pipe_serial);
8605 %}
8606
8607 instruct membar_volatile() %{
8608 match(MemBarVolatile);
8609 ins_cost(VOLATILE_REF_COST*100);
8610
8611 format %{ "membar_volatile\n\t"
8612 "dmb ish"%}
8613
8614 ins_encode %{
8615 __ block_comment("membar_volatile");
8616 __ membar(Assembler::StoreLoad);
8617 %}
8618
8619 ins_pipe(pipe_serial);
8620 %}
8621
8622 // ============================================================================
8623 // Cast/Convert Instructions
8624
8625 instruct castX2P(iRegPNoSp dst, iRegL src) %{
8626 match(Set dst (CastX2P src));
8627
8628 ins_cost(INSN_COST);
8629 format %{ "mov $dst, $src\t# long -> ptr" %}
8630
8631 ins_encode %{
8632 if ($dst$$reg != $src$$reg) {
8633 __ mov(as_Register($dst$$reg), as_Register($src$$reg));
8634 }
8635 %}
8636
8637 ins_pipe(ialu_reg);
8638 %}
8639
8640 instruct castP2X(iRegLNoSp dst, iRegP src) %{
8641 match(Set dst (CastP2X src));
8642
8643 ins_cost(INSN_COST);
8644 format %{ "mov $dst, $src\t# ptr -> long" %}
8645
8646 ins_encode %{
8647 if ($dst$$reg != $src$$reg) {
8648 __ mov(as_Register($dst$$reg), as_Register($src$$reg));
8649 }
8650 %}
8651
8652 ins_pipe(ialu_reg);
8653 %}
8654
8655 // Convert oop into int for vectors alignment masking
8656 instruct convP2I(iRegINoSp dst, iRegP src) %{
8657 match(Set dst (ConvL2I (CastP2X src)));
8658
8659 ins_cost(INSN_COST);
8660 format %{ "movw $dst, $src\t# ptr -> int" %}
8661 ins_encode %{
8662 __ movw($dst$$Register, $src$$Register);
8663 %}
8664
8665 ins_pipe(ialu_reg);
8666 %}
8667
8668 // Convert compressed oop into int for vectors alignment masking
8669 // in case of 32bit oops (heap < 4Gb).
8670 instruct convN2I(iRegINoSp dst, iRegN src)
8671 %{
8672 predicate(CompressedOops::shift() == 0);
8673 match(Set dst (ConvL2I (CastP2X (DecodeN src))));
8674
8675 ins_cost(INSN_COST);
8676 format %{ "mov dst, $src\t# compressed ptr -> int" %}
8677 ins_encode %{
8678 __ movw($dst$$Register, $src$$Register);
8679 %}
8680
8681 ins_pipe(ialu_reg);
8682 %}
8683
8684
8685 // Convert oop pointer into compressed form
8686 instruct encodeHeapOop(iRegNNoSp dst, iRegP src, rFlagsReg cr) %{
8687 predicate(n->bottom_type()->make_ptr()->ptr() != TypePtr::NotNull);
8688 match(Set dst (EncodeP src));
8689 effect(KILL cr);
8690 ins_cost(INSN_COST * 3);
8691 format %{ "encode_heap_oop $dst, $src" %}
8692 ins_encode %{
8693 Register s = $src$$Register;
8694 Register d = $dst$$Register;
8695 __ encode_heap_oop(d, s);
8696 %}
8697 ins_pipe(ialu_reg);
8698 %}
8699
8700 instruct encodeHeapOop_not_null(iRegNNoSp dst, iRegP src, rFlagsReg cr) %{
8701 predicate(n->bottom_type()->make_ptr()->ptr() == TypePtr::NotNull);
8702 match(Set dst (EncodeP src));
8703 ins_cost(INSN_COST * 3);
8704 format %{ "encode_heap_oop_not_null $dst, $src" %}
8705 ins_encode %{
8706 __ encode_heap_oop_not_null($dst$$Register, $src$$Register);
8707 %}
8708 ins_pipe(ialu_reg);
8709 %}
8710
8711 instruct decodeHeapOop(iRegPNoSp dst, iRegN src, rFlagsReg cr) %{
8712 predicate(n->bottom_type()->is_ptr()->ptr() != TypePtr::NotNull &&
8713 n->bottom_type()->is_ptr()->ptr() != TypePtr::Constant);
8714 match(Set dst (DecodeN src));
8715 ins_cost(INSN_COST * 3);
8716 format %{ "decode_heap_oop $dst, $src" %}
8717 ins_encode %{
8718 Register s = $src$$Register;
8719 Register d = $dst$$Register;
8720 __ decode_heap_oop(d, s);
8721 %}
8722 ins_pipe(ialu_reg);
8723 %}
8724
8725 instruct decodeHeapOop_not_null(iRegPNoSp dst, iRegN src, rFlagsReg cr) %{
8726 predicate(n->bottom_type()->is_ptr()->ptr() == TypePtr::NotNull ||
8727 n->bottom_type()->is_ptr()->ptr() == TypePtr::Constant);
8728 match(Set dst (DecodeN src));
8729 ins_cost(INSN_COST * 3);
8730 format %{ "decode_heap_oop_not_null $dst, $src" %}
8731 ins_encode %{
8732 Register s = $src$$Register;
8733 Register d = $dst$$Register;
8734 __ decode_heap_oop_not_null(d, s);
8735 %}
8736 ins_pipe(ialu_reg);
8737 %}
8738
8739 // n.b. AArch64 implementations of encode_klass_not_null and
8740 // decode_klass_not_null do not modify the flags register so, unlike
8741 // Intel, we don't kill CR as a side effect here
8742
8743 instruct encodeKlass_not_null(iRegNNoSp dst, iRegP src) %{
8744 match(Set dst (EncodePKlass src));
8745
8746 ins_cost(INSN_COST * 3);
8747 format %{ "encode_klass_not_null $dst,$src" %}
8748
8749 ins_encode %{
8750 Register src_reg = as_Register($src$$reg);
8751 Register dst_reg = as_Register($dst$$reg);
8752 __ encode_klass_not_null(dst_reg, src_reg);
8753 %}
8754
8755 ins_pipe(ialu_reg);
8756 %}
8757
8758 instruct decodeKlass_not_null(iRegPNoSp dst, iRegN src) %{
8759 match(Set dst (DecodeNKlass src));
8760
8761 ins_cost(INSN_COST * 3);
8762 format %{ "decode_klass_not_null $dst,$src" %}
8763
8764 ins_encode %{
8765 Register src_reg = as_Register($src$$reg);
8766 Register dst_reg = as_Register($dst$$reg);
8767 if (dst_reg != src_reg) {
8768 __ decode_klass_not_null(dst_reg, src_reg);
8769 } else {
8770 __ decode_klass_not_null(dst_reg);
8771 }
8772 %}
8773
8774 ins_pipe(ialu_reg);
8775 %}
8776
8777 instruct checkCastPP(iRegPNoSp dst)
8778 %{
8779 match(Set dst (CheckCastPP dst));
8780
8781 size(0);
8782 format %{ "# checkcastPP of $dst" %}
8783 ins_encode(/* empty encoding */);
8784 ins_pipe(pipe_class_empty);
8785 %}
8786
8787 instruct castPP(iRegPNoSp dst)
8788 %{
8789 match(Set dst (CastPP dst));
8790
8791 size(0);
8792 format %{ "# castPP of $dst" %}
8793 ins_encode(/* empty encoding */);
8794 ins_pipe(pipe_class_empty);
8795 %}
8796
8797 instruct castII(iRegI dst)
8798 %{
8799 match(Set dst (CastII dst));
8800
8801 size(0);
8802 format %{ "# castII of $dst" %}
8803 ins_encode(/* empty encoding */);
8804 ins_cost(0);
8805 ins_pipe(pipe_class_empty);
8806 %}
8807
8808 instruct castLL(iRegL dst)
8809 %{
8810 match(Set dst (CastLL dst));
8811
8812 size(0);
8813 format %{ "# castLL of $dst" %}
8814 ins_encode(/* empty encoding */);
8815 ins_cost(0);
8816 ins_pipe(pipe_class_empty);
8817 %}
8818
8819 instruct castFF(vRegF dst)
8820 %{
8821 match(Set dst (CastFF dst));
8822
8823 size(0);
8824 format %{ "# castFF of $dst" %}
8825 ins_encode(/* empty encoding */);
8826 ins_cost(0);
8827 ins_pipe(pipe_class_empty);
8828 %}
8829
8830 instruct castDD(vRegD dst)
8831 %{
8832 match(Set dst (CastDD dst));
8833
8834 size(0);
8835 format %{ "# castDD of $dst" %}
8836 ins_encode(/* empty encoding */);
8837 ins_cost(0);
8838 ins_pipe(pipe_class_empty);
8839 %}
8840
8841 instruct castVVD(vecD dst)
8842 %{
8843 match(Set dst (CastVV dst));
8844
8845 size(0);
8846 format %{ "# castVV of $dst" %}
8847 ins_encode(/* empty encoding */);
8848 ins_cost(0);
8849 ins_pipe(pipe_class_empty);
8850 %}
8851
8852 instruct castVVX(vecX dst)
8853 %{
8854 match(Set dst (CastVV dst));
8855
8856 size(0);
8857 format %{ "# castVV of $dst" %}
8858 ins_encode(/* empty encoding */);
8859 ins_cost(0);
8860 ins_pipe(pipe_class_empty);
8861 %}
8862
8863 instruct castVV(vReg dst)
8864 %{
8865 match(Set dst (CastVV dst));
8866
8867 size(0);
8868 format %{ "# castVV of $dst" %}
8869 ins_encode(/* empty encoding */);
8870 ins_cost(0);
8871 ins_pipe(pipe_class_empty);
8872 %}
8873
8874 // ============================================================================
8875 // Atomic operation instructions
8876 //
8877 // Intel and SPARC both implement Ideal Node LoadPLocked and
8878 // Store{PIL}Conditional instructions using a normal load for the
8879 // LoadPLocked and a CAS for the Store{PIL}Conditional.
8880 //
8881 // The ideal code appears only to use LoadPLocked/StorePLocked as a
8882 // pair to lock object allocations from Eden space when not using
8883 // TLABs.
8884 //
8885 // There does not appear to be a Load{IL}Locked Ideal Node and the
8886 // Ideal code appears to use Store{IL}Conditional as an alias for CAS
8887 // and to use StoreIConditional only for 32-bit and StoreLConditional
8888 // only for 64-bit.
8889 //
8890 // We implement LoadPLocked and StorePLocked instructions using,
8891 // respectively the AArch64 hw load-exclusive and store-conditional
8892 // instructions. Whereas we must implement each of
8893 // Store{IL}Conditional using a CAS which employs a pair of
8894 // instructions comprising a load-exclusive followed by a
8895 // store-conditional.
8896
8897
8898 // Locked-load (linked load) of the current heap-top
8899 // used when updating the eden heap top
8900 // implemented using ldaxr on AArch64
8901
8902 instruct loadPLocked(iRegPNoSp dst, indirect mem)
8903 %{
8904 match(Set dst (LoadPLocked mem));
8905
8906 ins_cost(VOLATILE_REF_COST);
8907
8908 format %{ "ldaxr $dst, $mem\t# ptr linked acquire" %}
8909
8910 ins_encode(aarch64_enc_ldaxr(dst, mem));
8911
8912 ins_pipe(pipe_serial);
8913 %}
8914
8915 // Conditional-store of the updated heap-top.
8916 // Used during allocation of the shared heap.
8917 // Sets flag (EQ) on success.
8918 // implemented using stlxr on AArch64.
8919
8920 instruct storePConditional(memory8 heap_top_ptr, iRegP oldval, iRegP newval, rFlagsReg cr)
8921 %{
8922 match(Set cr (StorePConditional heap_top_ptr (Binary oldval newval)));
8923
8924 ins_cost(VOLATILE_REF_COST);
8925
8926 // TODO
8927 // do we need to do a store-conditional release or can we just use a
8928 // plain store-conditional?
8929
8930 format %{
8931 "stlxr rscratch1, $newval, $heap_top_ptr\t# ptr cond release"
8932 "cmpw rscratch1, zr\t# EQ on successful write"
8933 %}
8934
8935 ins_encode(aarch64_enc_stlxr(newval, heap_top_ptr));
8936
8937 ins_pipe(pipe_serial);
8938 %}
8939
8940 instruct storeLConditional(indirect mem, iRegLNoSp oldval, iRegLNoSp newval, rFlagsReg cr)
8941 %{
8942 match(Set cr (StoreLConditional mem (Binary oldval newval)));
8943
8944 ins_cost(VOLATILE_REF_COST);
8945
8946 format %{
8947 "cmpxchg rscratch1, $mem, $oldval, $newval, $mem\t# if $mem == $oldval then $mem <-- $newval"
8948 "cmpw rscratch1, zr\t# EQ on successful write"
8949 %}
8950
8951 ins_encode(aarch64_enc_cmpxchg_acq(mem, oldval, newval));
8952
8953 ins_pipe(pipe_slow);
8954 %}
8955
8956 // storeIConditional also has acquire semantics, for no better reason
8957 // than matching storeLConditional. At the time of writing this
8958 // comment storeIConditional was not used anywhere by AArch64.
8959 instruct storeIConditional(indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr)
8960 %{
8961 match(Set cr (StoreIConditional mem (Binary oldval newval)));
8962
8963 ins_cost(VOLATILE_REF_COST);
8964
8965 format %{
8966 "cmpxchgw rscratch1, $mem, $oldval, $newval, $mem\t# if $mem == $oldval then $mem <-- $newval"
8967 "cmpw rscratch1, zr\t# EQ on successful write"
8968 %}
8969
8970 ins_encode(aarch64_enc_cmpxchgw_acq(mem, oldval, newval));
8971
8972 ins_pipe(pipe_slow);
8973 %}
8974
8975 // standard CompareAndSwapX when we are using barriers
8976 // these have higher priority than the rules selected by a predicate
8977
8978 // XXX No flag versions for CompareAndSwap{I,L,P,N} because matcher
8979 // can't match them
8980
8981 instruct compareAndSwapB(iRegINoSp res, indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr) %{
8982
8983 match(Set res (CompareAndSwapB mem (Binary oldval newval)));
8984 ins_cost(2 * VOLATILE_REF_COST);
8985
8986 effect(KILL cr);
8987
8988 format %{
8989 "cmpxchgb $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval"
8990 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8991 %}
8992
8993 ins_encode(aarch64_enc_cmpxchgb(mem, oldval, newval),
8994 aarch64_enc_cset_eq(res));
8995
8996 ins_pipe(pipe_slow);
8997 %}
8998
8999 instruct compareAndSwapS(iRegINoSp res, indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr) %{
9000
9001 match(Set res (CompareAndSwapS mem (Binary oldval newval)));
9002 ins_cost(2 * VOLATILE_REF_COST);
9003
9004 effect(KILL cr);
9005
9006 format %{
9007 "cmpxchgs $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval"
9008 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
9009 %}
9010
9011 ins_encode(aarch64_enc_cmpxchgs(mem, oldval, newval),
9012 aarch64_enc_cset_eq(res));
9013
9014 ins_pipe(pipe_slow);
9015 %}
9016
9017 instruct compareAndSwapI(iRegINoSp res, indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr) %{
9018
9019 match(Set res (CompareAndSwapI mem (Binary oldval newval)));
9020 ins_cost(2 * VOLATILE_REF_COST);
9021
9022 effect(KILL cr);
9023
9024 format %{
9025 "cmpxchgw $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval"
9026 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
9027 %}
9028
9029 ins_encode(aarch64_enc_cmpxchgw(mem, oldval, newval),
9030 aarch64_enc_cset_eq(res));
9031
9032 ins_pipe(pipe_slow);
9033 %}
9034
9035 instruct compareAndSwapL(iRegINoSp res, indirect mem, iRegLNoSp oldval, iRegLNoSp newval, rFlagsReg cr) %{
9036
9037 match(Set res (CompareAndSwapL mem (Binary oldval newval)));
9038 ins_cost(2 * VOLATILE_REF_COST);
9039
9040 effect(KILL cr);
9041
9042 format %{
9043 "cmpxchg $mem, $oldval, $newval\t# (long) if $mem == $oldval then $mem <-- $newval"
9044 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
9045 %}
9046
9047 ins_encode(aarch64_enc_cmpxchg(mem, oldval, newval),
9048 aarch64_enc_cset_eq(res));
9049
9050 ins_pipe(pipe_slow);
9051 %}
9052
9053 instruct compareAndSwapP(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
9054
9055 match(Set res (CompareAndSwapP mem (Binary oldval newval)));
9056 predicate(n->as_LoadStore()->barrier_data() == 0);
9057 ins_cost(2 * VOLATILE_REF_COST);
9058
9059 effect(KILL cr);
9060
9061 format %{
9062 "cmpxchg $mem, $oldval, $newval\t# (ptr) if $mem == $oldval then $mem <-- $newval"
9063 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
9064 %}
9065
9066 ins_encode(aarch64_enc_cmpxchg(mem, oldval, newval),
9067 aarch64_enc_cset_eq(res));
9068
9069 ins_pipe(pipe_slow);
9070 %}
9071
9072 instruct compareAndSwapN(iRegINoSp res, indirect mem, iRegNNoSp oldval, iRegNNoSp newval, rFlagsReg cr) %{
9073
9074 match(Set res (CompareAndSwapN mem (Binary oldval newval)));
9075 ins_cost(2 * VOLATILE_REF_COST);
9076
9077 effect(KILL cr);
9078
9079 format %{
9080 "cmpxchgw $mem, $oldval, $newval\t# (narrow oop) if $mem == $oldval then $mem <-- $newval"
9081 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
9082 %}
9083
9084 ins_encode(aarch64_enc_cmpxchgw(mem, oldval, newval),
9085 aarch64_enc_cset_eq(res));
9086
9087 ins_pipe(pipe_slow);
9088 %}
9089
9090 // alternative CompareAndSwapX when we are eliding barriers
9091
9092 instruct compareAndSwapBAcq(iRegINoSp res, indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr) %{
9093
9094 predicate(needs_acquiring_load_exclusive(n));
9095 match(Set res (CompareAndSwapB mem (Binary oldval newval)));
9096 ins_cost(VOLATILE_REF_COST);
9097
9098 effect(KILL cr);
9099
9100 format %{
9101 "cmpxchgb_acq $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval"
9102 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
9103 %}
9104
9105 ins_encode(aarch64_enc_cmpxchgb_acq(mem, oldval, newval),
9106 aarch64_enc_cset_eq(res));
9107
9108 ins_pipe(pipe_slow);
9109 %}
9110
9111 instruct compareAndSwapSAcq(iRegINoSp res, indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr) %{
9112
9113 predicate(needs_acquiring_load_exclusive(n));
9114 match(Set res (CompareAndSwapS mem (Binary oldval newval)));
9115 ins_cost(VOLATILE_REF_COST);
9116
9117 effect(KILL cr);
9118
9119 format %{
9120 "cmpxchgs_acq $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval"
9121 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
9122 %}
9123
9124 ins_encode(aarch64_enc_cmpxchgs_acq(mem, oldval, newval),
9125 aarch64_enc_cset_eq(res));
9126
9127 ins_pipe(pipe_slow);
9128 %}
9129
9130 instruct compareAndSwapIAcq(iRegINoSp res, indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr) %{
9131
9132 predicate(needs_acquiring_load_exclusive(n));
9133 match(Set res (CompareAndSwapI mem (Binary oldval newval)));
9134 ins_cost(VOLATILE_REF_COST);
9135
9136 effect(KILL cr);
9137
9138 format %{
9139 "cmpxchgw_acq $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval"
9140 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
9141 %}
9142
9143 ins_encode(aarch64_enc_cmpxchgw_acq(mem, oldval, newval),
9144 aarch64_enc_cset_eq(res));
9145
9146 ins_pipe(pipe_slow);
9147 %}
9148
9149 instruct compareAndSwapLAcq(iRegINoSp res, indirect mem, iRegLNoSp oldval, iRegLNoSp newval, rFlagsReg cr) %{
9150
9151 predicate(needs_acquiring_load_exclusive(n));
9152 match(Set res (CompareAndSwapL mem (Binary oldval newval)));
9153 ins_cost(VOLATILE_REF_COST);
9154
9155 effect(KILL cr);
9156
9157 format %{
9158 "cmpxchg_acq $mem, $oldval, $newval\t# (long) if $mem == $oldval then $mem <-- $newval"
9159 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
9160 %}
9161
9162 ins_encode(aarch64_enc_cmpxchg_acq(mem, oldval, newval),
9163 aarch64_enc_cset_eq(res));
9164
9165 ins_pipe(pipe_slow);
9166 %}
9167
9168 instruct compareAndSwapPAcq(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
9169
9170 predicate(needs_acquiring_load_exclusive(n) && (n->as_LoadStore()->barrier_data() == 0));
9171 match(Set res (CompareAndSwapP mem (Binary oldval newval)));
9172 ins_cost(VOLATILE_REF_COST);
9173
9174 effect(KILL cr);
9175
9176 format %{
9177 "cmpxchg_acq $mem, $oldval, $newval\t# (ptr) if $mem == $oldval then $mem <-- $newval"
9178 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
9179 %}
9180
9181 ins_encode(aarch64_enc_cmpxchg_acq(mem, oldval, newval),
9182 aarch64_enc_cset_eq(res));
9183
9184 ins_pipe(pipe_slow);
9185 %}
9186
9187 instruct compareAndSwapNAcq(iRegINoSp res, indirect mem, iRegNNoSp oldval, iRegNNoSp newval, rFlagsReg cr) %{
9188
9189 predicate(needs_acquiring_load_exclusive(n));
9190 match(Set res (CompareAndSwapN mem (Binary oldval newval)));
9191 ins_cost(VOLATILE_REF_COST);
9192
9193 effect(KILL cr);
9194
9195 format %{
9196 "cmpxchgw_acq $mem, $oldval, $newval\t# (narrow oop) if $mem == $oldval then $mem <-- $newval"
9197 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
9198 %}
9199
9200 ins_encode(aarch64_enc_cmpxchgw_acq(mem, oldval, newval),
9201 aarch64_enc_cset_eq(res));
9202
9203 ins_pipe(pipe_slow);
9204 %}
9205
9206
9207 // ---------------------------------------------------------------------
9208
9209 // BEGIN This section of the file is automatically generated. Do not edit --------------
9210
9211 // Sundry CAS operations. Note that release is always true,
9212 // regardless of the memory ordering of the CAS. This is because we
9213 // need the volatile case to be sequentially consistent but there is
9214 // no trailing StoreLoad barrier emitted by C2. Unfortunately we
9215 // can't check the type of memory ordering here, so we always emit a
9216 // STLXR.
9217
9218 // This section is generated from aarch64_ad_cas.m4
9219
9220
9221
9222 // This pattern is generated automatically from cas.m4.
9223 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9224 instruct compareAndExchangeB(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
9225
9226 match(Set res (CompareAndExchangeB mem (Binary oldval newval)));
9227 ins_cost(2 * VOLATILE_REF_COST);
9228 effect(TEMP_DEF res, KILL cr);
9229 format %{
9230 "cmpxchgb $res = $mem, $oldval, $newval\t# (byte, weak) if $mem == $oldval then $mem <-- $newval"
9231 %}
9232 ins_encode %{
9233 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
9234 Assembler::byte, /*acquire*/ false, /*release*/ true,
9235 /*weak*/ false, $res$$Register);
9236 __ sxtbw($res$$Register, $res$$Register);
9237 %}
9238 ins_pipe(pipe_slow);
9239 %}
9240
9241 // This pattern is generated automatically from cas.m4.
9242 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9243 instruct compareAndExchangeS(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
9244
9245 match(Set res (CompareAndExchangeS mem (Binary oldval newval)));
9246 ins_cost(2 * VOLATILE_REF_COST);
9247 effect(TEMP_DEF res, KILL cr);
9248 format %{
9249 "cmpxchgs $res = $mem, $oldval, $newval\t# (short, weak) if $mem == $oldval then $mem <-- $newval"
9250 %}
9251 ins_encode %{
9252 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
9253 Assembler::halfword, /*acquire*/ false, /*release*/ true,
9254 /*weak*/ false, $res$$Register);
9255 __ sxthw($res$$Register, $res$$Register);
9256 %}
9257 ins_pipe(pipe_slow);
9258 %}
9259
9260 // This pattern is generated automatically from cas.m4.
9261 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9262 instruct compareAndExchangeI(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
9263
9264 match(Set res (CompareAndExchangeI mem (Binary oldval newval)));
9265 ins_cost(2 * VOLATILE_REF_COST);
9266 effect(TEMP_DEF res, KILL cr);
9267 format %{
9268 "cmpxchgw $res = $mem, $oldval, $newval\t# (int, weak) if $mem == $oldval then $mem <-- $newval"
9269 %}
9270 ins_encode %{
9271 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
9272 Assembler::word, /*acquire*/ false, /*release*/ true,
9273 /*weak*/ false, $res$$Register);
9274 %}
9275 ins_pipe(pipe_slow);
9276 %}
9277
9278 // This pattern is generated automatically from cas.m4.
9279 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9280 instruct compareAndExchangeL(iRegLNoSp res, indirect mem, iRegL oldval, iRegL newval, rFlagsReg cr) %{
9281
9282 match(Set res (CompareAndExchangeL mem (Binary oldval newval)));
9283 ins_cost(2 * VOLATILE_REF_COST);
9284 effect(TEMP_DEF res, KILL cr);
9285 format %{
9286 "cmpxchg $res = $mem, $oldval, $newval\t# (long, weak) if $mem == $oldval then $mem <-- $newval"
9287 %}
9288 ins_encode %{
9289 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
9290 Assembler::xword, /*acquire*/ false, /*release*/ true,
9291 /*weak*/ false, $res$$Register);
9292 %}
9293 ins_pipe(pipe_slow);
9294 %}
9295
9296 // This pattern is generated automatically from cas.m4.
9297 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9298 instruct compareAndExchangeN(iRegNNoSp res, indirect mem, iRegN oldval, iRegN newval, rFlagsReg cr) %{
9299
9300 match(Set res (CompareAndExchangeN mem (Binary oldval newval)));
9301 ins_cost(2 * VOLATILE_REF_COST);
9302 effect(TEMP_DEF res, KILL cr);
9303 format %{
9304 "cmpxchgw $res = $mem, $oldval, $newval\t# (narrow oop, weak) if $mem == $oldval then $mem <-- $newval"
9305 %}
9306 ins_encode %{
9307 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
9308 Assembler::word, /*acquire*/ false, /*release*/ true,
9309 /*weak*/ false, $res$$Register);
9310 %}
9311 ins_pipe(pipe_slow);
9312 %}
9313
9314 // This pattern is generated automatically from cas.m4.
9315 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9316 instruct compareAndExchangeP(iRegPNoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
9317 predicate(n->as_LoadStore()->barrier_data() == 0);
9318 match(Set res (CompareAndExchangeP mem (Binary oldval newval)));
9319 ins_cost(2 * VOLATILE_REF_COST);
9320 effect(TEMP_DEF res, KILL cr);
9321 format %{
9322 "cmpxchg $res = $mem, $oldval, $newval\t# (ptr, weak) if $mem == $oldval then $mem <-- $newval"
9323 %}
9324 ins_encode %{
9325 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
9326 Assembler::xword, /*acquire*/ false, /*release*/ true,
9327 /*weak*/ false, $res$$Register);
9328 %}
9329 ins_pipe(pipe_slow);
9330 %}
9331
9332 // This pattern is generated automatically from cas.m4.
9333 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9334 instruct compareAndExchangeBAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
9335 predicate(needs_acquiring_load_exclusive(n));
9336 match(Set res (CompareAndExchangeB mem (Binary oldval newval)));
9337 ins_cost(VOLATILE_REF_COST);
9338 effect(TEMP_DEF res, KILL cr);
9339 format %{
9340 "cmpxchgb_acq $res = $mem, $oldval, $newval\t# (byte, weak) if $mem == $oldval then $mem <-- $newval"
9341 %}
9342 ins_encode %{
9343 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
9344 Assembler::byte, /*acquire*/ true, /*release*/ true,
9345 /*weak*/ false, $res$$Register);
9346 __ sxtbw($res$$Register, $res$$Register);
9347 %}
9348 ins_pipe(pipe_slow);
9349 %}
9350
9351 // This pattern is generated automatically from cas.m4.
9352 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9353 instruct compareAndExchangeSAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
9354 predicate(needs_acquiring_load_exclusive(n));
9355 match(Set res (CompareAndExchangeS mem (Binary oldval newval)));
9356 ins_cost(VOLATILE_REF_COST);
9357 effect(TEMP_DEF res, KILL cr);
9358 format %{
9359 "cmpxchgs_acq $res = $mem, $oldval, $newval\t# (short, weak) if $mem == $oldval then $mem <-- $newval"
9360 %}
9361 ins_encode %{
9362 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
9363 Assembler::halfword, /*acquire*/ true, /*release*/ true,
9364 /*weak*/ false, $res$$Register);
9365 __ sxthw($res$$Register, $res$$Register);
9366 %}
9367 ins_pipe(pipe_slow);
9368 %}
9369
9370 // This pattern is generated automatically from cas.m4.
9371 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9372 instruct compareAndExchangeIAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
9373 predicate(needs_acquiring_load_exclusive(n));
9374 match(Set res (CompareAndExchangeI mem (Binary oldval newval)));
9375 ins_cost(VOLATILE_REF_COST);
9376 effect(TEMP_DEF res, KILL cr);
9377 format %{
9378 "cmpxchgw_acq $res = $mem, $oldval, $newval\t# (int, weak) if $mem == $oldval then $mem <-- $newval"
9379 %}
9380 ins_encode %{
9381 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
9382 Assembler::word, /*acquire*/ true, /*release*/ true,
9383 /*weak*/ false, $res$$Register);
9384 %}
9385 ins_pipe(pipe_slow);
9386 %}
9387
9388 // This pattern is generated automatically from cas.m4.
9389 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9390 instruct compareAndExchangeLAcq(iRegLNoSp res, indirect mem, iRegL oldval, iRegL newval, rFlagsReg cr) %{
9391 predicate(needs_acquiring_load_exclusive(n));
9392 match(Set res (CompareAndExchangeL mem (Binary oldval newval)));
9393 ins_cost(VOLATILE_REF_COST);
9394 effect(TEMP_DEF res, KILL cr);
9395 format %{
9396 "cmpxchg_acq $res = $mem, $oldval, $newval\t# (long, weak) if $mem == $oldval then $mem <-- $newval"
9397 %}
9398 ins_encode %{
9399 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
9400 Assembler::xword, /*acquire*/ true, /*release*/ true,
9401 /*weak*/ false, $res$$Register);
9402 %}
9403 ins_pipe(pipe_slow);
9404 %}
9405
9406 // This pattern is generated automatically from cas.m4.
9407 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9408 instruct compareAndExchangeNAcq(iRegNNoSp res, indirect mem, iRegN oldval, iRegN newval, rFlagsReg cr) %{
9409 predicate(needs_acquiring_load_exclusive(n));
9410 match(Set res (CompareAndExchangeN mem (Binary oldval newval)));
9411 ins_cost(VOLATILE_REF_COST);
9412 effect(TEMP_DEF res, KILL cr);
9413 format %{
9414 "cmpxchgw_acq $res = $mem, $oldval, $newval\t# (narrow oop, weak) if $mem == $oldval then $mem <-- $newval"
9415 %}
9416 ins_encode %{
9417 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
9418 Assembler::word, /*acquire*/ true, /*release*/ true,
9419 /*weak*/ false, $res$$Register);
9420 %}
9421 ins_pipe(pipe_slow);
9422 %}
9423
9424 // This pattern is generated automatically from cas.m4.
9425 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9426 instruct compareAndExchangePAcq(iRegPNoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
9427 predicate(needs_acquiring_load_exclusive(n) && (n->as_LoadStore()->barrier_data() == 0));
9428 match(Set res (CompareAndExchangeP mem (Binary oldval newval)));
9429 ins_cost(VOLATILE_REF_COST);
9430 effect(TEMP_DEF res, KILL cr);
9431 format %{
9432 "cmpxchg_acq $res = $mem, $oldval, $newval\t# (ptr, weak) if $mem == $oldval then $mem <-- $newval"
9433 %}
9434 ins_encode %{
9435 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
9436 Assembler::xword, /*acquire*/ true, /*release*/ true,
9437 /*weak*/ false, $res$$Register);
9438 %}
9439 ins_pipe(pipe_slow);
9440 %}
9441
9442 // This pattern is generated automatically from cas.m4.
9443 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9444 instruct weakCompareAndSwapB(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
9445
9446 match(Set res (WeakCompareAndSwapB mem (Binary oldval newval)));
9447 ins_cost(2 * VOLATILE_REF_COST);
9448 effect(KILL cr);
9449 format %{
9450 "cmpxchgb $res = $mem, $oldval, $newval\t# (byte, weak) if $mem == $oldval then $mem <-- $newval"
9451 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
9452 %}
9453 ins_encode %{
9454 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
9455 Assembler::byte, /*acquire*/ false, /*release*/ true,
9456 /*weak*/ true, noreg);
9457 __ csetw($res$$Register, Assembler::EQ);
9458 %}
9459 ins_pipe(pipe_slow);
9460 %}
9461
9462 // This pattern is generated automatically from cas.m4.
9463 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9464 instruct weakCompareAndSwapS(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
9465
9466 match(Set res (WeakCompareAndSwapS mem (Binary oldval newval)));
9467 ins_cost(2 * VOLATILE_REF_COST);
9468 effect(KILL cr);
9469 format %{
9470 "cmpxchgs $res = $mem, $oldval, $newval\t# (short, weak) if $mem == $oldval then $mem <-- $newval"
9471 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
9472 %}
9473 ins_encode %{
9474 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
9475 Assembler::halfword, /*acquire*/ false, /*release*/ true,
9476 /*weak*/ true, noreg);
9477 __ csetw($res$$Register, Assembler::EQ);
9478 %}
9479 ins_pipe(pipe_slow);
9480 %}
9481
9482 // This pattern is generated automatically from cas.m4.
9483 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9484 instruct weakCompareAndSwapI(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
9485
9486 match(Set res (WeakCompareAndSwapI mem (Binary oldval newval)));
9487 ins_cost(2 * VOLATILE_REF_COST);
9488 effect(KILL cr);
9489 format %{
9490 "cmpxchgw $res = $mem, $oldval, $newval\t# (int, weak) if $mem == $oldval then $mem <-- $newval"
9491 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
9492 %}
9493 ins_encode %{
9494 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
9495 Assembler::word, /*acquire*/ false, /*release*/ true,
9496 /*weak*/ true, noreg);
9497 __ csetw($res$$Register, Assembler::EQ);
9498 %}
9499 ins_pipe(pipe_slow);
9500 %}
9501
9502 // This pattern is generated automatically from cas.m4.
9503 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9504 instruct weakCompareAndSwapL(iRegINoSp res, indirect mem, iRegL oldval, iRegL newval, rFlagsReg cr) %{
9505
9506 match(Set res (WeakCompareAndSwapL mem (Binary oldval newval)));
9507 ins_cost(2 * VOLATILE_REF_COST);
9508 effect(KILL cr);
9509 format %{
9510 "cmpxchg $res = $mem, $oldval, $newval\t# (long, weak) if $mem == $oldval then $mem <-- $newval"
9511 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
9512 %}
9513 ins_encode %{
9514 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
9515 Assembler::xword, /*acquire*/ false, /*release*/ true,
9516 /*weak*/ true, noreg);
9517 __ csetw($res$$Register, Assembler::EQ);
9518 %}
9519 ins_pipe(pipe_slow);
9520 %}
9521
9522 // This pattern is generated automatically from cas.m4.
9523 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9524 instruct weakCompareAndSwapN(iRegINoSp res, indirect mem, iRegN oldval, iRegN newval, rFlagsReg cr) %{
9525
9526 match(Set res (WeakCompareAndSwapN mem (Binary oldval newval)));
9527 ins_cost(2 * VOLATILE_REF_COST);
9528 effect(KILL cr);
9529 format %{
9530 "cmpxchgw $res = $mem, $oldval, $newval\t# (narrow oop, weak) if $mem == $oldval then $mem <-- $newval"
9531 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
9532 %}
9533 ins_encode %{
9534 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
9535 Assembler::word, /*acquire*/ false, /*release*/ true,
9536 /*weak*/ true, noreg);
9537 __ csetw($res$$Register, Assembler::EQ);
9538 %}
9539 ins_pipe(pipe_slow);
9540 %}
9541
9542 // This pattern is generated automatically from cas.m4.
9543 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9544 instruct weakCompareAndSwapP(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
9545 predicate(n->as_LoadStore()->barrier_data() == 0);
9546 match(Set res (WeakCompareAndSwapP mem (Binary oldval newval)));
9547 ins_cost(2 * VOLATILE_REF_COST);
9548 effect(KILL cr);
9549 format %{
9550 "cmpxchg $res = $mem, $oldval, $newval\t# (ptr, weak) if $mem == $oldval then $mem <-- $newval"
9551 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
9552 %}
9553 ins_encode %{
9554 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
9555 Assembler::xword, /*acquire*/ false, /*release*/ true,
9556 /*weak*/ true, noreg);
9557 __ csetw($res$$Register, Assembler::EQ);
9558 %}
9559 ins_pipe(pipe_slow);
9560 %}
9561
9562 // This pattern is generated automatically from cas.m4.
9563 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9564 instruct weakCompareAndSwapBAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
9565 predicate(needs_acquiring_load_exclusive(n));
9566 match(Set res (WeakCompareAndSwapB mem (Binary oldval newval)));
9567 ins_cost(VOLATILE_REF_COST);
9568 effect(KILL cr);
9569 format %{
9570 "cmpxchgb_acq $res = $mem, $oldval, $newval\t# (byte, weak) if $mem == $oldval then $mem <-- $newval"
9571 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
9572 %}
9573 ins_encode %{
9574 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
9575 Assembler::byte, /*acquire*/ true, /*release*/ true,
9576 /*weak*/ true, noreg);
9577 __ csetw($res$$Register, Assembler::EQ);
9578 %}
9579 ins_pipe(pipe_slow);
9580 %}
9581
9582 // This pattern is generated automatically from cas.m4.
9583 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9584 instruct weakCompareAndSwapSAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
9585 predicate(needs_acquiring_load_exclusive(n));
9586 match(Set res (WeakCompareAndSwapS mem (Binary oldval newval)));
9587 ins_cost(VOLATILE_REF_COST);
9588 effect(KILL cr);
9589 format %{
9590 "cmpxchgs_acq $res = $mem, $oldval, $newval\t# (short, weak) if $mem == $oldval then $mem <-- $newval"
9591 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
9592 %}
9593 ins_encode %{
9594 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
9595 Assembler::halfword, /*acquire*/ true, /*release*/ true,
9596 /*weak*/ true, noreg);
9597 __ csetw($res$$Register, Assembler::EQ);
9598 %}
9599 ins_pipe(pipe_slow);
9600 %}
9601
9602 // This pattern is generated automatically from cas.m4.
9603 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9604 instruct weakCompareAndSwapIAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
9605 predicate(needs_acquiring_load_exclusive(n));
9606 match(Set res (WeakCompareAndSwapI mem (Binary oldval newval)));
9607 ins_cost(VOLATILE_REF_COST);
9608 effect(KILL cr);
9609 format %{
9610 "cmpxchgw_acq $res = $mem, $oldval, $newval\t# (int, weak) if $mem == $oldval then $mem <-- $newval"
9611 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
9612 %}
9613 ins_encode %{
9614 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
9615 Assembler::word, /*acquire*/ true, /*release*/ true,
9616 /*weak*/ true, noreg);
9617 __ csetw($res$$Register, Assembler::EQ);
9618 %}
9619 ins_pipe(pipe_slow);
9620 %}
9621
9622 // This pattern is generated automatically from cas.m4.
9623 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9624 instruct weakCompareAndSwapLAcq(iRegINoSp res, indirect mem, iRegL oldval, iRegL newval, rFlagsReg cr) %{
9625 predicate(needs_acquiring_load_exclusive(n));
9626 match(Set res (WeakCompareAndSwapL mem (Binary oldval newval)));
9627 ins_cost(VOLATILE_REF_COST);
9628 effect(KILL cr);
9629 format %{
9630 "cmpxchg_acq $res = $mem, $oldval, $newval\t# (long, weak) if $mem == $oldval then $mem <-- $newval"
9631 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
9632 %}
9633 ins_encode %{
9634 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
9635 Assembler::xword, /*acquire*/ true, /*release*/ true,
9636 /*weak*/ true, noreg);
9637 __ csetw($res$$Register, Assembler::EQ);
9638 %}
9639 ins_pipe(pipe_slow);
9640 %}
9641
9642 // This pattern is generated automatically from cas.m4.
9643 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9644 instruct weakCompareAndSwapNAcq(iRegINoSp res, indirect mem, iRegN oldval, iRegN newval, rFlagsReg cr) %{
9645 predicate(needs_acquiring_load_exclusive(n));
9646 match(Set res (WeakCompareAndSwapN mem (Binary oldval newval)));
9647 ins_cost(VOLATILE_REF_COST);
9648 effect(KILL cr);
9649 format %{
9650 "cmpxchgw_acq $res = $mem, $oldval, $newval\t# (narrow oop, weak) if $mem == $oldval then $mem <-- $newval"
9651 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
9652 %}
9653 ins_encode %{
9654 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
9655 Assembler::word, /*acquire*/ true, /*release*/ true,
9656 /*weak*/ true, noreg);
9657 __ csetw($res$$Register, Assembler::EQ);
9658 %}
9659 ins_pipe(pipe_slow);
9660 %}
9661
9662 // This pattern is generated automatically from cas.m4.
9663 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9664 instruct weakCompareAndSwapPAcq(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
9665 predicate(needs_acquiring_load_exclusive(n) && (n->as_LoadStore()->barrier_data() == 0));
9666 match(Set res (WeakCompareAndSwapP mem (Binary oldval newval)));
9667 ins_cost(VOLATILE_REF_COST);
9668 effect(KILL cr);
9669 format %{
9670 "cmpxchg_acq $res = $mem, $oldval, $newval\t# (ptr, weak) if $mem == $oldval then $mem <-- $newval"
9671 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
9672 %}
9673 ins_encode %{
9674 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
9675 Assembler::xword, /*acquire*/ true, /*release*/ true,
9676 /*weak*/ true, noreg);
9677 __ csetw($res$$Register, Assembler::EQ);
9678 %}
9679 ins_pipe(pipe_slow);
9680 %}
9681
9682 // END This section of the file is automatically generated. Do not edit --------------
9683 // ---------------------------------------------------------------------
9684
9685 instruct get_and_setI(indirect mem, iRegI newv, iRegINoSp prev) %{
9686 match(Set prev (GetAndSetI mem newv));
9687 ins_cost(2 * VOLATILE_REF_COST);
9688 format %{ "atomic_xchgw $prev, $newv, [$mem]" %}
9689 ins_encode %{
9690 __ atomic_xchgw($prev$$Register, $newv$$Register, as_Register($mem$$base));
9691 %}
9692 ins_pipe(pipe_serial);
9693 %}
9694
9695 instruct get_and_setL(indirect mem, iRegL newv, iRegLNoSp prev) %{
9696 match(Set prev (GetAndSetL mem newv));
9697 ins_cost(2 * VOLATILE_REF_COST);
9698 format %{ "atomic_xchg $prev, $newv, [$mem]" %}
9699 ins_encode %{
9700 __ atomic_xchg($prev$$Register, $newv$$Register, as_Register($mem$$base));
9701 %}
9702 ins_pipe(pipe_serial);
9703 %}
9704
9705 instruct get_and_setN(indirect mem, iRegN newv, iRegINoSp prev) %{
9706 match(Set prev (GetAndSetN mem newv));
9707 ins_cost(2 * VOLATILE_REF_COST);
9708 format %{ "atomic_xchgw $prev, $newv, [$mem]" %}
9709 ins_encode %{
9710 __ atomic_xchgw($prev$$Register, $newv$$Register, as_Register($mem$$base));
9711 %}
9712 ins_pipe(pipe_serial);
9713 %}
9714
9715 instruct get_and_setP(indirect mem, iRegP newv, iRegPNoSp prev) %{
9716 predicate(n->as_LoadStore()->barrier_data() == 0);
9717 match(Set prev (GetAndSetP mem newv));
9718 ins_cost(2 * VOLATILE_REF_COST);
9719 format %{ "atomic_xchg $prev, $newv, [$mem]" %}
9720 ins_encode %{
9721 __ atomic_xchg($prev$$Register, $newv$$Register, as_Register($mem$$base));
9722 %}
9723 ins_pipe(pipe_serial);
9724 %}
9725
9726 instruct get_and_setIAcq(indirect mem, iRegI newv, iRegINoSp prev) %{
9727 predicate(needs_acquiring_load_exclusive(n));
9728 match(Set prev (GetAndSetI mem newv));
9729 ins_cost(VOLATILE_REF_COST);
9730 format %{ "atomic_xchgw_acq $prev, $newv, [$mem]" %}
9731 ins_encode %{
9732 __ atomic_xchgalw($prev$$Register, $newv$$Register, as_Register($mem$$base));
9733 %}
9734 ins_pipe(pipe_serial);
9735 %}
9736
9737 instruct get_and_setLAcq(indirect mem, iRegL newv, iRegLNoSp prev) %{
9738 predicate(needs_acquiring_load_exclusive(n));
9739 match(Set prev (GetAndSetL mem newv));
9740 ins_cost(VOLATILE_REF_COST);
9741 format %{ "atomic_xchg_acq $prev, $newv, [$mem]" %}
9742 ins_encode %{
9743 __ atomic_xchgal($prev$$Register, $newv$$Register, as_Register($mem$$base));
9744 %}
9745 ins_pipe(pipe_serial);
9746 %}
9747
9748 instruct get_and_setNAcq(indirect mem, iRegN newv, iRegINoSp prev) %{
9749 predicate(needs_acquiring_load_exclusive(n));
9750 match(Set prev (GetAndSetN mem newv));
9751 ins_cost(VOLATILE_REF_COST);
9752 format %{ "atomic_xchgw_acq $prev, $newv, [$mem]" %}
9753 ins_encode %{
9754 __ atomic_xchgalw($prev$$Register, $newv$$Register, as_Register($mem$$base));
9755 %}
9756 ins_pipe(pipe_serial);
9757 %}
9758
9759 instruct get_and_setPAcq(indirect mem, iRegP newv, iRegPNoSp prev) %{
9760 predicate(needs_acquiring_load_exclusive(n) && (n->as_LoadStore()->barrier_data() == 0));
9761 match(Set prev (GetAndSetP mem newv));
9762 ins_cost(VOLATILE_REF_COST);
9763 format %{ "atomic_xchg_acq $prev, $newv, [$mem]" %}
9764 ins_encode %{
9765 __ atomic_xchgal($prev$$Register, $newv$$Register, as_Register($mem$$base));
9766 %}
9767 ins_pipe(pipe_serial);
9768 %}
9769
9770
9771 instruct get_and_addL(indirect mem, iRegLNoSp newval, iRegL incr) %{
9772 match(Set newval (GetAndAddL mem incr));
9773 ins_cost(2 * VOLATILE_REF_COST + 1);
9774 format %{ "get_and_addL $newval, [$mem], $incr" %}
9775 ins_encode %{
9776 __ atomic_add($newval$$Register, $incr$$Register, as_Register($mem$$base));
9777 %}
9778 ins_pipe(pipe_serial);
9779 %}
9780
9781 instruct get_and_addL_no_res(indirect mem, Universe dummy, iRegL incr) %{
9782 predicate(n->as_LoadStore()->result_not_used());
9783 match(Set dummy (GetAndAddL mem incr));
9784 ins_cost(2 * VOLATILE_REF_COST);
9785 format %{ "get_and_addL [$mem], $incr" %}
9786 ins_encode %{
9787 __ atomic_add(noreg, $incr$$Register, as_Register($mem$$base));
9788 %}
9789 ins_pipe(pipe_serial);
9790 %}
9791
9792 instruct get_and_addLi(indirect mem, iRegLNoSp newval, immLAddSub incr) %{
9793 match(Set newval (GetAndAddL mem incr));
9794 ins_cost(2 * VOLATILE_REF_COST + 1);
9795 format %{ "get_and_addL $newval, [$mem], $incr" %}
9796 ins_encode %{
9797 __ atomic_add($newval$$Register, $incr$$constant, as_Register($mem$$base));
9798 %}
9799 ins_pipe(pipe_serial);
9800 %}
9801
9802 instruct get_and_addLi_no_res(indirect mem, Universe dummy, immLAddSub incr) %{
9803 predicate(n->as_LoadStore()->result_not_used());
9804 match(Set dummy (GetAndAddL mem incr));
9805 ins_cost(2 * VOLATILE_REF_COST);
9806 format %{ "get_and_addL [$mem], $incr" %}
9807 ins_encode %{
9808 __ atomic_add(noreg, $incr$$constant, as_Register($mem$$base));
9809 %}
9810 ins_pipe(pipe_serial);
9811 %}
9812
9813 instruct get_and_addI(indirect mem, iRegINoSp newval, iRegIorL2I incr) %{
9814 match(Set newval (GetAndAddI mem incr));
9815 ins_cost(2 * VOLATILE_REF_COST + 1);
9816 format %{ "get_and_addI $newval, [$mem], $incr" %}
9817 ins_encode %{
9818 __ atomic_addw($newval$$Register, $incr$$Register, as_Register($mem$$base));
9819 %}
9820 ins_pipe(pipe_serial);
9821 %}
9822
9823 instruct get_and_addI_no_res(indirect mem, Universe dummy, iRegIorL2I incr) %{
9824 predicate(n->as_LoadStore()->result_not_used());
9825 match(Set dummy (GetAndAddI mem incr));
9826 ins_cost(2 * VOLATILE_REF_COST);
9827 format %{ "get_and_addI [$mem], $incr" %}
9828 ins_encode %{
9829 __ atomic_addw(noreg, $incr$$Register, as_Register($mem$$base));
9830 %}
9831 ins_pipe(pipe_serial);
9832 %}
9833
9834 instruct get_and_addIi(indirect mem, iRegINoSp newval, immIAddSub incr) %{
9835 match(Set newval (GetAndAddI mem incr));
9836 ins_cost(2 * VOLATILE_REF_COST + 1);
9837 format %{ "get_and_addI $newval, [$mem], $incr" %}
9838 ins_encode %{
9839 __ atomic_addw($newval$$Register, $incr$$constant, as_Register($mem$$base));
9840 %}
9841 ins_pipe(pipe_serial);
9842 %}
9843
9844 instruct get_and_addIi_no_res(indirect mem, Universe dummy, immIAddSub incr) %{
9845 predicate(n->as_LoadStore()->result_not_used());
9846 match(Set dummy (GetAndAddI mem incr));
9847 ins_cost(2 * VOLATILE_REF_COST);
9848 format %{ "get_and_addI [$mem], $incr" %}
9849 ins_encode %{
9850 __ atomic_addw(noreg, $incr$$constant, as_Register($mem$$base));
9851 %}
9852 ins_pipe(pipe_serial);
9853 %}
9854
9855 instruct get_and_addLAcq(indirect mem, iRegLNoSp newval, iRegL incr) %{
9856 predicate(needs_acquiring_load_exclusive(n));
9857 match(Set newval (GetAndAddL mem incr));
9858 ins_cost(VOLATILE_REF_COST + 1);
9859 format %{ "get_and_addL_acq $newval, [$mem], $incr" %}
9860 ins_encode %{
9861 __ atomic_addal($newval$$Register, $incr$$Register, as_Register($mem$$base));
9862 %}
9863 ins_pipe(pipe_serial);
9864 %}
9865
9866 instruct get_and_addL_no_resAcq(indirect mem, Universe dummy, iRegL incr) %{
9867 predicate(n->as_LoadStore()->result_not_used() && needs_acquiring_load_exclusive(n));
9868 match(Set dummy (GetAndAddL mem incr));
9869 ins_cost(VOLATILE_REF_COST);
9870 format %{ "get_and_addL_acq [$mem], $incr" %}
9871 ins_encode %{
9872 __ atomic_addal(noreg, $incr$$Register, as_Register($mem$$base));
9873 %}
9874 ins_pipe(pipe_serial);
9875 %}
9876
9877 instruct get_and_addLiAcq(indirect mem, iRegLNoSp newval, immLAddSub incr) %{
9878 predicate(needs_acquiring_load_exclusive(n));
9879 match(Set newval (GetAndAddL mem incr));
9880 ins_cost(VOLATILE_REF_COST + 1);
9881 format %{ "get_and_addL_acq $newval, [$mem], $incr" %}
9882 ins_encode %{
9883 __ atomic_addal($newval$$Register, $incr$$constant, as_Register($mem$$base));
9884 %}
9885 ins_pipe(pipe_serial);
9886 %}
9887
9888 instruct get_and_addLi_no_resAcq(indirect mem, Universe dummy, immLAddSub incr) %{
9889 predicate(n->as_LoadStore()->result_not_used() && needs_acquiring_load_exclusive(n));
9890 match(Set dummy (GetAndAddL mem incr));
9891 ins_cost(VOLATILE_REF_COST);
9892 format %{ "get_and_addL_acq [$mem], $incr" %}
9893 ins_encode %{
9894 __ atomic_addal(noreg, $incr$$constant, as_Register($mem$$base));
9895 %}
9896 ins_pipe(pipe_serial);
9897 %}
9898
9899 instruct get_and_addIAcq(indirect mem, iRegINoSp newval, iRegIorL2I incr) %{
9900 predicate(needs_acquiring_load_exclusive(n));
9901 match(Set newval (GetAndAddI mem incr));
9902 ins_cost(VOLATILE_REF_COST + 1);
9903 format %{ "get_and_addI_acq $newval, [$mem], $incr" %}
9904 ins_encode %{
9905 __ atomic_addalw($newval$$Register, $incr$$Register, as_Register($mem$$base));
9906 %}
9907 ins_pipe(pipe_serial);
9908 %}
9909
9910 instruct get_and_addI_no_resAcq(indirect mem, Universe dummy, iRegIorL2I incr) %{
9911 predicate(n->as_LoadStore()->result_not_used() && needs_acquiring_load_exclusive(n));
9912 match(Set dummy (GetAndAddI mem incr));
9913 ins_cost(VOLATILE_REF_COST);
9914 format %{ "get_and_addI_acq [$mem], $incr" %}
9915 ins_encode %{
9916 __ atomic_addalw(noreg, $incr$$Register, as_Register($mem$$base));
9917 %}
9918 ins_pipe(pipe_serial);
9919 %}
9920
9921 instruct get_and_addIiAcq(indirect mem, iRegINoSp newval, immIAddSub incr) %{
9922 predicate(needs_acquiring_load_exclusive(n));
9923 match(Set newval (GetAndAddI mem incr));
9924 ins_cost(VOLATILE_REF_COST + 1);
9925 format %{ "get_and_addI_acq $newval, [$mem], $incr" %}
9926 ins_encode %{
9927 __ atomic_addalw($newval$$Register, $incr$$constant, as_Register($mem$$base));
9928 %}
9929 ins_pipe(pipe_serial);
9930 %}
9931
9932 instruct get_and_addIi_no_resAcq(indirect mem, Universe dummy, immIAddSub incr) %{
9933 predicate(n->as_LoadStore()->result_not_used() && needs_acquiring_load_exclusive(n));
9934 match(Set dummy (GetAndAddI mem incr));
9935 ins_cost(VOLATILE_REF_COST);
9936 format %{ "get_and_addI_acq [$mem], $incr" %}
9937 ins_encode %{
9938 __ atomic_addalw(noreg, $incr$$constant, as_Register($mem$$base));
9939 %}
9940 ins_pipe(pipe_serial);
9941 %}
9942
9943 // Manifest a CmpL result in an integer register.
9944 // (src1 < src2) ? -1 : ((src1 > src2) ? 1 : 0)
9945 instruct cmpL3_reg_reg(iRegINoSp dst, iRegL src1, iRegL src2, rFlagsReg flags)
9946 %{
9947 match(Set dst (CmpL3 src1 src2));
9948 effect(KILL flags);
9949
9950 ins_cost(INSN_COST * 6);
9951 format %{
9952 "cmp $src1, $src2"
9953 "csetw $dst, ne"
9954 "cnegw $dst, lt"
9955 %}
9956 // format %{ "CmpL3 $dst, $src1, $src2" %}
9957 ins_encode %{
9958 __ cmp($src1$$Register, $src2$$Register);
9959 __ csetw($dst$$Register, Assembler::NE);
9960 __ cnegw($dst$$Register, $dst$$Register, Assembler::LT);
9961 %}
9962
9963 ins_pipe(pipe_class_default);
9964 %}
9965
9966 instruct cmpL3_reg_imm(iRegINoSp dst, iRegL src1, immLAddSub src2, rFlagsReg flags)
9967 %{
9968 match(Set dst (CmpL3 src1 src2));
9969 effect(KILL flags);
9970
9971 ins_cost(INSN_COST * 6);
9972 format %{
9973 "cmp $src1, $src2"
9974 "csetw $dst, ne"
9975 "cnegw $dst, lt"
9976 %}
9977 ins_encode %{
9978 int32_t con = (int32_t)$src2$$constant;
9979 if (con < 0) {
9980 __ adds(zr, $src1$$Register, -con);
9981 } else {
9982 __ subs(zr, $src1$$Register, con);
9983 }
9984 __ csetw($dst$$Register, Assembler::NE);
9985 __ cnegw($dst$$Register, $dst$$Register, Assembler::LT);
9986 %}
9987
9988 ins_pipe(pipe_class_default);
9989 %}
9990
9991 // ============================================================================
9992 // Conditional Move Instructions
9993
9994 // n.b. we have identical rules for both a signed compare op (cmpOp)
9995 // and an unsigned compare op (cmpOpU). it would be nice if we could
9996 // define an op class which merged both inputs and use it to type the
9997 // argument to a single rule. unfortunatelyt his fails because the
9998 // opclass does not live up to the COND_INTER interface of its
9999 // component operands. When the generic code tries to negate the
10000 // operand it ends up running the generci Machoper::negate method
10001 // which throws a ShouldNotHappen. So, we have to provide two flavours
10002 // of each rule, one for a cmpOp and a second for a cmpOpU (sigh).
10003
10004 instruct cmovI_reg_reg(cmpOp cmp, rFlagsReg cr, iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10005 match(Set dst (CMoveI (Binary cmp cr) (Binary src1 src2)));
10006
10007 ins_cost(INSN_COST * 2);
10008 format %{ "cselw $dst, $src2, $src1 $cmp\t# signed, int" %}
10009
10010 ins_encode %{
10011 __ cselw(as_Register($dst$$reg),
10012 as_Register($src2$$reg),
10013 as_Register($src1$$reg),
10014 (Assembler::Condition)$cmp$$cmpcode);
10015 %}
10016
10017 ins_pipe(icond_reg_reg);
10018 %}
10019
10020 instruct cmovUI_reg_reg(cmpOpU cmp, rFlagsRegU cr, iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10021 match(Set dst (CMoveI (Binary cmp cr) (Binary src1 src2)));
10022
10023 ins_cost(INSN_COST * 2);
10024 format %{ "cselw $dst, $src2, $src1 $cmp\t# unsigned, int" %}
10025
10026 ins_encode %{
10027 __ cselw(as_Register($dst$$reg),
10028 as_Register($src2$$reg),
10029 as_Register($src1$$reg),
10030 (Assembler::Condition)$cmp$$cmpcode);
10031 %}
10032
10033 ins_pipe(icond_reg_reg);
10034 %}
10035
10036 // special cases where one arg is zero
10037
10038 // n.b. this is selected in preference to the rule above because it
10039 // avoids loading constant 0 into a source register
10040
10041 // TODO
10042 // we ought only to be able to cull one of these variants as the ideal
10043 // transforms ought always to order the zero consistently (to left/right?)
10044
10045 instruct cmovI_zero_reg(cmpOp cmp, rFlagsReg cr, iRegINoSp dst, immI0 zero, iRegIorL2I src) %{
10046 match(Set dst (CMoveI (Binary cmp cr) (Binary zero src)));
10047
10048 ins_cost(INSN_COST * 2);
10049 format %{ "cselw $dst, $src, zr $cmp\t# signed, int" %}
10050
10051 ins_encode %{
10052 __ cselw(as_Register($dst$$reg),
10053 as_Register($src$$reg),
10054 zr,
10055 (Assembler::Condition)$cmp$$cmpcode);
10056 %}
10057
10058 ins_pipe(icond_reg);
10059 %}
10060
10061 instruct cmovUI_zero_reg(cmpOpU cmp, rFlagsRegU cr, iRegINoSp dst, immI0 zero, iRegIorL2I src) %{
10062 match(Set dst (CMoveI (Binary cmp cr) (Binary zero src)));
10063
10064 ins_cost(INSN_COST * 2);
10065 format %{ "cselw $dst, $src, zr $cmp\t# unsigned, int" %}
10066
10067 ins_encode %{
10068 __ cselw(as_Register($dst$$reg),
10069 as_Register($src$$reg),
10070 zr,
10071 (Assembler::Condition)$cmp$$cmpcode);
10072 %}
10073
10074 ins_pipe(icond_reg);
10075 %}
10076
10077 instruct cmovI_reg_zero(cmpOp cmp, rFlagsReg cr, iRegINoSp dst, iRegIorL2I src, immI0 zero) %{
10078 match(Set dst (CMoveI (Binary cmp cr) (Binary src zero)));
10079
10080 ins_cost(INSN_COST * 2);
10081 format %{ "cselw $dst, zr, $src $cmp\t# signed, int" %}
10082
10083 ins_encode %{
10084 __ cselw(as_Register($dst$$reg),
10085 zr,
10086 as_Register($src$$reg),
10087 (Assembler::Condition)$cmp$$cmpcode);
10088 %}
10089
10090 ins_pipe(icond_reg);
10091 %}
10092
10093 instruct cmovUI_reg_zero(cmpOpU cmp, rFlagsRegU cr, iRegINoSp dst, iRegIorL2I src, immI0 zero) %{
10094 match(Set dst (CMoveI (Binary cmp cr) (Binary src zero)));
10095
10096 ins_cost(INSN_COST * 2);
10097 format %{ "cselw $dst, zr, $src $cmp\t# unsigned, int" %}
10098
10099 ins_encode %{
10100 __ cselw(as_Register($dst$$reg),
10101 zr,
10102 as_Register($src$$reg),
10103 (Assembler::Condition)$cmp$$cmpcode);
10104 %}
10105
10106 ins_pipe(icond_reg);
10107 %}
10108
10109 // special case for creating a boolean 0 or 1
10110
10111 // n.b. this is selected in preference to the rule above because it
10112 // avoids loading constants 0 and 1 into a source register
10113
10114 instruct cmovI_reg_zero_one(cmpOp cmp, rFlagsReg cr, iRegINoSp dst, immI0 zero, immI_1 one) %{
10115 match(Set dst (CMoveI (Binary cmp cr) (Binary one zero)));
10116
10117 ins_cost(INSN_COST * 2);
10118 format %{ "csincw $dst, zr, zr $cmp\t# signed, int" %}
10119
10120 ins_encode %{
10121 // equivalently
10122 // cset(as_Register($dst$$reg),
10123 // negate_condition((Assembler::Condition)$cmp$$cmpcode));
10124 __ csincw(as_Register($dst$$reg),
10125 zr,
10126 zr,
10127 (Assembler::Condition)$cmp$$cmpcode);
10128 %}
10129
10130 ins_pipe(icond_none);
10131 %}
10132
10133 instruct cmovUI_reg_zero_one(cmpOpU cmp, rFlagsRegU cr, iRegINoSp dst, immI0 zero, immI_1 one) %{
10134 match(Set dst (CMoveI (Binary cmp cr) (Binary one zero)));
10135
10136 ins_cost(INSN_COST * 2);
10137 format %{ "csincw $dst, zr, zr $cmp\t# unsigned, int" %}
10138
10139 ins_encode %{
10140 // equivalently
10141 // cset(as_Register($dst$$reg),
10142 // negate_condition((Assembler::Condition)$cmp$$cmpcode));
10143 __ csincw(as_Register($dst$$reg),
10144 zr,
10145 zr,
10146 (Assembler::Condition)$cmp$$cmpcode);
10147 %}
10148
10149 ins_pipe(icond_none);
10150 %}
10151
10152 instruct cmovL_reg_reg(cmpOp cmp, rFlagsReg cr, iRegLNoSp dst, iRegL src1, iRegL src2) %{
10153 match(Set dst (CMoveL (Binary cmp cr) (Binary src1 src2)));
10154
10155 ins_cost(INSN_COST * 2);
10156 format %{ "csel $dst, $src2, $src1 $cmp\t# signed, long" %}
10157
10158 ins_encode %{
10159 __ csel(as_Register($dst$$reg),
10160 as_Register($src2$$reg),
10161 as_Register($src1$$reg),
10162 (Assembler::Condition)$cmp$$cmpcode);
10163 %}
10164
10165 ins_pipe(icond_reg_reg);
10166 %}
10167
10168 instruct cmovUL_reg_reg(cmpOpU cmp, rFlagsRegU cr, iRegLNoSp dst, iRegL src1, iRegL src2) %{
10169 match(Set dst (CMoveL (Binary cmp cr) (Binary src1 src2)));
10170
10171 ins_cost(INSN_COST * 2);
10172 format %{ "csel $dst, $src2, $src1 $cmp\t# unsigned, long" %}
10173
10174 ins_encode %{
10175 __ csel(as_Register($dst$$reg),
10176 as_Register($src2$$reg),
10177 as_Register($src1$$reg),
10178 (Assembler::Condition)$cmp$$cmpcode);
10179 %}
10180
10181 ins_pipe(icond_reg_reg);
10182 %}
10183
10184 // special cases where one arg is zero
10185
10186 instruct cmovL_reg_zero(cmpOp cmp, rFlagsReg cr, iRegLNoSp dst, iRegL src, immL0 zero) %{
10187 match(Set dst (CMoveL (Binary cmp cr) (Binary src zero)));
10188
10189 ins_cost(INSN_COST * 2);
10190 format %{ "csel $dst, zr, $src $cmp\t# signed, long" %}
10191
10192 ins_encode %{
10193 __ csel(as_Register($dst$$reg),
10194 zr,
10195 as_Register($src$$reg),
10196 (Assembler::Condition)$cmp$$cmpcode);
10197 %}
10198
10199 ins_pipe(icond_reg);
10200 %}
10201
10202 instruct cmovUL_reg_zero(cmpOpU cmp, rFlagsRegU cr, iRegLNoSp dst, iRegL src, immL0 zero) %{
10203 match(Set dst (CMoveL (Binary cmp cr) (Binary src zero)));
10204
10205 ins_cost(INSN_COST * 2);
10206 format %{ "csel $dst, zr, $src $cmp\t# unsigned, long" %}
10207
10208 ins_encode %{
10209 __ csel(as_Register($dst$$reg),
10210 zr,
10211 as_Register($src$$reg),
10212 (Assembler::Condition)$cmp$$cmpcode);
10213 %}
10214
10215 ins_pipe(icond_reg);
10216 %}
10217
10218 instruct cmovL_zero_reg(cmpOp cmp, rFlagsReg cr, iRegLNoSp dst, immL0 zero, iRegL src) %{
10219 match(Set dst (CMoveL (Binary cmp cr) (Binary zero src)));
10220
10221 ins_cost(INSN_COST * 2);
10222 format %{ "csel $dst, $src, zr $cmp\t# signed, long" %}
10223
10224 ins_encode %{
10225 __ csel(as_Register($dst$$reg),
10226 as_Register($src$$reg),
10227 zr,
10228 (Assembler::Condition)$cmp$$cmpcode);
10229 %}
10230
10231 ins_pipe(icond_reg);
10232 %}
10233
10234 instruct cmovUL_zero_reg(cmpOpU cmp, rFlagsRegU cr, iRegLNoSp dst, immL0 zero, iRegL src) %{
10235 match(Set dst (CMoveL (Binary cmp cr) (Binary zero src)));
10236
10237 ins_cost(INSN_COST * 2);
10238 format %{ "csel $dst, $src, zr $cmp\t# unsigned, long" %}
10239
10240 ins_encode %{
10241 __ csel(as_Register($dst$$reg),
10242 as_Register($src$$reg),
10243 zr,
10244 (Assembler::Condition)$cmp$$cmpcode);
10245 %}
10246
10247 ins_pipe(icond_reg);
10248 %}
10249
10250 instruct cmovP_reg_reg(cmpOp cmp, rFlagsReg cr, iRegPNoSp dst, iRegP src1, iRegP src2) %{
10251 match(Set dst (CMoveP (Binary cmp cr) (Binary src1 src2)));
10252
10253 ins_cost(INSN_COST * 2);
10254 format %{ "csel $dst, $src2, $src1 $cmp\t# signed, ptr" %}
10255
10256 ins_encode %{
10257 __ csel(as_Register($dst$$reg),
10258 as_Register($src2$$reg),
10259 as_Register($src1$$reg),
10260 (Assembler::Condition)$cmp$$cmpcode);
10261 %}
10262
10263 ins_pipe(icond_reg_reg);
10264 %}
10265
10266 instruct cmovUP_reg_reg(cmpOpU cmp, rFlagsRegU cr, iRegPNoSp dst, iRegP src1, iRegP src2) %{
10267 match(Set dst (CMoveP (Binary cmp cr) (Binary src1 src2)));
10268
10269 ins_cost(INSN_COST * 2);
10270 format %{ "csel $dst, $src2, $src1 $cmp\t# unsigned, ptr" %}
10271
10272 ins_encode %{
10273 __ csel(as_Register($dst$$reg),
10274 as_Register($src2$$reg),
10275 as_Register($src1$$reg),
10276 (Assembler::Condition)$cmp$$cmpcode);
10277 %}
10278
10279 ins_pipe(icond_reg_reg);
10280 %}
10281
10282 // special cases where one arg is zero
10283
10284 instruct cmovP_reg_zero(cmpOp cmp, rFlagsReg cr, iRegPNoSp dst, iRegP src, immP0 zero) %{
10285 match(Set dst (CMoveP (Binary cmp cr) (Binary src zero)));
10286
10287 ins_cost(INSN_COST * 2);
10288 format %{ "csel $dst, zr, $src $cmp\t# signed, ptr" %}
10289
10290 ins_encode %{
10291 __ csel(as_Register($dst$$reg),
10292 zr,
10293 as_Register($src$$reg),
10294 (Assembler::Condition)$cmp$$cmpcode);
10295 %}
10296
10297 ins_pipe(icond_reg);
10298 %}
10299
10300 instruct cmovUP_reg_zero(cmpOpU cmp, rFlagsRegU cr, iRegPNoSp dst, iRegP src, immP0 zero) %{
10301 match(Set dst (CMoveP (Binary cmp cr) (Binary src zero)));
10302
10303 ins_cost(INSN_COST * 2);
10304 format %{ "csel $dst, zr, $src $cmp\t# unsigned, ptr" %}
10305
10306 ins_encode %{
10307 __ csel(as_Register($dst$$reg),
10308 zr,
10309 as_Register($src$$reg),
10310 (Assembler::Condition)$cmp$$cmpcode);
10311 %}
10312
10313 ins_pipe(icond_reg);
10314 %}
10315
10316 instruct cmovP_zero_reg(cmpOp cmp, rFlagsReg cr, iRegPNoSp dst, immP0 zero, iRegP src) %{
10317 match(Set dst (CMoveP (Binary cmp cr) (Binary zero src)));
10318
10319 ins_cost(INSN_COST * 2);
10320 format %{ "csel $dst, $src, zr $cmp\t# signed, ptr" %}
10321
10322 ins_encode %{
10323 __ csel(as_Register($dst$$reg),
10324 as_Register($src$$reg),
10325 zr,
10326 (Assembler::Condition)$cmp$$cmpcode);
10327 %}
10328
10329 ins_pipe(icond_reg);
10330 %}
10331
10332 instruct cmovUP_zero_reg(cmpOpU cmp, rFlagsRegU cr, iRegPNoSp dst, immP0 zero, iRegP src) %{
10333 match(Set dst (CMoveP (Binary cmp cr) (Binary zero src)));
10334
10335 ins_cost(INSN_COST * 2);
10336 format %{ "csel $dst, $src, zr $cmp\t# unsigned, ptr" %}
10337
10338 ins_encode %{
10339 __ csel(as_Register($dst$$reg),
10340 as_Register($src$$reg),
10341 zr,
10342 (Assembler::Condition)$cmp$$cmpcode);
10343 %}
10344
10345 ins_pipe(icond_reg);
10346 %}
10347
10348 instruct cmovN_reg_reg(cmpOp cmp, rFlagsReg cr, iRegNNoSp dst, iRegN src1, iRegN src2) %{
10349 match(Set dst (CMoveN (Binary cmp cr) (Binary src1 src2)));
10350
10351 ins_cost(INSN_COST * 2);
10352 format %{ "cselw $dst, $src2, $src1 $cmp\t# signed, compressed ptr" %}
10353
10354 ins_encode %{
10355 __ cselw(as_Register($dst$$reg),
10356 as_Register($src2$$reg),
10357 as_Register($src1$$reg),
10358 (Assembler::Condition)$cmp$$cmpcode);
10359 %}
10360
10361 ins_pipe(icond_reg_reg);
10362 %}
10363
10364 instruct cmovUN_reg_reg(cmpOpU cmp, rFlagsRegU cr, iRegNNoSp dst, iRegN src1, iRegN src2) %{
10365 match(Set dst (CMoveN (Binary cmp cr) (Binary src1 src2)));
10366
10367 ins_cost(INSN_COST * 2);
10368 format %{ "cselw $dst, $src2, $src1 $cmp\t# signed, compressed ptr" %}
10369
10370 ins_encode %{
10371 __ cselw(as_Register($dst$$reg),
10372 as_Register($src2$$reg),
10373 as_Register($src1$$reg),
10374 (Assembler::Condition)$cmp$$cmpcode);
10375 %}
10376
10377 ins_pipe(icond_reg_reg);
10378 %}
10379
10380 // special cases where one arg is zero
10381
10382 instruct cmovN_reg_zero(cmpOp cmp, rFlagsReg cr, iRegNNoSp dst, iRegN src, immN0 zero) %{
10383 match(Set dst (CMoveN (Binary cmp cr) (Binary src zero)));
10384
10385 ins_cost(INSN_COST * 2);
10386 format %{ "cselw $dst, zr, $src $cmp\t# signed, compressed ptr" %}
10387
10388 ins_encode %{
10389 __ cselw(as_Register($dst$$reg),
10390 zr,
10391 as_Register($src$$reg),
10392 (Assembler::Condition)$cmp$$cmpcode);
10393 %}
10394
10395 ins_pipe(icond_reg);
10396 %}
10397
10398 instruct cmovUN_reg_zero(cmpOpU cmp, rFlagsRegU cr, iRegNNoSp dst, iRegN src, immN0 zero) %{
10399 match(Set dst (CMoveN (Binary cmp cr) (Binary src zero)));
10400
10401 ins_cost(INSN_COST * 2);
10402 format %{ "cselw $dst, zr, $src $cmp\t# unsigned, compressed ptr" %}
10403
10404 ins_encode %{
10405 __ cselw(as_Register($dst$$reg),
10406 zr,
10407 as_Register($src$$reg),
10408 (Assembler::Condition)$cmp$$cmpcode);
10409 %}
10410
10411 ins_pipe(icond_reg);
10412 %}
10413
10414 instruct cmovN_zero_reg(cmpOp cmp, rFlagsReg cr, iRegNNoSp dst, immN0 zero, iRegN src) %{
10415 match(Set dst (CMoveN (Binary cmp cr) (Binary zero src)));
10416
10417 ins_cost(INSN_COST * 2);
10418 format %{ "cselw $dst, $src, zr $cmp\t# signed, compressed ptr" %}
10419
10420 ins_encode %{
10421 __ cselw(as_Register($dst$$reg),
10422 as_Register($src$$reg),
10423 zr,
10424 (Assembler::Condition)$cmp$$cmpcode);
10425 %}
10426
10427 ins_pipe(icond_reg);
10428 %}
10429
10430 instruct cmovUN_zero_reg(cmpOpU cmp, rFlagsRegU cr, iRegNNoSp dst, immN0 zero, iRegN src) %{
10431 match(Set dst (CMoveN (Binary cmp cr) (Binary zero src)));
10432
10433 ins_cost(INSN_COST * 2);
10434 format %{ "cselw $dst, $src, zr $cmp\t# unsigned, compressed ptr" %}
10435
10436 ins_encode %{
10437 __ cselw(as_Register($dst$$reg),
10438 as_Register($src$$reg),
10439 zr,
10440 (Assembler::Condition)$cmp$$cmpcode);
10441 %}
10442
10443 ins_pipe(icond_reg);
10444 %}
10445
10446 instruct cmovF_reg(cmpOp cmp, rFlagsReg cr, vRegF dst, vRegF src1, vRegF src2)
10447 %{
10448 match(Set dst (CMoveF (Binary cmp cr) (Binary src1 src2)));
10449
10450 ins_cost(INSN_COST * 3);
10451
10452 format %{ "fcsels $dst, $src1, $src2, $cmp\t# signed cmove float\n\t" %}
10453 ins_encode %{
10454 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
10455 __ fcsels(as_FloatRegister($dst$$reg),
10456 as_FloatRegister($src2$$reg),
10457 as_FloatRegister($src1$$reg),
10458 cond);
10459 %}
10460
10461 ins_pipe(fp_cond_reg_reg_s);
10462 %}
10463
10464 instruct cmovUF_reg(cmpOpU cmp, rFlagsRegU cr, vRegF dst, vRegF src1, vRegF src2)
10465 %{
10466 match(Set dst (CMoveF (Binary cmp cr) (Binary src1 src2)));
10467
10468 ins_cost(INSN_COST * 3);
10469
10470 format %{ "fcsels $dst, $src1, $src2, $cmp\t# unsigned cmove float\n\t" %}
10471 ins_encode %{
10472 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
10473 __ fcsels(as_FloatRegister($dst$$reg),
10474 as_FloatRegister($src2$$reg),
10475 as_FloatRegister($src1$$reg),
10476 cond);
10477 %}
10478
10479 ins_pipe(fp_cond_reg_reg_s);
10480 %}
10481
10482 instruct cmovD_reg(cmpOp cmp, rFlagsReg cr, vRegD dst, vRegD src1, vRegD src2)
10483 %{
10484 match(Set dst (CMoveD (Binary cmp cr) (Binary src1 src2)));
10485
10486 ins_cost(INSN_COST * 3);
10487
10488 format %{ "fcseld $dst, $src1, $src2, $cmp\t# signed cmove float\n\t" %}
10489 ins_encode %{
10490 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
10491 __ fcseld(as_FloatRegister($dst$$reg),
10492 as_FloatRegister($src2$$reg),
10493 as_FloatRegister($src1$$reg),
10494 cond);
10495 %}
10496
10497 ins_pipe(fp_cond_reg_reg_d);
10498 %}
10499
10500 instruct cmovUD_reg(cmpOpU cmp, rFlagsRegU cr, vRegD dst, vRegD src1, vRegD src2)
10501 %{
10502 match(Set dst (CMoveD (Binary cmp cr) (Binary src1 src2)));
10503
10504 ins_cost(INSN_COST * 3);
10505
10506 format %{ "fcseld $dst, $src1, $src2, $cmp\t# unsigned cmove float\n\t" %}
10507 ins_encode %{
10508 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
10509 __ fcseld(as_FloatRegister($dst$$reg),
10510 as_FloatRegister($src2$$reg),
10511 as_FloatRegister($src1$$reg),
10512 cond);
10513 %}
10514
10515 ins_pipe(fp_cond_reg_reg_d);
10516 %}
10517
10518 // ============================================================================
10519 // Arithmetic Instructions
10520 //
10521
10522 // Integer Addition
10523
10524 // TODO
10525 // these currently employ operations which do not set CR and hence are
10526 // not flagged as killing CR but we would like to isolate the cases
10527 // where we want to set flags from those where we don't. need to work
10528 // out how to do that.
10529
10530 instruct addI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10531 match(Set dst (AddI src1 src2));
10532
10533 ins_cost(INSN_COST);
10534 format %{ "addw $dst, $src1, $src2" %}
10535
10536 ins_encode %{
10537 __ addw(as_Register($dst$$reg),
10538 as_Register($src1$$reg),
10539 as_Register($src2$$reg));
10540 %}
10541
10542 ins_pipe(ialu_reg_reg);
10543 %}
10544
10545 instruct addI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immIAddSub src2) %{
10546 match(Set dst (AddI src1 src2));
10547
10548 ins_cost(INSN_COST);
10549 format %{ "addw $dst, $src1, $src2" %}
10550
10551 // use opcode to indicate that this is an add not a sub
10552 opcode(0x0);
10553
10554 ins_encode(aarch64_enc_addsubw_imm(dst, src1, src2));
10555
10556 ins_pipe(ialu_reg_imm);
10557 %}
10558
10559 instruct addI_reg_imm_i2l(iRegINoSp dst, iRegL src1, immIAddSub src2) %{
10560 match(Set dst (AddI (ConvL2I src1) src2));
10561
10562 ins_cost(INSN_COST);
10563 format %{ "addw $dst, $src1, $src2" %}
10564
10565 // use opcode to indicate that this is an add not a sub
10566 opcode(0x0);
10567
10568 ins_encode(aarch64_enc_addsubw_imm(dst, src1, src2));
10569
10570 ins_pipe(ialu_reg_imm);
10571 %}
10572
10573 // Pointer Addition
10574 instruct addP_reg_reg(iRegPNoSp dst, iRegP src1, iRegL src2) %{
10575 match(Set dst (AddP src1 src2));
10576
10577 ins_cost(INSN_COST);
10578 format %{ "add $dst, $src1, $src2\t# ptr" %}
10579
10580 ins_encode %{
10581 __ add(as_Register($dst$$reg),
10582 as_Register($src1$$reg),
10583 as_Register($src2$$reg));
10584 %}
10585
10586 ins_pipe(ialu_reg_reg);
10587 %}
10588
10589 instruct addP_reg_reg_ext(iRegPNoSp dst, iRegP src1, iRegIorL2I src2) %{
10590 match(Set dst (AddP src1 (ConvI2L src2)));
10591
10592 ins_cost(1.9 * INSN_COST);
10593 format %{ "add $dst, $src1, $src2, sxtw\t# ptr" %}
10594
10595 ins_encode %{
10596 __ add(as_Register($dst$$reg),
10597 as_Register($src1$$reg),
10598 as_Register($src2$$reg), ext::sxtw);
10599 %}
10600
10601 ins_pipe(ialu_reg_reg);
10602 %}
10603
10604 instruct addP_reg_reg_lsl(iRegPNoSp dst, iRegP src1, iRegL src2, immIScale scale) %{
10605 match(Set dst (AddP src1 (LShiftL src2 scale)));
10606
10607 ins_cost(1.9 * INSN_COST);
10608 format %{ "add $dst, $src1, $src2, LShiftL $scale\t# ptr" %}
10609
10610 ins_encode %{
10611 __ lea(as_Register($dst$$reg),
10612 Address(as_Register($src1$$reg), as_Register($src2$$reg),
10613 Address::lsl($scale$$constant)));
10614 %}
10615
10616 ins_pipe(ialu_reg_reg_shift);
10617 %}
10618
10619 instruct addP_reg_reg_ext_shift(iRegPNoSp dst, iRegP src1, iRegIorL2I src2, immIScale scale) %{
10620 match(Set dst (AddP src1 (LShiftL (ConvI2L src2) scale)));
10621
10622 ins_cost(1.9 * INSN_COST);
10623 format %{ "add $dst, $src1, $src2, I2L $scale\t# ptr" %}
10624
10625 ins_encode %{
10626 __ lea(as_Register($dst$$reg),
10627 Address(as_Register($src1$$reg), as_Register($src2$$reg),
10628 Address::sxtw($scale$$constant)));
10629 %}
10630
10631 ins_pipe(ialu_reg_reg_shift);
10632 %}
10633
10634 instruct lshift_ext(iRegLNoSp dst, iRegIorL2I src, immI scale, rFlagsReg cr) %{
10635 match(Set dst (LShiftL (ConvI2L src) scale));
10636
10637 ins_cost(INSN_COST);
10638 format %{ "sbfiz $dst, $src, $scale & 63, -$scale & 63\t" %}
10639
10640 ins_encode %{
10641 __ sbfiz(as_Register($dst$$reg),
10642 as_Register($src$$reg),
10643 $scale$$constant & 63, MIN2(32, (int)((-$scale$$constant) & 63)));
10644 %}
10645
10646 ins_pipe(ialu_reg_shift);
10647 %}
10648
10649 // Pointer Immediate Addition
10650 // n.b. this needs to be more expensive than using an indirect memory
10651 // operand
10652 instruct addP_reg_imm(iRegPNoSp dst, iRegP src1, immLAddSub src2) %{
10653 match(Set dst (AddP src1 src2));
10654
10655 ins_cost(INSN_COST);
10656 format %{ "add $dst, $src1, $src2\t# ptr" %}
10657
10658 // use opcode to indicate that this is an add not a sub
10659 opcode(0x0);
10660
10661 ins_encode( aarch64_enc_addsub_imm(dst, src1, src2) );
10662
10663 ins_pipe(ialu_reg_imm);
10664 %}
10665
10666 // Long Addition
10667 instruct addL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
10668
10669 match(Set dst (AddL src1 src2));
10670
10671 ins_cost(INSN_COST);
10672 format %{ "add $dst, $src1, $src2" %}
10673
10674 ins_encode %{
10675 __ add(as_Register($dst$$reg),
10676 as_Register($src1$$reg),
10677 as_Register($src2$$reg));
10678 %}
10679
10680 ins_pipe(ialu_reg_reg);
10681 %}
10682
10683 // No constant pool entries requiredLong Immediate Addition.
10684 instruct addL_reg_imm(iRegLNoSp dst, iRegL src1, immLAddSub src2) %{
10685 match(Set dst (AddL src1 src2));
10686
10687 ins_cost(INSN_COST);
10688 format %{ "add $dst, $src1, $src2" %}
10689
10690 // use opcode to indicate that this is an add not a sub
10691 opcode(0x0);
10692
10693 ins_encode( aarch64_enc_addsub_imm(dst, src1, src2) );
10694
10695 ins_pipe(ialu_reg_imm);
10696 %}
10697
10698 // Integer Subtraction
10699 instruct subI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10700 match(Set dst (SubI src1 src2));
10701
10702 ins_cost(INSN_COST);
10703 format %{ "subw $dst, $src1, $src2" %}
10704
10705 ins_encode %{
10706 __ subw(as_Register($dst$$reg),
10707 as_Register($src1$$reg),
10708 as_Register($src2$$reg));
10709 %}
10710
10711 ins_pipe(ialu_reg_reg);
10712 %}
10713
10714 // Immediate Subtraction
10715 instruct subI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immIAddSub src2) %{
10716 match(Set dst (SubI src1 src2));
10717
10718 ins_cost(INSN_COST);
10719 format %{ "subw $dst, $src1, $src2" %}
10720
10721 // use opcode to indicate that this is a sub not an add
10722 opcode(0x1);
10723
10724 ins_encode(aarch64_enc_addsubw_imm(dst, src1, src2));
10725
10726 ins_pipe(ialu_reg_imm);
10727 %}
10728
10729 // Long Subtraction
10730 instruct subL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
10731
10732 match(Set dst (SubL src1 src2));
10733
10734 ins_cost(INSN_COST);
10735 format %{ "sub $dst, $src1, $src2" %}
10736
10737 ins_encode %{
10738 __ sub(as_Register($dst$$reg),
10739 as_Register($src1$$reg),
10740 as_Register($src2$$reg));
10741 %}
10742
10743 ins_pipe(ialu_reg_reg);
10744 %}
10745
10746 // No constant pool entries requiredLong Immediate Subtraction.
10747 instruct subL_reg_imm(iRegLNoSp dst, iRegL src1, immLAddSub src2) %{
10748 match(Set dst (SubL src1 src2));
10749
10750 ins_cost(INSN_COST);
10751 format %{ "sub$dst, $src1, $src2" %}
10752
10753 // use opcode to indicate that this is a sub not an add
10754 opcode(0x1);
10755
10756 ins_encode( aarch64_enc_addsub_imm(dst, src1, src2) );
10757
10758 ins_pipe(ialu_reg_imm);
10759 %}
10760
10761 // Integer Negation (special case for sub)
10762
10763 instruct negI_reg(iRegINoSp dst, iRegIorL2I src, immI0 zero, rFlagsReg cr) %{
10764 match(Set dst (SubI zero src));
10765
10766 ins_cost(INSN_COST);
10767 format %{ "negw $dst, $src\t# int" %}
10768
10769 ins_encode %{
10770 __ negw(as_Register($dst$$reg),
10771 as_Register($src$$reg));
10772 %}
10773
10774 ins_pipe(ialu_reg);
10775 %}
10776
10777 // Long Negation
10778
10779 instruct negL_reg(iRegLNoSp dst, iRegL src, immL0 zero, rFlagsReg cr) %{
10780 match(Set dst (SubL zero src));
10781
10782 ins_cost(INSN_COST);
10783 format %{ "neg $dst, $src\t# long" %}
10784
10785 ins_encode %{
10786 __ neg(as_Register($dst$$reg),
10787 as_Register($src$$reg));
10788 %}
10789
10790 ins_pipe(ialu_reg);
10791 %}
10792
10793 // Integer Multiply
10794
10795 instruct mulI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10796 match(Set dst (MulI src1 src2));
10797
10798 ins_cost(INSN_COST * 3);
10799 format %{ "mulw $dst, $src1, $src2" %}
10800
10801 ins_encode %{
10802 __ mulw(as_Register($dst$$reg),
10803 as_Register($src1$$reg),
10804 as_Register($src2$$reg));
10805 %}
10806
10807 ins_pipe(imul_reg_reg);
10808 %}
10809
10810 instruct smulI(iRegLNoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10811 match(Set dst (MulL (ConvI2L src1) (ConvI2L src2)));
10812
10813 ins_cost(INSN_COST * 3);
10814 format %{ "smull $dst, $src1, $src2" %}
10815
10816 ins_encode %{
10817 __ smull(as_Register($dst$$reg),
10818 as_Register($src1$$reg),
10819 as_Register($src2$$reg));
10820 %}
10821
10822 ins_pipe(imul_reg_reg);
10823 %}
10824
10825 // Long Multiply
10826
10827 instruct mulL(iRegLNoSp dst, iRegL src1, iRegL src2) %{
10828 match(Set dst (MulL src1 src2));
10829
10830 ins_cost(INSN_COST * 5);
10831 format %{ "mul $dst, $src1, $src2" %}
10832
10833 ins_encode %{
10834 __ mul(as_Register($dst$$reg),
10835 as_Register($src1$$reg),
10836 as_Register($src2$$reg));
10837 %}
10838
10839 ins_pipe(lmul_reg_reg);
10840 %}
10841
10842 instruct mulHiL_rReg(iRegLNoSp dst, iRegL src1, iRegL src2, rFlagsReg cr)
10843 %{
10844 match(Set dst (MulHiL src1 src2));
10845
10846 ins_cost(INSN_COST * 7);
10847 format %{ "smulh $dst, $src1, $src2, \t# mulhi" %}
10848
10849 ins_encode %{
10850 __ smulh(as_Register($dst$$reg),
10851 as_Register($src1$$reg),
10852 as_Register($src2$$reg));
10853 %}
10854
10855 ins_pipe(lmul_reg_reg);
10856 %}
10857
10858 // Combined Integer Multiply & Add/Sub
10859
10860 instruct maddI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegIorL2I src3) %{
10861 match(Set dst (AddI src3 (MulI src1 src2)));
10862
10863 ins_cost(INSN_COST * 3);
10864 format %{ "madd $dst, $src1, $src2, $src3" %}
10865
10866 ins_encode %{
10867 __ maddw(as_Register($dst$$reg),
10868 as_Register($src1$$reg),
10869 as_Register($src2$$reg),
10870 as_Register($src3$$reg));
10871 %}
10872
10873 ins_pipe(imac_reg_reg);
10874 %}
10875
10876 instruct msubI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegIorL2I src3) %{
10877 match(Set dst (SubI src3 (MulI src1 src2)));
10878
10879 ins_cost(INSN_COST * 3);
10880 format %{ "msub $dst, $src1, $src2, $src3" %}
10881
10882 ins_encode %{
10883 __ msubw(as_Register($dst$$reg),
10884 as_Register($src1$$reg),
10885 as_Register($src2$$reg),
10886 as_Register($src3$$reg));
10887 %}
10888
10889 ins_pipe(imac_reg_reg);
10890 %}
10891
10892 // Combined Integer Multiply & Neg
10893
10894 instruct mnegI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI0 zero) %{
10895 match(Set dst (MulI (SubI zero src1) src2));
10896 match(Set dst (MulI src1 (SubI zero src2)));
10897
10898 ins_cost(INSN_COST * 3);
10899 format %{ "mneg $dst, $src1, $src2" %}
10900
10901 ins_encode %{
10902 __ mnegw(as_Register($dst$$reg),
10903 as_Register($src1$$reg),
10904 as_Register($src2$$reg));
10905 %}
10906
10907 ins_pipe(imac_reg_reg);
10908 %}
10909
10910 // Combined Long Multiply & Add/Sub
10911
10912 instruct maddL(iRegLNoSp dst, iRegL src1, iRegL src2, iRegL src3) %{
10913 match(Set dst (AddL src3 (MulL src1 src2)));
10914
10915 ins_cost(INSN_COST * 5);
10916 format %{ "madd $dst, $src1, $src2, $src3" %}
10917
10918 ins_encode %{
10919 __ madd(as_Register($dst$$reg),
10920 as_Register($src1$$reg),
10921 as_Register($src2$$reg),
10922 as_Register($src3$$reg));
10923 %}
10924
10925 ins_pipe(lmac_reg_reg);
10926 %}
10927
10928 instruct msubL(iRegLNoSp dst, iRegL src1, iRegL src2, iRegL src3) %{
10929 match(Set dst (SubL src3 (MulL src1 src2)));
10930
10931 ins_cost(INSN_COST * 5);
10932 format %{ "msub $dst, $src1, $src2, $src3" %}
10933
10934 ins_encode %{
10935 __ msub(as_Register($dst$$reg),
10936 as_Register($src1$$reg),
10937 as_Register($src2$$reg),
10938 as_Register($src3$$reg));
10939 %}
10940
10941 ins_pipe(lmac_reg_reg);
10942 %}
10943
10944 // Combined Long Multiply & Neg
10945
10946 instruct mnegL(iRegLNoSp dst, iRegL src1, iRegL src2, immL0 zero) %{
10947 match(Set dst (MulL (SubL zero src1) src2));
10948 match(Set dst (MulL src1 (SubL zero src2)));
10949
10950 ins_cost(INSN_COST * 5);
10951 format %{ "mneg $dst, $src1, $src2" %}
10952
10953 ins_encode %{
10954 __ mneg(as_Register($dst$$reg),
10955 as_Register($src1$$reg),
10956 as_Register($src2$$reg));
10957 %}
10958
10959 ins_pipe(lmac_reg_reg);
10960 %}
10961
10962 // Combine Integer Signed Multiply & Add/Sub/Neg Long
10963
10964 instruct smaddL(iRegLNoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegLNoSp src3) %{
10965 match(Set dst (AddL src3 (MulL (ConvI2L src1) (ConvI2L src2))));
10966
10967 ins_cost(INSN_COST * 3);
10968 format %{ "smaddl $dst, $src1, $src2, $src3" %}
10969
10970 ins_encode %{
10971 __ smaddl(as_Register($dst$$reg),
10972 as_Register($src1$$reg),
10973 as_Register($src2$$reg),
10974 as_Register($src3$$reg));
10975 %}
10976
10977 ins_pipe(imac_reg_reg);
10978 %}
10979
10980 instruct smsubL(iRegLNoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegLNoSp src3) %{
10981 match(Set dst (SubL src3 (MulL (ConvI2L src1) (ConvI2L src2))));
10982
10983 ins_cost(INSN_COST * 3);
10984 format %{ "smsubl $dst, $src1, $src2, $src3" %}
10985
10986 ins_encode %{
10987 __ smsubl(as_Register($dst$$reg),
10988 as_Register($src1$$reg),
10989 as_Register($src2$$reg),
10990 as_Register($src3$$reg));
10991 %}
10992
10993 ins_pipe(imac_reg_reg);
10994 %}
10995
10996 instruct smnegL(iRegLNoSp dst, iRegIorL2I src1, iRegIorL2I src2, immL0 zero) %{
10997 match(Set dst (MulL (SubL zero (ConvI2L src1)) (ConvI2L src2)));
10998 match(Set dst (MulL (ConvI2L src1) (SubL zero (ConvI2L src2))));
10999
11000 ins_cost(INSN_COST * 3);
11001 format %{ "smnegl $dst, $src1, $src2" %}
11002
11003 ins_encode %{
11004 __ smnegl(as_Register($dst$$reg),
11005 as_Register($src1$$reg),
11006 as_Register($src2$$reg));
11007 %}
11008
11009 ins_pipe(imac_reg_reg);
11010 %}
11011
11012 // Combined Multiply-Add Shorts into Integer (dst = src1 * src2 + src3 * src4)
11013
11014 instruct muladdS2I(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegIorL2I src3, iRegIorL2I src4) %{
11015 match(Set dst (MulAddS2I (Binary src1 src2) (Binary src3 src4)));
11016
11017 ins_cost(INSN_COST * 5);
11018 format %{ "mulw rscratch1, $src1, $src2\n\t"
11019 "maddw $dst, $src3, $src4, rscratch1" %}
11020
11021 ins_encode %{
11022 __ mulw(rscratch1, as_Register($src1$$reg), as_Register($src2$$reg));
11023 __ maddw(as_Register($dst$$reg), as_Register($src3$$reg), as_Register($src4$$reg), rscratch1); %}
11024
11025 ins_pipe(imac_reg_reg);
11026 %}
11027
11028 // Integer Divide
11029
11030 instruct divI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
11031 match(Set dst (DivI src1 src2));
11032
11033 ins_cost(INSN_COST * 19);
11034 format %{ "sdivw $dst, $src1, $src2" %}
11035
11036 ins_encode(aarch64_enc_divw(dst, src1, src2));
11037 ins_pipe(idiv_reg_reg);
11038 %}
11039
11040 // Long Divide
11041
11042 instruct divL(iRegLNoSp dst, iRegL src1, iRegL src2) %{
11043 match(Set dst (DivL src1 src2));
11044
11045 ins_cost(INSN_COST * 35);
11046 format %{ "sdiv $dst, $src1, $src2" %}
11047
11048 ins_encode(aarch64_enc_div(dst, src1, src2));
11049 ins_pipe(ldiv_reg_reg);
11050 %}
11051
11052 // Integer Remainder
11053
11054 instruct modI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
11055 match(Set dst (ModI src1 src2));
11056
11057 ins_cost(INSN_COST * 22);
11058 format %{ "sdivw rscratch1, $src1, $src2\n\t"
11059 "msubw($dst, rscratch1, $src2, $src1" %}
11060
11061 ins_encode(aarch64_enc_modw(dst, src1, src2));
11062 ins_pipe(idiv_reg_reg);
11063 %}
11064
11065 // Long Remainder
11066
11067 instruct modL(iRegLNoSp dst, iRegL src1, iRegL src2) %{
11068 match(Set dst (ModL src1 src2));
11069
11070 ins_cost(INSN_COST * 38);
11071 format %{ "sdiv rscratch1, $src1, $src2\n"
11072 "msub($dst, rscratch1, $src2, $src1" %}
11073
11074 ins_encode(aarch64_enc_mod(dst, src1, src2));
11075 ins_pipe(ldiv_reg_reg);
11076 %}
11077
11078 // Integer Shifts
11079
11080 // Shift Left Register
11081 instruct lShiftI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
11082 match(Set dst (LShiftI src1 src2));
11083
11084 ins_cost(INSN_COST * 2);
11085 format %{ "lslvw $dst, $src1, $src2" %}
11086
11087 ins_encode %{
11088 __ lslvw(as_Register($dst$$reg),
11089 as_Register($src1$$reg),
11090 as_Register($src2$$reg));
11091 %}
11092
11093 ins_pipe(ialu_reg_reg_vshift);
11094 %}
11095
11096 // Shift Left Immediate
11097 instruct lShiftI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immI src2) %{
11098 match(Set dst (LShiftI src1 src2));
11099
11100 ins_cost(INSN_COST);
11101 format %{ "lslw $dst, $src1, ($src2 & 0x1f)" %}
11102
11103 ins_encode %{
11104 __ lslw(as_Register($dst$$reg),
11105 as_Register($src1$$reg),
11106 $src2$$constant & 0x1f);
11107 %}
11108
11109 ins_pipe(ialu_reg_shift);
11110 %}
11111
11112 // Shift Right Logical Register
11113 instruct urShiftI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
11114 match(Set dst (URShiftI src1 src2));
11115
11116 ins_cost(INSN_COST * 2);
11117 format %{ "lsrvw $dst, $src1, $src2" %}
11118
11119 ins_encode %{
11120 __ lsrvw(as_Register($dst$$reg),
11121 as_Register($src1$$reg),
11122 as_Register($src2$$reg));
11123 %}
11124
11125 ins_pipe(ialu_reg_reg_vshift);
11126 %}
11127
11128 // Shift Right Logical Immediate
11129 instruct urShiftI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immI src2) %{
11130 match(Set dst (URShiftI src1 src2));
11131
11132 ins_cost(INSN_COST);
11133 format %{ "lsrw $dst, $src1, ($src2 & 0x1f)" %}
11134
11135 ins_encode %{
11136 __ lsrw(as_Register($dst$$reg),
11137 as_Register($src1$$reg),
11138 $src2$$constant & 0x1f);
11139 %}
11140
11141 ins_pipe(ialu_reg_shift);
11142 %}
11143
11144 // Shift Right Arithmetic Register
11145 instruct rShiftI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
11146 match(Set dst (RShiftI src1 src2));
11147
11148 ins_cost(INSN_COST * 2);
11149 format %{ "asrvw $dst, $src1, $src2" %}
11150
11151 ins_encode %{
11152 __ asrvw(as_Register($dst$$reg),
11153 as_Register($src1$$reg),
11154 as_Register($src2$$reg));
11155 %}
11156
11157 ins_pipe(ialu_reg_reg_vshift);
11158 %}
11159
11160 // Shift Right Arithmetic Immediate
11161 instruct rShiftI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immI src2) %{
11162 match(Set dst (RShiftI src1 src2));
11163
11164 ins_cost(INSN_COST);
11165 format %{ "asrw $dst, $src1, ($src2 & 0x1f)" %}
11166
11167 ins_encode %{
11168 __ asrw(as_Register($dst$$reg),
11169 as_Register($src1$$reg),
11170 $src2$$constant & 0x1f);
11171 %}
11172
11173 ins_pipe(ialu_reg_shift);
11174 %}
11175
11176 // Combined Int Mask and Right Shift (using UBFM)
11177 // TODO
11178
11179 // Long Shifts
11180
11181 // Shift Left Register
11182 instruct lShiftL_reg_reg(iRegLNoSp dst, iRegL src1, iRegIorL2I src2) %{
11183 match(Set dst (LShiftL src1 src2));
11184
11185 ins_cost(INSN_COST * 2);
11186 format %{ "lslv $dst, $src1, $src2" %}
11187
11188 ins_encode %{
11189 __ lslv(as_Register($dst$$reg),
11190 as_Register($src1$$reg),
11191 as_Register($src2$$reg));
11192 %}
11193
11194 ins_pipe(ialu_reg_reg_vshift);
11195 %}
11196
11197 // Shift Left Immediate
11198 instruct lShiftL_reg_imm(iRegLNoSp dst, iRegL src1, immI src2) %{
11199 match(Set dst (LShiftL src1 src2));
11200
11201 ins_cost(INSN_COST);
11202 format %{ "lsl $dst, $src1, ($src2 & 0x3f)" %}
11203
11204 ins_encode %{
11205 __ lsl(as_Register($dst$$reg),
11206 as_Register($src1$$reg),
11207 $src2$$constant & 0x3f);
11208 %}
11209
11210 ins_pipe(ialu_reg_shift);
11211 %}
11212
11213 // Shift Right Logical Register
11214 instruct urShiftL_reg_reg(iRegLNoSp dst, iRegL src1, iRegIorL2I src2) %{
11215 match(Set dst (URShiftL src1 src2));
11216
11217 ins_cost(INSN_COST * 2);
11218 format %{ "lsrv $dst, $src1, $src2" %}
11219
11220 ins_encode %{
11221 __ lsrv(as_Register($dst$$reg),
11222 as_Register($src1$$reg),
11223 as_Register($src2$$reg));
11224 %}
11225
11226 ins_pipe(ialu_reg_reg_vshift);
11227 %}
11228
11229 // Shift Right Logical Immediate
11230 instruct urShiftL_reg_imm(iRegLNoSp dst, iRegL src1, immI src2) %{
11231 match(Set dst (URShiftL src1 src2));
11232
11233 ins_cost(INSN_COST);
11234 format %{ "lsr $dst, $src1, ($src2 & 0x3f)" %}
11235
11236 ins_encode %{
11237 __ lsr(as_Register($dst$$reg),
11238 as_Register($src1$$reg),
11239 $src2$$constant & 0x3f);
11240 %}
11241
11242 ins_pipe(ialu_reg_shift);
11243 %}
11244
11245 // A special-case pattern for card table stores.
11246 instruct urShiftP_reg_imm(iRegLNoSp dst, iRegP src1, immI src2) %{
11247 match(Set dst (URShiftL (CastP2X src1) src2));
11248
11249 ins_cost(INSN_COST);
11250 format %{ "lsr $dst, p2x($src1), ($src2 & 0x3f)" %}
11251
11252 ins_encode %{
11253 __ lsr(as_Register($dst$$reg),
11254 as_Register($src1$$reg),
11255 $src2$$constant & 0x3f);
11256 %}
11257
11258 ins_pipe(ialu_reg_shift);
11259 %}
11260
11261 // Shift Right Arithmetic Register
11262 instruct rShiftL_reg_reg(iRegLNoSp dst, iRegL src1, iRegIorL2I src2) %{
11263 match(Set dst (RShiftL src1 src2));
11264
11265 ins_cost(INSN_COST * 2);
11266 format %{ "asrv $dst, $src1, $src2" %}
11267
11268 ins_encode %{
11269 __ asrv(as_Register($dst$$reg),
11270 as_Register($src1$$reg),
11271 as_Register($src2$$reg));
11272 %}
11273
11274 ins_pipe(ialu_reg_reg_vshift);
11275 %}
11276
11277 // Shift Right Arithmetic Immediate
11278 instruct rShiftL_reg_imm(iRegLNoSp dst, iRegL src1, immI src2) %{
11279 match(Set dst (RShiftL src1 src2));
11280
11281 ins_cost(INSN_COST);
11282 format %{ "asr $dst, $src1, ($src2 & 0x3f)" %}
11283
11284 ins_encode %{
11285 __ asr(as_Register($dst$$reg),
11286 as_Register($src1$$reg),
11287 $src2$$constant & 0x3f);
11288 %}
11289
11290 ins_pipe(ialu_reg_shift);
11291 %}
11292
11293 // BEGIN This section of the file is automatically generated. Do not edit --------------
11294 // This section is generated from aarch64_ad.m4
11295
11296
11297 // This pattern is automatically generated from aarch64_ad.m4
11298 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11299 instruct regL_not_reg(iRegLNoSp dst,
11300 iRegL src1, immL_M1 m1,
11301 rFlagsReg cr) %{
11302 match(Set dst (XorL src1 m1));
11303 ins_cost(INSN_COST);
11304 format %{ "eon $dst, $src1, zr" %}
11305
11306 ins_encode %{
11307 __ eon(as_Register($dst$$reg),
11308 as_Register($src1$$reg),
11309 zr,
11310 Assembler::LSL, 0);
11311 %}
11312
11313 ins_pipe(ialu_reg);
11314 %}
11315
11316 // This pattern is automatically generated from aarch64_ad.m4
11317 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11318 instruct regI_not_reg(iRegINoSp dst,
11319 iRegIorL2I src1, immI_M1 m1,
11320 rFlagsReg cr) %{
11321 match(Set dst (XorI src1 m1));
11322 ins_cost(INSN_COST);
11323 format %{ "eonw $dst, $src1, zr" %}
11324
11325 ins_encode %{
11326 __ eonw(as_Register($dst$$reg),
11327 as_Register($src1$$reg),
11328 zr,
11329 Assembler::LSL, 0);
11330 %}
11331
11332 ins_pipe(ialu_reg);
11333 %}
11334
11335 // This pattern is automatically generated from aarch64_ad.m4
11336 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11337 instruct AndI_reg_not_reg(iRegINoSp dst,
11338 iRegIorL2I src1, iRegIorL2I src2, immI_M1 m1) %{
11339 match(Set dst (AndI src1 (XorI src2 m1)));
11340 ins_cost(INSN_COST);
11341 format %{ "bicw $dst, $src1, $src2" %}
11342
11343 ins_encode %{
11344 __ bicw(as_Register($dst$$reg),
11345 as_Register($src1$$reg),
11346 as_Register($src2$$reg),
11347 Assembler::LSL, 0);
11348 %}
11349
11350 ins_pipe(ialu_reg_reg);
11351 %}
11352
11353 // This pattern is automatically generated from aarch64_ad.m4
11354 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11355 instruct AndL_reg_not_reg(iRegLNoSp dst,
11356 iRegL src1, iRegL src2, immL_M1 m1) %{
11357 match(Set dst (AndL src1 (XorL src2 m1)));
11358 ins_cost(INSN_COST);
11359 format %{ "bic $dst, $src1, $src2" %}
11360
11361 ins_encode %{
11362 __ bic(as_Register($dst$$reg),
11363 as_Register($src1$$reg),
11364 as_Register($src2$$reg),
11365 Assembler::LSL, 0);
11366 %}
11367
11368 ins_pipe(ialu_reg_reg);
11369 %}
11370
11371 // This pattern is automatically generated from aarch64_ad.m4
11372 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11373 instruct OrI_reg_not_reg(iRegINoSp dst,
11374 iRegIorL2I src1, iRegIorL2I src2, immI_M1 m1) %{
11375 match(Set dst (OrI src1 (XorI src2 m1)));
11376 ins_cost(INSN_COST);
11377 format %{ "ornw $dst, $src1, $src2" %}
11378
11379 ins_encode %{
11380 __ ornw(as_Register($dst$$reg),
11381 as_Register($src1$$reg),
11382 as_Register($src2$$reg),
11383 Assembler::LSL, 0);
11384 %}
11385
11386 ins_pipe(ialu_reg_reg);
11387 %}
11388
11389 // This pattern is automatically generated from aarch64_ad.m4
11390 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11391 instruct OrL_reg_not_reg(iRegLNoSp dst,
11392 iRegL src1, iRegL src2, immL_M1 m1) %{
11393 match(Set dst (OrL src1 (XorL src2 m1)));
11394 ins_cost(INSN_COST);
11395 format %{ "orn $dst, $src1, $src2" %}
11396
11397 ins_encode %{
11398 __ orn(as_Register($dst$$reg),
11399 as_Register($src1$$reg),
11400 as_Register($src2$$reg),
11401 Assembler::LSL, 0);
11402 %}
11403
11404 ins_pipe(ialu_reg_reg);
11405 %}
11406
11407 // This pattern is automatically generated from aarch64_ad.m4
11408 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11409 instruct XorI_reg_not_reg(iRegINoSp dst,
11410 iRegIorL2I src1, iRegIorL2I src2, immI_M1 m1) %{
11411 match(Set dst (XorI m1 (XorI src2 src1)));
11412 ins_cost(INSN_COST);
11413 format %{ "eonw $dst, $src1, $src2" %}
11414
11415 ins_encode %{
11416 __ eonw(as_Register($dst$$reg),
11417 as_Register($src1$$reg),
11418 as_Register($src2$$reg),
11419 Assembler::LSL, 0);
11420 %}
11421
11422 ins_pipe(ialu_reg_reg);
11423 %}
11424
11425 // This pattern is automatically generated from aarch64_ad.m4
11426 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11427 instruct XorL_reg_not_reg(iRegLNoSp dst,
11428 iRegL src1, iRegL src2, immL_M1 m1) %{
11429 match(Set dst (XorL m1 (XorL src2 src1)));
11430 ins_cost(INSN_COST);
11431 format %{ "eon $dst, $src1, $src2" %}
11432
11433 ins_encode %{
11434 __ eon(as_Register($dst$$reg),
11435 as_Register($src1$$reg),
11436 as_Register($src2$$reg),
11437 Assembler::LSL, 0);
11438 %}
11439
11440 ins_pipe(ialu_reg_reg);
11441 %}
11442
11443 // This pattern is automatically generated from aarch64_ad.m4
11444 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11445 // val & (-1 ^ (val >>> shift)) ==> bicw
11446 instruct AndI_reg_URShift_not_reg(iRegINoSp dst,
11447 iRegIorL2I src1, iRegIorL2I src2,
11448 immI src3, immI_M1 src4) %{
11449 match(Set dst (AndI src1 (XorI(URShiftI src2 src3) src4)));
11450 ins_cost(1.9 * INSN_COST);
11451 format %{ "bicw $dst, $src1, $src2, LSR $src3" %}
11452
11453 ins_encode %{
11454 __ bicw(as_Register($dst$$reg),
11455 as_Register($src1$$reg),
11456 as_Register($src2$$reg),
11457 Assembler::LSR,
11458 $src3$$constant & 0x1f);
11459 %}
11460
11461 ins_pipe(ialu_reg_reg_shift);
11462 %}
11463
11464 // This pattern is automatically generated from aarch64_ad.m4
11465 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11466 // val & (-1 ^ (val >>> shift)) ==> bic
11467 instruct AndL_reg_URShift_not_reg(iRegLNoSp dst,
11468 iRegL src1, iRegL src2,
11469 immI src3, immL_M1 src4) %{
11470 match(Set dst (AndL src1 (XorL(URShiftL src2 src3) src4)));
11471 ins_cost(1.9 * INSN_COST);
11472 format %{ "bic $dst, $src1, $src2, LSR $src3" %}
11473
11474 ins_encode %{
11475 __ bic(as_Register($dst$$reg),
11476 as_Register($src1$$reg),
11477 as_Register($src2$$reg),
11478 Assembler::LSR,
11479 $src3$$constant & 0x3f);
11480 %}
11481
11482 ins_pipe(ialu_reg_reg_shift);
11483 %}
11484
11485 // This pattern is automatically generated from aarch64_ad.m4
11486 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11487 // val & (-1 ^ (val >> shift)) ==> bicw
11488 instruct AndI_reg_RShift_not_reg(iRegINoSp dst,
11489 iRegIorL2I src1, iRegIorL2I src2,
11490 immI src3, immI_M1 src4) %{
11491 match(Set dst (AndI src1 (XorI(RShiftI src2 src3) src4)));
11492 ins_cost(1.9 * INSN_COST);
11493 format %{ "bicw $dst, $src1, $src2, ASR $src3" %}
11494
11495 ins_encode %{
11496 __ bicw(as_Register($dst$$reg),
11497 as_Register($src1$$reg),
11498 as_Register($src2$$reg),
11499 Assembler::ASR,
11500 $src3$$constant & 0x1f);
11501 %}
11502
11503 ins_pipe(ialu_reg_reg_shift);
11504 %}
11505
11506 // This pattern is automatically generated from aarch64_ad.m4
11507 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11508 // val & (-1 ^ (val >> shift)) ==> bic
11509 instruct AndL_reg_RShift_not_reg(iRegLNoSp dst,
11510 iRegL src1, iRegL src2,
11511 immI src3, immL_M1 src4) %{
11512 match(Set dst (AndL src1 (XorL(RShiftL src2 src3) src4)));
11513 ins_cost(1.9 * INSN_COST);
11514 format %{ "bic $dst, $src1, $src2, ASR $src3" %}
11515
11516 ins_encode %{
11517 __ bic(as_Register($dst$$reg),
11518 as_Register($src1$$reg),
11519 as_Register($src2$$reg),
11520 Assembler::ASR,
11521 $src3$$constant & 0x3f);
11522 %}
11523
11524 ins_pipe(ialu_reg_reg_shift);
11525 %}
11526
11527 // This pattern is automatically generated from aarch64_ad.m4
11528 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11529 // val & (-1 ^ (val ror shift)) ==> bicw
11530 instruct AndI_reg_RotateRight_not_reg(iRegINoSp dst,
11531 iRegIorL2I src1, iRegIorL2I src2,
11532 immI src3, immI_M1 src4) %{
11533 match(Set dst (AndI src1 (XorI(RotateRight src2 src3) src4)));
11534 ins_cost(1.9 * INSN_COST);
11535 format %{ "bicw $dst, $src1, $src2, ROR $src3" %}
11536
11537 ins_encode %{
11538 __ bicw(as_Register($dst$$reg),
11539 as_Register($src1$$reg),
11540 as_Register($src2$$reg),
11541 Assembler::ROR,
11542 $src3$$constant & 0x1f);
11543 %}
11544
11545 ins_pipe(ialu_reg_reg_shift);
11546 %}
11547
11548 // This pattern is automatically generated from aarch64_ad.m4
11549 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11550 // val & (-1 ^ (val ror shift)) ==> bic
11551 instruct AndL_reg_RotateRight_not_reg(iRegLNoSp dst,
11552 iRegL src1, iRegL src2,
11553 immI src3, immL_M1 src4) %{
11554 match(Set dst (AndL src1 (XorL(RotateRight src2 src3) src4)));
11555 ins_cost(1.9 * INSN_COST);
11556 format %{ "bic $dst, $src1, $src2, ROR $src3" %}
11557
11558 ins_encode %{
11559 __ bic(as_Register($dst$$reg),
11560 as_Register($src1$$reg),
11561 as_Register($src2$$reg),
11562 Assembler::ROR,
11563 $src3$$constant & 0x3f);
11564 %}
11565
11566 ins_pipe(ialu_reg_reg_shift);
11567 %}
11568
11569 // This pattern is automatically generated from aarch64_ad.m4
11570 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11571 // val & (-1 ^ (val << shift)) ==> bicw
11572 instruct AndI_reg_LShift_not_reg(iRegINoSp dst,
11573 iRegIorL2I src1, iRegIorL2I src2,
11574 immI src3, immI_M1 src4) %{
11575 match(Set dst (AndI src1 (XorI(LShiftI src2 src3) src4)));
11576 ins_cost(1.9 * INSN_COST);
11577 format %{ "bicw $dst, $src1, $src2, LSL $src3" %}
11578
11579 ins_encode %{
11580 __ bicw(as_Register($dst$$reg),
11581 as_Register($src1$$reg),
11582 as_Register($src2$$reg),
11583 Assembler::LSL,
11584 $src3$$constant & 0x1f);
11585 %}
11586
11587 ins_pipe(ialu_reg_reg_shift);
11588 %}
11589
11590 // This pattern is automatically generated from aarch64_ad.m4
11591 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11592 // val & (-1 ^ (val << shift)) ==> bic
11593 instruct AndL_reg_LShift_not_reg(iRegLNoSp dst,
11594 iRegL src1, iRegL src2,
11595 immI src3, immL_M1 src4) %{
11596 match(Set dst (AndL src1 (XorL(LShiftL src2 src3) src4)));
11597 ins_cost(1.9 * INSN_COST);
11598 format %{ "bic $dst, $src1, $src2, LSL $src3" %}
11599
11600 ins_encode %{
11601 __ bic(as_Register($dst$$reg),
11602 as_Register($src1$$reg),
11603 as_Register($src2$$reg),
11604 Assembler::LSL,
11605 $src3$$constant & 0x3f);
11606 %}
11607
11608 ins_pipe(ialu_reg_reg_shift);
11609 %}
11610
11611 // This pattern is automatically generated from aarch64_ad.m4
11612 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11613 // val ^ (-1 ^ (val >>> shift)) ==> eonw
11614 instruct XorI_reg_URShift_not_reg(iRegINoSp dst,
11615 iRegIorL2I src1, iRegIorL2I src2,
11616 immI src3, immI_M1 src4) %{
11617 match(Set dst (XorI src4 (XorI(URShiftI src2 src3) src1)));
11618 ins_cost(1.9 * INSN_COST);
11619 format %{ "eonw $dst, $src1, $src2, LSR $src3" %}
11620
11621 ins_encode %{
11622 __ eonw(as_Register($dst$$reg),
11623 as_Register($src1$$reg),
11624 as_Register($src2$$reg),
11625 Assembler::LSR,
11626 $src3$$constant & 0x1f);
11627 %}
11628
11629 ins_pipe(ialu_reg_reg_shift);
11630 %}
11631
11632 // This pattern is automatically generated from aarch64_ad.m4
11633 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11634 // val ^ (-1 ^ (val >>> shift)) ==> eon
11635 instruct XorL_reg_URShift_not_reg(iRegLNoSp dst,
11636 iRegL src1, iRegL src2,
11637 immI src3, immL_M1 src4) %{
11638 match(Set dst (XorL src4 (XorL(URShiftL src2 src3) src1)));
11639 ins_cost(1.9 * INSN_COST);
11640 format %{ "eon $dst, $src1, $src2, LSR $src3" %}
11641
11642 ins_encode %{
11643 __ eon(as_Register($dst$$reg),
11644 as_Register($src1$$reg),
11645 as_Register($src2$$reg),
11646 Assembler::LSR,
11647 $src3$$constant & 0x3f);
11648 %}
11649
11650 ins_pipe(ialu_reg_reg_shift);
11651 %}
11652
11653 // This pattern is automatically generated from aarch64_ad.m4
11654 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11655 // val ^ (-1 ^ (val >> shift)) ==> eonw
11656 instruct XorI_reg_RShift_not_reg(iRegINoSp dst,
11657 iRegIorL2I src1, iRegIorL2I src2,
11658 immI src3, immI_M1 src4) %{
11659 match(Set dst (XorI src4 (XorI(RShiftI src2 src3) src1)));
11660 ins_cost(1.9 * INSN_COST);
11661 format %{ "eonw $dst, $src1, $src2, ASR $src3" %}
11662
11663 ins_encode %{
11664 __ eonw(as_Register($dst$$reg),
11665 as_Register($src1$$reg),
11666 as_Register($src2$$reg),
11667 Assembler::ASR,
11668 $src3$$constant & 0x1f);
11669 %}
11670
11671 ins_pipe(ialu_reg_reg_shift);
11672 %}
11673
11674 // This pattern is automatically generated from aarch64_ad.m4
11675 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11676 // val ^ (-1 ^ (val >> shift)) ==> eon
11677 instruct XorL_reg_RShift_not_reg(iRegLNoSp dst,
11678 iRegL src1, iRegL src2,
11679 immI src3, immL_M1 src4) %{
11680 match(Set dst (XorL src4 (XorL(RShiftL src2 src3) src1)));
11681 ins_cost(1.9 * INSN_COST);
11682 format %{ "eon $dst, $src1, $src2, ASR $src3" %}
11683
11684 ins_encode %{
11685 __ eon(as_Register($dst$$reg),
11686 as_Register($src1$$reg),
11687 as_Register($src2$$reg),
11688 Assembler::ASR,
11689 $src3$$constant & 0x3f);
11690 %}
11691
11692 ins_pipe(ialu_reg_reg_shift);
11693 %}
11694
11695 // This pattern is automatically generated from aarch64_ad.m4
11696 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11697 // val ^ (-1 ^ (val ror shift)) ==> eonw
11698 instruct XorI_reg_RotateRight_not_reg(iRegINoSp dst,
11699 iRegIorL2I src1, iRegIorL2I src2,
11700 immI src3, immI_M1 src4) %{
11701 match(Set dst (XorI src4 (XorI(RotateRight src2 src3) src1)));
11702 ins_cost(1.9 * INSN_COST);
11703 format %{ "eonw $dst, $src1, $src2, ROR $src3" %}
11704
11705 ins_encode %{
11706 __ eonw(as_Register($dst$$reg),
11707 as_Register($src1$$reg),
11708 as_Register($src2$$reg),
11709 Assembler::ROR,
11710 $src3$$constant & 0x1f);
11711 %}
11712
11713 ins_pipe(ialu_reg_reg_shift);
11714 %}
11715
11716 // This pattern is automatically generated from aarch64_ad.m4
11717 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11718 // val ^ (-1 ^ (val ror shift)) ==> eon
11719 instruct XorL_reg_RotateRight_not_reg(iRegLNoSp dst,
11720 iRegL src1, iRegL src2,
11721 immI src3, immL_M1 src4) %{
11722 match(Set dst (XorL src4 (XorL(RotateRight src2 src3) src1)));
11723 ins_cost(1.9 * INSN_COST);
11724 format %{ "eon $dst, $src1, $src2, ROR $src3" %}
11725
11726 ins_encode %{
11727 __ eon(as_Register($dst$$reg),
11728 as_Register($src1$$reg),
11729 as_Register($src2$$reg),
11730 Assembler::ROR,
11731 $src3$$constant & 0x3f);
11732 %}
11733
11734 ins_pipe(ialu_reg_reg_shift);
11735 %}
11736
11737 // This pattern is automatically generated from aarch64_ad.m4
11738 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11739 // val ^ (-1 ^ (val << shift)) ==> eonw
11740 instruct XorI_reg_LShift_not_reg(iRegINoSp dst,
11741 iRegIorL2I src1, iRegIorL2I src2,
11742 immI src3, immI_M1 src4) %{
11743 match(Set dst (XorI src4 (XorI(LShiftI src2 src3) src1)));
11744 ins_cost(1.9 * INSN_COST);
11745 format %{ "eonw $dst, $src1, $src2, LSL $src3" %}
11746
11747 ins_encode %{
11748 __ eonw(as_Register($dst$$reg),
11749 as_Register($src1$$reg),
11750 as_Register($src2$$reg),
11751 Assembler::LSL,
11752 $src3$$constant & 0x1f);
11753 %}
11754
11755 ins_pipe(ialu_reg_reg_shift);
11756 %}
11757
11758 // This pattern is automatically generated from aarch64_ad.m4
11759 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11760 // val ^ (-1 ^ (val << shift)) ==> eon
11761 instruct XorL_reg_LShift_not_reg(iRegLNoSp dst,
11762 iRegL src1, iRegL src2,
11763 immI src3, immL_M1 src4) %{
11764 match(Set dst (XorL src4 (XorL(LShiftL src2 src3) src1)));
11765 ins_cost(1.9 * INSN_COST);
11766 format %{ "eon $dst, $src1, $src2, LSL $src3" %}
11767
11768 ins_encode %{
11769 __ eon(as_Register($dst$$reg),
11770 as_Register($src1$$reg),
11771 as_Register($src2$$reg),
11772 Assembler::LSL,
11773 $src3$$constant & 0x3f);
11774 %}
11775
11776 ins_pipe(ialu_reg_reg_shift);
11777 %}
11778
11779 // This pattern is automatically generated from aarch64_ad.m4
11780 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11781 // val | (-1 ^ (val >>> shift)) ==> ornw
11782 instruct OrI_reg_URShift_not_reg(iRegINoSp dst,
11783 iRegIorL2I src1, iRegIorL2I src2,
11784 immI src3, immI_M1 src4) %{
11785 match(Set dst (OrI src1 (XorI(URShiftI src2 src3) src4)));
11786 ins_cost(1.9 * INSN_COST);
11787 format %{ "ornw $dst, $src1, $src2, LSR $src3" %}
11788
11789 ins_encode %{
11790 __ ornw(as_Register($dst$$reg),
11791 as_Register($src1$$reg),
11792 as_Register($src2$$reg),
11793 Assembler::LSR,
11794 $src3$$constant & 0x1f);
11795 %}
11796
11797 ins_pipe(ialu_reg_reg_shift);
11798 %}
11799
11800 // This pattern is automatically generated from aarch64_ad.m4
11801 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11802 // val | (-1 ^ (val >>> shift)) ==> orn
11803 instruct OrL_reg_URShift_not_reg(iRegLNoSp dst,
11804 iRegL src1, iRegL src2,
11805 immI src3, immL_M1 src4) %{
11806 match(Set dst (OrL src1 (XorL(URShiftL src2 src3) src4)));
11807 ins_cost(1.9 * INSN_COST);
11808 format %{ "orn $dst, $src1, $src2, LSR $src3" %}
11809
11810 ins_encode %{
11811 __ orn(as_Register($dst$$reg),
11812 as_Register($src1$$reg),
11813 as_Register($src2$$reg),
11814 Assembler::LSR,
11815 $src3$$constant & 0x3f);
11816 %}
11817
11818 ins_pipe(ialu_reg_reg_shift);
11819 %}
11820
11821 // This pattern is automatically generated from aarch64_ad.m4
11822 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11823 // val | (-1 ^ (val >> shift)) ==> ornw
11824 instruct OrI_reg_RShift_not_reg(iRegINoSp dst,
11825 iRegIorL2I src1, iRegIorL2I src2,
11826 immI src3, immI_M1 src4) %{
11827 match(Set dst (OrI src1 (XorI(RShiftI src2 src3) src4)));
11828 ins_cost(1.9 * INSN_COST);
11829 format %{ "ornw $dst, $src1, $src2, ASR $src3" %}
11830
11831 ins_encode %{
11832 __ ornw(as_Register($dst$$reg),
11833 as_Register($src1$$reg),
11834 as_Register($src2$$reg),
11835 Assembler::ASR,
11836 $src3$$constant & 0x1f);
11837 %}
11838
11839 ins_pipe(ialu_reg_reg_shift);
11840 %}
11841
11842 // This pattern is automatically generated from aarch64_ad.m4
11843 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11844 // val | (-1 ^ (val >> shift)) ==> orn
11845 instruct OrL_reg_RShift_not_reg(iRegLNoSp dst,
11846 iRegL src1, iRegL src2,
11847 immI src3, immL_M1 src4) %{
11848 match(Set dst (OrL src1 (XorL(RShiftL src2 src3) src4)));
11849 ins_cost(1.9 * INSN_COST);
11850 format %{ "orn $dst, $src1, $src2, ASR $src3" %}
11851
11852 ins_encode %{
11853 __ orn(as_Register($dst$$reg),
11854 as_Register($src1$$reg),
11855 as_Register($src2$$reg),
11856 Assembler::ASR,
11857 $src3$$constant & 0x3f);
11858 %}
11859
11860 ins_pipe(ialu_reg_reg_shift);
11861 %}
11862
11863 // This pattern is automatically generated from aarch64_ad.m4
11864 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11865 // val | (-1 ^ (val ror shift)) ==> ornw
11866 instruct OrI_reg_RotateRight_not_reg(iRegINoSp dst,
11867 iRegIorL2I src1, iRegIorL2I src2,
11868 immI src3, immI_M1 src4) %{
11869 match(Set dst (OrI src1 (XorI(RotateRight src2 src3) src4)));
11870 ins_cost(1.9 * INSN_COST);
11871 format %{ "ornw $dst, $src1, $src2, ROR $src3" %}
11872
11873 ins_encode %{
11874 __ ornw(as_Register($dst$$reg),
11875 as_Register($src1$$reg),
11876 as_Register($src2$$reg),
11877 Assembler::ROR,
11878 $src3$$constant & 0x1f);
11879 %}
11880
11881 ins_pipe(ialu_reg_reg_shift);
11882 %}
11883
11884 // This pattern is automatically generated from aarch64_ad.m4
11885 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11886 // val | (-1 ^ (val ror shift)) ==> orn
11887 instruct OrL_reg_RotateRight_not_reg(iRegLNoSp dst,
11888 iRegL src1, iRegL src2,
11889 immI src3, immL_M1 src4) %{
11890 match(Set dst (OrL src1 (XorL(RotateRight src2 src3) src4)));
11891 ins_cost(1.9 * INSN_COST);
11892 format %{ "orn $dst, $src1, $src2, ROR $src3" %}
11893
11894 ins_encode %{
11895 __ orn(as_Register($dst$$reg),
11896 as_Register($src1$$reg),
11897 as_Register($src2$$reg),
11898 Assembler::ROR,
11899 $src3$$constant & 0x3f);
11900 %}
11901
11902 ins_pipe(ialu_reg_reg_shift);
11903 %}
11904
11905 // This pattern is automatically generated from aarch64_ad.m4
11906 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11907 // val | (-1 ^ (val << shift)) ==> ornw
11908 instruct OrI_reg_LShift_not_reg(iRegINoSp dst,
11909 iRegIorL2I src1, iRegIorL2I src2,
11910 immI src3, immI_M1 src4) %{
11911 match(Set dst (OrI src1 (XorI(LShiftI src2 src3) src4)));
11912 ins_cost(1.9 * INSN_COST);
11913 format %{ "ornw $dst, $src1, $src2, LSL $src3" %}
11914
11915 ins_encode %{
11916 __ ornw(as_Register($dst$$reg),
11917 as_Register($src1$$reg),
11918 as_Register($src2$$reg),
11919 Assembler::LSL,
11920 $src3$$constant & 0x1f);
11921 %}
11922
11923 ins_pipe(ialu_reg_reg_shift);
11924 %}
11925
11926 // This pattern is automatically generated from aarch64_ad.m4
11927 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11928 // val | (-1 ^ (val << shift)) ==> orn
11929 instruct OrL_reg_LShift_not_reg(iRegLNoSp dst,
11930 iRegL src1, iRegL src2,
11931 immI src3, immL_M1 src4) %{
11932 match(Set dst (OrL src1 (XorL(LShiftL src2 src3) src4)));
11933 ins_cost(1.9 * INSN_COST);
11934 format %{ "orn $dst, $src1, $src2, LSL $src3" %}
11935
11936 ins_encode %{
11937 __ orn(as_Register($dst$$reg),
11938 as_Register($src1$$reg),
11939 as_Register($src2$$reg),
11940 Assembler::LSL,
11941 $src3$$constant & 0x3f);
11942 %}
11943
11944 ins_pipe(ialu_reg_reg_shift);
11945 %}
11946
11947 // This pattern is automatically generated from aarch64_ad.m4
11948 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11949 instruct AndI_reg_URShift_reg(iRegINoSp dst,
11950 iRegIorL2I src1, iRegIorL2I src2,
11951 immI src3) %{
11952 match(Set dst (AndI src1 (URShiftI src2 src3)));
11953
11954 ins_cost(1.9 * INSN_COST);
11955 format %{ "andw $dst, $src1, $src2, LSR $src3" %}
11956
11957 ins_encode %{
11958 __ andw(as_Register($dst$$reg),
11959 as_Register($src1$$reg),
11960 as_Register($src2$$reg),
11961 Assembler::LSR,
11962 $src3$$constant & 0x1f);
11963 %}
11964
11965 ins_pipe(ialu_reg_reg_shift);
11966 %}
11967
11968 // This pattern is automatically generated from aarch64_ad.m4
11969 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11970 instruct AndL_reg_URShift_reg(iRegLNoSp dst,
11971 iRegL src1, iRegL src2,
11972 immI src3) %{
11973 match(Set dst (AndL src1 (URShiftL src2 src3)));
11974
11975 ins_cost(1.9 * INSN_COST);
11976 format %{ "andr $dst, $src1, $src2, LSR $src3" %}
11977
11978 ins_encode %{
11979 __ andr(as_Register($dst$$reg),
11980 as_Register($src1$$reg),
11981 as_Register($src2$$reg),
11982 Assembler::LSR,
11983 $src3$$constant & 0x3f);
11984 %}
11985
11986 ins_pipe(ialu_reg_reg_shift);
11987 %}
11988
11989 // This pattern is automatically generated from aarch64_ad.m4
11990 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11991 instruct AndI_reg_RShift_reg(iRegINoSp dst,
11992 iRegIorL2I src1, iRegIorL2I src2,
11993 immI src3) %{
11994 match(Set dst (AndI src1 (RShiftI src2 src3)));
11995
11996 ins_cost(1.9 * INSN_COST);
11997 format %{ "andw $dst, $src1, $src2, ASR $src3" %}
11998
11999 ins_encode %{
12000 __ andw(as_Register($dst$$reg),
12001 as_Register($src1$$reg),
12002 as_Register($src2$$reg),
12003 Assembler::ASR,
12004 $src3$$constant & 0x1f);
12005 %}
12006
12007 ins_pipe(ialu_reg_reg_shift);
12008 %}
12009
12010 // This pattern is automatically generated from aarch64_ad.m4
12011 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12012 instruct AndL_reg_RShift_reg(iRegLNoSp dst,
12013 iRegL src1, iRegL src2,
12014 immI src3) %{
12015 match(Set dst (AndL src1 (RShiftL src2 src3)));
12016
12017 ins_cost(1.9 * INSN_COST);
12018 format %{ "andr $dst, $src1, $src2, ASR $src3" %}
12019
12020 ins_encode %{
12021 __ andr(as_Register($dst$$reg),
12022 as_Register($src1$$reg),
12023 as_Register($src2$$reg),
12024 Assembler::ASR,
12025 $src3$$constant & 0x3f);
12026 %}
12027
12028 ins_pipe(ialu_reg_reg_shift);
12029 %}
12030
12031 // This pattern is automatically generated from aarch64_ad.m4
12032 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12033 instruct AndI_reg_LShift_reg(iRegINoSp dst,
12034 iRegIorL2I src1, iRegIorL2I src2,
12035 immI src3) %{
12036 match(Set dst (AndI src1 (LShiftI src2 src3)));
12037
12038 ins_cost(1.9 * INSN_COST);
12039 format %{ "andw $dst, $src1, $src2, LSL $src3" %}
12040
12041 ins_encode %{
12042 __ andw(as_Register($dst$$reg),
12043 as_Register($src1$$reg),
12044 as_Register($src2$$reg),
12045 Assembler::LSL,
12046 $src3$$constant & 0x1f);
12047 %}
12048
12049 ins_pipe(ialu_reg_reg_shift);
12050 %}
12051
12052 // This pattern is automatically generated from aarch64_ad.m4
12053 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12054 instruct AndL_reg_LShift_reg(iRegLNoSp dst,
12055 iRegL src1, iRegL src2,
12056 immI src3) %{
12057 match(Set dst (AndL src1 (LShiftL src2 src3)));
12058
12059 ins_cost(1.9 * INSN_COST);
12060 format %{ "andr $dst, $src1, $src2, LSL $src3" %}
12061
12062 ins_encode %{
12063 __ andr(as_Register($dst$$reg),
12064 as_Register($src1$$reg),
12065 as_Register($src2$$reg),
12066 Assembler::LSL,
12067 $src3$$constant & 0x3f);
12068 %}
12069
12070 ins_pipe(ialu_reg_reg_shift);
12071 %}
12072
12073 // This pattern is automatically generated from aarch64_ad.m4
12074 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12075 instruct AndI_reg_RotateRight_reg(iRegINoSp dst,
12076 iRegIorL2I src1, iRegIorL2I src2,
12077 immI src3) %{
12078 match(Set dst (AndI src1 (RotateRight src2 src3)));
12079
12080 ins_cost(1.9 * INSN_COST);
12081 format %{ "andw $dst, $src1, $src2, ROR $src3" %}
12082
12083 ins_encode %{
12084 __ andw(as_Register($dst$$reg),
12085 as_Register($src1$$reg),
12086 as_Register($src2$$reg),
12087 Assembler::ROR,
12088 $src3$$constant & 0x1f);
12089 %}
12090
12091 ins_pipe(ialu_reg_reg_shift);
12092 %}
12093
12094 // This pattern is automatically generated from aarch64_ad.m4
12095 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12096 instruct AndL_reg_RotateRight_reg(iRegLNoSp dst,
12097 iRegL src1, iRegL src2,
12098 immI src3) %{
12099 match(Set dst (AndL src1 (RotateRight src2 src3)));
12100
12101 ins_cost(1.9 * INSN_COST);
12102 format %{ "andr $dst, $src1, $src2, ROR $src3" %}
12103
12104 ins_encode %{
12105 __ andr(as_Register($dst$$reg),
12106 as_Register($src1$$reg),
12107 as_Register($src2$$reg),
12108 Assembler::ROR,
12109 $src3$$constant & 0x3f);
12110 %}
12111
12112 ins_pipe(ialu_reg_reg_shift);
12113 %}
12114
12115 // This pattern is automatically generated from aarch64_ad.m4
12116 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12117 instruct XorI_reg_URShift_reg(iRegINoSp dst,
12118 iRegIorL2I src1, iRegIorL2I src2,
12119 immI src3) %{
12120 match(Set dst (XorI src1 (URShiftI src2 src3)));
12121
12122 ins_cost(1.9 * INSN_COST);
12123 format %{ "eorw $dst, $src1, $src2, LSR $src3" %}
12124
12125 ins_encode %{
12126 __ eorw(as_Register($dst$$reg),
12127 as_Register($src1$$reg),
12128 as_Register($src2$$reg),
12129 Assembler::LSR,
12130 $src3$$constant & 0x1f);
12131 %}
12132
12133 ins_pipe(ialu_reg_reg_shift);
12134 %}
12135
12136 // This pattern is automatically generated from aarch64_ad.m4
12137 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12138 instruct XorL_reg_URShift_reg(iRegLNoSp dst,
12139 iRegL src1, iRegL src2,
12140 immI src3) %{
12141 match(Set dst (XorL src1 (URShiftL src2 src3)));
12142
12143 ins_cost(1.9 * INSN_COST);
12144 format %{ "eor $dst, $src1, $src2, LSR $src3" %}
12145
12146 ins_encode %{
12147 __ eor(as_Register($dst$$reg),
12148 as_Register($src1$$reg),
12149 as_Register($src2$$reg),
12150 Assembler::LSR,
12151 $src3$$constant & 0x3f);
12152 %}
12153
12154 ins_pipe(ialu_reg_reg_shift);
12155 %}
12156
12157 // This pattern is automatically generated from aarch64_ad.m4
12158 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12159 instruct XorI_reg_RShift_reg(iRegINoSp dst,
12160 iRegIorL2I src1, iRegIorL2I src2,
12161 immI src3) %{
12162 match(Set dst (XorI src1 (RShiftI src2 src3)));
12163
12164 ins_cost(1.9 * INSN_COST);
12165 format %{ "eorw $dst, $src1, $src2, ASR $src3" %}
12166
12167 ins_encode %{
12168 __ eorw(as_Register($dst$$reg),
12169 as_Register($src1$$reg),
12170 as_Register($src2$$reg),
12171 Assembler::ASR,
12172 $src3$$constant & 0x1f);
12173 %}
12174
12175 ins_pipe(ialu_reg_reg_shift);
12176 %}
12177
12178 // This pattern is automatically generated from aarch64_ad.m4
12179 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12180 instruct XorL_reg_RShift_reg(iRegLNoSp dst,
12181 iRegL src1, iRegL src2,
12182 immI src3) %{
12183 match(Set dst (XorL src1 (RShiftL src2 src3)));
12184
12185 ins_cost(1.9 * INSN_COST);
12186 format %{ "eor $dst, $src1, $src2, ASR $src3" %}
12187
12188 ins_encode %{
12189 __ eor(as_Register($dst$$reg),
12190 as_Register($src1$$reg),
12191 as_Register($src2$$reg),
12192 Assembler::ASR,
12193 $src3$$constant & 0x3f);
12194 %}
12195
12196 ins_pipe(ialu_reg_reg_shift);
12197 %}
12198
12199 // This pattern is automatically generated from aarch64_ad.m4
12200 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12201 instruct XorI_reg_LShift_reg(iRegINoSp dst,
12202 iRegIorL2I src1, iRegIorL2I src2,
12203 immI src3) %{
12204 match(Set dst (XorI src1 (LShiftI src2 src3)));
12205
12206 ins_cost(1.9 * INSN_COST);
12207 format %{ "eorw $dst, $src1, $src2, LSL $src3" %}
12208
12209 ins_encode %{
12210 __ eorw(as_Register($dst$$reg),
12211 as_Register($src1$$reg),
12212 as_Register($src2$$reg),
12213 Assembler::LSL,
12214 $src3$$constant & 0x1f);
12215 %}
12216
12217 ins_pipe(ialu_reg_reg_shift);
12218 %}
12219
12220 // This pattern is automatically generated from aarch64_ad.m4
12221 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12222 instruct XorL_reg_LShift_reg(iRegLNoSp dst,
12223 iRegL src1, iRegL src2,
12224 immI src3) %{
12225 match(Set dst (XorL src1 (LShiftL src2 src3)));
12226
12227 ins_cost(1.9 * INSN_COST);
12228 format %{ "eor $dst, $src1, $src2, LSL $src3" %}
12229
12230 ins_encode %{
12231 __ eor(as_Register($dst$$reg),
12232 as_Register($src1$$reg),
12233 as_Register($src2$$reg),
12234 Assembler::LSL,
12235 $src3$$constant & 0x3f);
12236 %}
12237
12238 ins_pipe(ialu_reg_reg_shift);
12239 %}
12240
12241 // This pattern is automatically generated from aarch64_ad.m4
12242 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12243 instruct XorI_reg_RotateRight_reg(iRegINoSp dst,
12244 iRegIorL2I src1, iRegIorL2I src2,
12245 immI src3) %{
12246 match(Set dst (XorI src1 (RotateRight src2 src3)));
12247
12248 ins_cost(1.9 * INSN_COST);
12249 format %{ "eorw $dst, $src1, $src2, ROR $src3" %}
12250
12251 ins_encode %{
12252 __ eorw(as_Register($dst$$reg),
12253 as_Register($src1$$reg),
12254 as_Register($src2$$reg),
12255 Assembler::ROR,
12256 $src3$$constant & 0x1f);
12257 %}
12258
12259 ins_pipe(ialu_reg_reg_shift);
12260 %}
12261
12262 // This pattern is automatically generated from aarch64_ad.m4
12263 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12264 instruct XorL_reg_RotateRight_reg(iRegLNoSp dst,
12265 iRegL src1, iRegL src2,
12266 immI src3) %{
12267 match(Set dst (XorL src1 (RotateRight src2 src3)));
12268
12269 ins_cost(1.9 * INSN_COST);
12270 format %{ "eor $dst, $src1, $src2, ROR $src3" %}
12271
12272 ins_encode %{
12273 __ eor(as_Register($dst$$reg),
12274 as_Register($src1$$reg),
12275 as_Register($src2$$reg),
12276 Assembler::ROR,
12277 $src3$$constant & 0x3f);
12278 %}
12279
12280 ins_pipe(ialu_reg_reg_shift);
12281 %}
12282
12283 // This pattern is automatically generated from aarch64_ad.m4
12284 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12285 instruct OrI_reg_URShift_reg(iRegINoSp dst,
12286 iRegIorL2I src1, iRegIorL2I src2,
12287 immI src3) %{
12288 match(Set dst (OrI src1 (URShiftI src2 src3)));
12289
12290 ins_cost(1.9 * INSN_COST);
12291 format %{ "orrw $dst, $src1, $src2, LSR $src3" %}
12292
12293 ins_encode %{
12294 __ orrw(as_Register($dst$$reg),
12295 as_Register($src1$$reg),
12296 as_Register($src2$$reg),
12297 Assembler::LSR,
12298 $src3$$constant & 0x1f);
12299 %}
12300
12301 ins_pipe(ialu_reg_reg_shift);
12302 %}
12303
12304 // This pattern is automatically generated from aarch64_ad.m4
12305 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12306 instruct OrL_reg_URShift_reg(iRegLNoSp dst,
12307 iRegL src1, iRegL src2,
12308 immI src3) %{
12309 match(Set dst (OrL src1 (URShiftL src2 src3)));
12310
12311 ins_cost(1.9 * INSN_COST);
12312 format %{ "orr $dst, $src1, $src2, LSR $src3" %}
12313
12314 ins_encode %{
12315 __ orr(as_Register($dst$$reg),
12316 as_Register($src1$$reg),
12317 as_Register($src2$$reg),
12318 Assembler::LSR,
12319 $src3$$constant & 0x3f);
12320 %}
12321
12322 ins_pipe(ialu_reg_reg_shift);
12323 %}
12324
12325 // This pattern is automatically generated from aarch64_ad.m4
12326 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12327 instruct OrI_reg_RShift_reg(iRegINoSp dst,
12328 iRegIorL2I src1, iRegIorL2I src2,
12329 immI src3) %{
12330 match(Set dst (OrI src1 (RShiftI src2 src3)));
12331
12332 ins_cost(1.9 * INSN_COST);
12333 format %{ "orrw $dst, $src1, $src2, ASR $src3" %}
12334
12335 ins_encode %{
12336 __ orrw(as_Register($dst$$reg),
12337 as_Register($src1$$reg),
12338 as_Register($src2$$reg),
12339 Assembler::ASR,
12340 $src3$$constant & 0x1f);
12341 %}
12342
12343 ins_pipe(ialu_reg_reg_shift);
12344 %}
12345
12346 // This pattern is automatically generated from aarch64_ad.m4
12347 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12348 instruct OrL_reg_RShift_reg(iRegLNoSp dst,
12349 iRegL src1, iRegL src2,
12350 immI src3) %{
12351 match(Set dst (OrL src1 (RShiftL src2 src3)));
12352
12353 ins_cost(1.9 * INSN_COST);
12354 format %{ "orr $dst, $src1, $src2, ASR $src3" %}
12355
12356 ins_encode %{
12357 __ orr(as_Register($dst$$reg),
12358 as_Register($src1$$reg),
12359 as_Register($src2$$reg),
12360 Assembler::ASR,
12361 $src3$$constant & 0x3f);
12362 %}
12363
12364 ins_pipe(ialu_reg_reg_shift);
12365 %}
12366
12367 // This pattern is automatically generated from aarch64_ad.m4
12368 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12369 instruct OrI_reg_LShift_reg(iRegINoSp dst,
12370 iRegIorL2I src1, iRegIorL2I src2,
12371 immI src3) %{
12372 match(Set dst (OrI src1 (LShiftI src2 src3)));
12373
12374 ins_cost(1.9 * INSN_COST);
12375 format %{ "orrw $dst, $src1, $src2, LSL $src3" %}
12376
12377 ins_encode %{
12378 __ orrw(as_Register($dst$$reg),
12379 as_Register($src1$$reg),
12380 as_Register($src2$$reg),
12381 Assembler::LSL,
12382 $src3$$constant & 0x1f);
12383 %}
12384
12385 ins_pipe(ialu_reg_reg_shift);
12386 %}
12387
12388 // This pattern is automatically generated from aarch64_ad.m4
12389 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12390 instruct OrL_reg_LShift_reg(iRegLNoSp dst,
12391 iRegL src1, iRegL src2,
12392 immI src3) %{
12393 match(Set dst (OrL src1 (LShiftL src2 src3)));
12394
12395 ins_cost(1.9 * INSN_COST);
12396 format %{ "orr $dst, $src1, $src2, LSL $src3" %}
12397
12398 ins_encode %{
12399 __ orr(as_Register($dst$$reg),
12400 as_Register($src1$$reg),
12401 as_Register($src2$$reg),
12402 Assembler::LSL,
12403 $src3$$constant & 0x3f);
12404 %}
12405
12406 ins_pipe(ialu_reg_reg_shift);
12407 %}
12408
12409 // This pattern is automatically generated from aarch64_ad.m4
12410 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12411 instruct OrI_reg_RotateRight_reg(iRegINoSp dst,
12412 iRegIorL2I src1, iRegIorL2I src2,
12413 immI src3) %{
12414 match(Set dst (OrI src1 (RotateRight src2 src3)));
12415
12416 ins_cost(1.9 * INSN_COST);
12417 format %{ "orrw $dst, $src1, $src2, ROR $src3" %}
12418
12419 ins_encode %{
12420 __ orrw(as_Register($dst$$reg),
12421 as_Register($src1$$reg),
12422 as_Register($src2$$reg),
12423 Assembler::ROR,
12424 $src3$$constant & 0x1f);
12425 %}
12426
12427 ins_pipe(ialu_reg_reg_shift);
12428 %}
12429
12430 // This pattern is automatically generated from aarch64_ad.m4
12431 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12432 instruct OrL_reg_RotateRight_reg(iRegLNoSp dst,
12433 iRegL src1, iRegL src2,
12434 immI src3) %{
12435 match(Set dst (OrL src1 (RotateRight src2 src3)));
12436
12437 ins_cost(1.9 * INSN_COST);
12438 format %{ "orr $dst, $src1, $src2, ROR $src3" %}
12439
12440 ins_encode %{
12441 __ orr(as_Register($dst$$reg),
12442 as_Register($src1$$reg),
12443 as_Register($src2$$reg),
12444 Assembler::ROR,
12445 $src3$$constant & 0x3f);
12446 %}
12447
12448 ins_pipe(ialu_reg_reg_shift);
12449 %}
12450
12451 // This pattern is automatically generated from aarch64_ad.m4
12452 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12453 instruct AddI_reg_URShift_reg(iRegINoSp dst,
12454 iRegIorL2I src1, iRegIorL2I src2,
12455 immI src3) %{
12456 match(Set dst (AddI src1 (URShiftI src2 src3)));
12457
12458 ins_cost(1.9 * INSN_COST);
12459 format %{ "addw $dst, $src1, $src2, LSR $src3" %}
12460
12461 ins_encode %{
12462 __ addw(as_Register($dst$$reg),
12463 as_Register($src1$$reg),
12464 as_Register($src2$$reg),
12465 Assembler::LSR,
12466 $src3$$constant & 0x1f);
12467 %}
12468
12469 ins_pipe(ialu_reg_reg_shift);
12470 %}
12471
12472 // This pattern is automatically generated from aarch64_ad.m4
12473 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12474 instruct AddL_reg_URShift_reg(iRegLNoSp dst,
12475 iRegL src1, iRegL src2,
12476 immI src3) %{
12477 match(Set dst (AddL src1 (URShiftL src2 src3)));
12478
12479 ins_cost(1.9 * INSN_COST);
12480 format %{ "add $dst, $src1, $src2, LSR $src3" %}
12481
12482 ins_encode %{
12483 __ add(as_Register($dst$$reg),
12484 as_Register($src1$$reg),
12485 as_Register($src2$$reg),
12486 Assembler::LSR,
12487 $src3$$constant & 0x3f);
12488 %}
12489
12490 ins_pipe(ialu_reg_reg_shift);
12491 %}
12492
12493 // This pattern is automatically generated from aarch64_ad.m4
12494 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12495 instruct AddI_reg_RShift_reg(iRegINoSp dst,
12496 iRegIorL2I src1, iRegIorL2I src2,
12497 immI src3) %{
12498 match(Set dst (AddI src1 (RShiftI src2 src3)));
12499
12500 ins_cost(1.9 * INSN_COST);
12501 format %{ "addw $dst, $src1, $src2, ASR $src3" %}
12502
12503 ins_encode %{
12504 __ addw(as_Register($dst$$reg),
12505 as_Register($src1$$reg),
12506 as_Register($src2$$reg),
12507 Assembler::ASR,
12508 $src3$$constant & 0x1f);
12509 %}
12510
12511 ins_pipe(ialu_reg_reg_shift);
12512 %}
12513
12514 // This pattern is automatically generated from aarch64_ad.m4
12515 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12516 instruct AddL_reg_RShift_reg(iRegLNoSp dst,
12517 iRegL src1, iRegL src2,
12518 immI src3) %{
12519 match(Set dst (AddL src1 (RShiftL src2 src3)));
12520
12521 ins_cost(1.9 * INSN_COST);
12522 format %{ "add $dst, $src1, $src2, ASR $src3" %}
12523
12524 ins_encode %{
12525 __ add(as_Register($dst$$reg),
12526 as_Register($src1$$reg),
12527 as_Register($src2$$reg),
12528 Assembler::ASR,
12529 $src3$$constant & 0x3f);
12530 %}
12531
12532 ins_pipe(ialu_reg_reg_shift);
12533 %}
12534
12535 // This pattern is automatically generated from aarch64_ad.m4
12536 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12537 instruct AddI_reg_LShift_reg(iRegINoSp dst,
12538 iRegIorL2I src1, iRegIorL2I src2,
12539 immI src3) %{
12540 match(Set dst (AddI src1 (LShiftI src2 src3)));
12541
12542 ins_cost(1.9 * INSN_COST);
12543 format %{ "addw $dst, $src1, $src2, LSL $src3" %}
12544
12545 ins_encode %{
12546 __ addw(as_Register($dst$$reg),
12547 as_Register($src1$$reg),
12548 as_Register($src2$$reg),
12549 Assembler::LSL,
12550 $src3$$constant & 0x1f);
12551 %}
12552
12553 ins_pipe(ialu_reg_reg_shift);
12554 %}
12555
12556 // This pattern is automatically generated from aarch64_ad.m4
12557 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12558 instruct AddL_reg_LShift_reg(iRegLNoSp dst,
12559 iRegL src1, iRegL src2,
12560 immI src3) %{
12561 match(Set dst (AddL src1 (LShiftL src2 src3)));
12562
12563 ins_cost(1.9 * INSN_COST);
12564 format %{ "add $dst, $src1, $src2, LSL $src3" %}
12565
12566 ins_encode %{
12567 __ add(as_Register($dst$$reg),
12568 as_Register($src1$$reg),
12569 as_Register($src2$$reg),
12570 Assembler::LSL,
12571 $src3$$constant & 0x3f);
12572 %}
12573
12574 ins_pipe(ialu_reg_reg_shift);
12575 %}
12576
12577 // This pattern is automatically generated from aarch64_ad.m4
12578 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12579 instruct SubI_reg_URShift_reg(iRegINoSp dst,
12580 iRegIorL2I src1, iRegIorL2I src2,
12581 immI src3) %{
12582 match(Set dst (SubI src1 (URShiftI src2 src3)));
12583
12584 ins_cost(1.9 * INSN_COST);
12585 format %{ "subw $dst, $src1, $src2, LSR $src3" %}
12586
12587 ins_encode %{
12588 __ subw(as_Register($dst$$reg),
12589 as_Register($src1$$reg),
12590 as_Register($src2$$reg),
12591 Assembler::LSR,
12592 $src3$$constant & 0x1f);
12593 %}
12594
12595 ins_pipe(ialu_reg_reg_shift);
12596 %}
12597
12598 // This pattern is automatically generated from aarch64_ad.m4
12599 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12600 instruct SubL_reg_URShift_reg(iRegLNoSp dst,
12601 iRegL src1, iRegL src2,
12602 immI src3) %{
12603 match(Set dst (SubL src1 (URShiftL src2 src3)));
12604
12605 ins_cost(1.9 * INSN_COST);
12606 format %{ "sub $dst, $src1, $src2, LSR $src3" %}
12607
12608 ins_encode %{
12609 __ sub(as_Register($dst$$reg),
12610 as_Register($src1$$reg),
12611 as_Register($src2$$reg),
12612 Assembler::LSR,
12613 $src3$$constant & 0x3f);
12614 %}
12615
12616 ins_pipe(ialu_reg_reg_shift);
12617 %}
12618
12619 // This pattern is automatically generated from aarch64_ad.m4
12620 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12621 instruct SubI_reg_RShift_reg(iRegINoSp dst,
12622 iRegIorL2I src1, iRegIorL2I src2,
12623 immI src3) %{
12624 match(Set dst (SubI src1 (RShiftI src2 src3)));
12625
12626 ins_cost(1.9 * INSN_COST);
12627 format %{ "subw $dst, $src1, $src2, ASR $src3" %}
12628
12629 ins_encode %{
12630 __ subw(as_Register($dst$$reg),
12631 as_Register($src1$$reg),
12632 as_Register($src2$$reg),
12633 Assembler::ASR,
12634 $src3$$constant & 0x1f);
12635 %}
12636
12637 ins_pipe(ialu_reg_reg_shift);
12638 %}
12639
12640 // This pattern is automatically generated from aarch64_ad.m4
12641 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12642 instruct SubL_reg_RShift_reg(iRegLNoSp dst,
12643 iRegL src1, iRegL src2,
12644 immI src3) %{
12645 match(Set dst (SubL src1 (RShiftL src2 src3)));
12646
12647 ins_cost(1.9 * INSN_COST);
12648 format %{ "sub $dst, $src1, $src2, ASR $src3" %}
12649
12650 ins_encode %{
12651 __ sub(as_Register($dst$$reg),
12652 as_Register($src1$$reg),
12653 as_Register($src2$$reg),
12654 Assembler::ASR,
12655 $src3$$constant & 0x3f);
12656 %}
12657
12658 ins_pipe(ialu_reg_reg_shift);
12659 %}
12660
12661 // This pattern is automatically generated from aarch64_ad.m4
12662 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12663 instruct SubI_reg_LShift_reg(iRegINoSp dst,
12664 iRegIorL2I src1, iRegIorL2I src2,
12665 immI src3) %{
12666 match(Set dst (SubI src1 (LShiftI src2 src3)));
12667
12668 ins_cost(1.9 * INSN_COST);
12669 format %{ "subw $dst, $src1, $src2, LSL $src3" %}
12670
12671 ins_encode %{
12672 __ subw(as_Register($dst$$reg),
12673 as_Register($src1$$reg),
12674 as_Register($src2$$reg),
12675 Assembler::LSL,
12676 $src3$$constant & 0x1f);
12677 %}
12678
12679 ins_pipe(ialu_reg_reg_shift);
12680 %}
12681
12682 // This pattern is automatically generated from aarch64_ad.m4
12683 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12684 instruct SubL_reg_LShift_reg(iRegLNoSp dst,
12685 iRegL src1, iRegL src2,
12686 immI src3) %{
12687 match(Set dst (SubL src1 (LShiftL src2 src3)));
12688
12689 ins_cost(1.9 * INSN_COST);
12690 format %{ "sub $dst, $src1, $src2, LSL $src3" %}
12691
12692 ins_encode %{
12693 __ sub(as_Register($dst$$reg),
12694 as_Register($src1$$reg),
12695 as_Register($src2$$reg),
12696 Assembler::LSL,
12697 $src3$$constant & 0x3f);
12698 %}
12699
12700 ins_pipe(ialu_reg_reg_shift);
12701 %}
12702
12703 // This pattern is automatically generated from aarch64_ad.m4
12704 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12705
12706 // Shift Left followed by Shift Right.
12707 // This idiom is used by the compiler for the i2b bytecode etc.
12708 instruct sbfmL(iRegLNoSp dst, iRegL src, immI lshift_count, immI rshift_count)
12709 %{
12710 match(Set dst (RShiftL (LShiftL src lshift_count) rshift_count));
12711 ins_cost(INSN_COST * 2);
12712 format %{ "sbfm $dst, $src, $rshift_count - $lshift_count, #63 - $lshift_count" %}
12713 ins_encode %{
12714 int lshift = $lshift_count$$constant & 63;
12715 int rshift = $rshift_count$$constant & 63;
12716 int s = 63 - lshift;
12717 int r = (rshift - lshift) & 63;
12718 __ sbfm(as_Register($dst$$reg),
12719 as_Register($src$$reg),
12720 r, s);
12721 %}
12722
12723 ins_pipe(ialu_reg_shift);
12724 %}
12725
12726 // This pattern is automatically generated from aarch64_ad.m4
12727 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12728
12729 // Shift Left followed by Shift Right.
12730 // This idiom is used by the compiler for the i2b bytecode etc.
12731 instruct sbfmwI(iRegINoSp dst, iRegIorL2I src, immI lshift_count, immI rshift_count)
12732 %{
12733 match(Set dst (RShiftI (LShiftI src lshift_count) rshift_count));
12734 ins_cost(INSN_COST * 2);
12735 format %{ "sbfmw $dst, $src, $rshift_count - $lshift_count, #31 - $lshift_count" %}
12736 ins_encode %{
12737 int lshift = $lshift_count$$constant & 31;
12738 int rshift = $rshift_count$$constant & 31;
12739 int s = 31 - lshift;
12740 int r = (rshift - lshift) & 31;
12741 __ sbfmw(as_Register($dst$$reg),
12742 as_Register($src$$reg),
12743 r, s);
12744 %}
12745
12746 ins_pipe(ialu_reg_shift);
12747 %}
12748
12749 // This pattern is automatically generated from aarch64_ad.m4
12750 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12751
12752 // Shift Left followed by Shift Right.
12753 // This idiom is used by the compiler for the i2b bytecode etc.
12754 instruct ubfmL(iRegLNoSp dst, iRegL src, immI lshift_count, immI rshift_count)
12755 %{
12756 match(Set dst (URShiftL (LShiftL src lshift_count) rshift_count));
12757 ins_cost(INSN_COST * 2);
12758 format %{ "ubfm $dst, $src, $rshift_count - $lshift_count, #63 - $lshift_count" %}
12759 ins_encode %{
12760 int lshift = $lshift_count$$constant & 63;
12761 int rshift = $rshift_count$$constant & 63;
12762 int s = 63 - lshift;
12763 int r = (rshift - lshift) & 63;
12764 __ ubfm(as_Register($dst$$reg),
12765 as_Register($src$$reg),
12766 r, s);
12767 %}
12768
12769 ins_pipe(ialu_reg_shift);
12770 %}
12771
12772 // This pattern is automatically generated from aarch64_ad.m4
12773 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12774
12775 // Shift Left followed by Shift Right.
12776 // This idiom is used by the compiler for the i2b bytecode etc.
12777 instruct ubfmwI(iRegINoSp dst, iRegIorL2I src, immI lshift_count, immI rshift_count)
12778 %{
12779 match(Set dst (URShiftI (LShiftI src lshift_count) rshift_count));
12780 ins_cost(INSN_COST * 2);
12781 format %{ "ubfmw $dst, $src, $rshift_count - $lshift_count, #31 - $lshift_count" %}
12782 ins_encode %{
12783 int lshift = $lshift_count$$constant & 31;
12784 int rshift = $rshift_count$$constant & 31;
12785 int s = 31 - lshift;
12786 int r = (rshift - lshift) & 31;
12787 __ ubfmw(as_Register($dst$$reg),
12788 as_Register($src$$reg),
12789 r, s);
12790 %}
12791
12792 ins_pipe(ialu_reg_shift);
12793 %}
12794
12795 // Bitfield extract with shift & mask
12796
12797 // This pattern is automatically generated from aarch64_ad.m4
12798 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12799 instruct ubfxwI(iRegINoSp dst, iRegIorL2I src, immI rshift, immI_bitmask mask)
12800 %{
12801 match(Set dst (AndI (URShiftI src rshift) mask));
12802 // Make sure we are not going to exceed what ubfxw can do.
12803 predicate((exact_log2(n->in(2)->get_int() + 1) + (n->in(1)->in(2)->get_int() & 31)) <= (31 + 1));
12804
12805 ins_cost(INSN_COST);
12806 format %{ "ubfxw $dst, $src, $rshift, $mask" %}
12807 ins_encode %{
12808 int rshift = $rshift$$constant & 31;
12809 intptr_t mask = $mask$$constant;
12810 int width = exact_log2(mask+1);
12811 __ ubfxw(as_Register($dst$$reg),
12812 as_Register($src$$reg), rshift, width);
12813 %}
12814 ins_pipe(ialu_reg_shift);
12815 %}
12816
12817 // This pattern is automatically generated from aarch64_ad.m4
12818 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12819 instruct ubfxL(iRegLNoSp dst, iRegL src, immI rshift, immL_bitmask mask)
12820 %{
12821 match(Set dst (AndL (URShiftL src rshift) mask));
12822 // Make sure we are not going to exceed what ubfx can do.
12823 predicate((exact_log2_long(n->in(2)->get_long() + 1) + (n->in(1)->in(2)->get_int() & 63)) <= (63 + 1));
12824
12825 ins_cost(INSN_COST);
12826 format %{ "ubfx $dst, $src, $rshift, $mask" %}
12827 ins_encode %{
12828 int rshift = $rshift$$constant & 63;
12829 intptr_t mask = $mask$$constant;
12830 int width = exact_log2_long(mask+1);
12831 __ ubfx(as_Register($dst$$reg),
12832 as_Register($src$$reg), rshift, width);
12833 %}
12834 ins_pipe(ialu_reg_shift);
12835 %}
12836
12837
12838 // This pattern is automatically generated from aarch64_ad.m4
12839 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12840
12841 // We can use ubfx when extending an And with a mask when we know mask
12842 // is positive. We know that because immI_bitmask guarantees it.
12843 instruct ubfxIConvI2L(iRegLNoSp dst, iRegIorL2I src, immI rshift, immI_bitmask mask)
12844 %{
12845 match(Set dst (ConvI2L (AndI (URShiftI src rshift) mask)));
12846 // Make sure we are not going to exceed what ubfxw can do.
12847 predicate((exact_log2(n->in(1)->in(2)->get_int() + 1) + (n->in(1)->in(1)->in(2)->get_int() & 31)) <= (31 + 1));
12848
12849 ins_cost(INSN_COST * 2);
12850 format %{ "ubfx $dst, $src, $rshift, $mask" %}
12851 ins_encode %{
12852 int rshift = $rshift$$constant & 31;
12853 intptr_t mask = $mask$$constant;
12854 int width = exact_log2(mask+1);
12855 __ ubfx(as_Register($dst$$reg),
12856 as_Register($src$$reg), rshift, width);
12857 %}
12858 ins_pipe(ialu_reg_shift);
12859 %}
12860
12861
12862 // This pattern is automatically generated from aarch64_ad.m4
12863 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12864
12865 // We can use ubfiz when masking by a positive number and then left shifting the result.
12866 // We know that the mask is positive because immI_bitmask guarantees it.
12867 instruct ubfizwI(iRegINoSp dst, iRegIorL2I src, immI lshift, immI_bitmask mask)
12868 %{
12869 match(Set dst (LShiftI (AndI src mask) lshift));
12870 predicate((exact_log2(n->in(1)->in(2)->get_int() + 1) + (n->in(2)->get_int() & 31)) <= (31 + 1));
12871
12872 ins_cost(INSN_COST);
12873 format %{ "ubfizw $dst, $src, $lshift, $mask" %}
12874 ins_encode %{
12875 int lshift = $lshift$$constant & 31;
12876 intptr_t mask = $mask$$constant;
12877 int width = exact_log2(mask+1);
12878 __ ubfizw(as_Register($dst$$reg),
12879 as_Register($src$$reg), lshift, width);
12880 %}
12881 ins_pipe(ialu_reg_shift);
12882 %}
12883
12884 // This pattern is automatically generated from aarch64_ad.m4
12885 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12886
12887 // We can use ubfiz when masking by a positive number and then left shifting the result.
12888 // We know that the mask is positive because immL_bitmask guarantees it.
12889 instruct ubfizL(iRegLNoSp dst, iRegL src, immI lshift, immL_bitmask mask)
12890 %{
12891 match(Set dst (LShiftL (AndL src mask) lshift));
12892 predicate((exact_log2_long(n->in(1)->in(2)->get_long() + 1) + (n->in(2)->get_int() & 63)) <= (63 + 1));
12893
12894 ins_cost(INSN_COST);
12895 format %{ "ubfiz $dst, $src, $lshift, $mask" %}
12896 ins_encode %{
12897 int lshift = $lshift$$constant & 63;
12898 intptr_t mask = $mask$$constant;
12899 int width = exact_log2_long(mask+1);
12900 __ ubfiz(as_Register($dst$$reg),
12901 as_Register($src$$reg), lshift, width);
12902 %}
12903 ins_pipe(ialu_reg_shift);
12904 %}
12905
12906 // This pattern is automatically generated from aarch64_ad.m4
12907 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12908
12909 // We can use ubfiz when masking by a positive number and then left shifting the result.
12910 // We know that the mask is positive because immI_bitmask guarantees it.
12911 instruct ubfizwIConvI2L(iRegLNoSp dst, iRegIorL2I src, immI lshift, immI_bitmask mask)
12912 %{
12913 match(Set dst (ConvI2L (LShiftI (AndI src mask) lshift)));
12914 predicate((exact_log2(n->in(1)->in(1)->in(2)->get_int() + 1) + (n->in(1)->in(2)->get_int() & 31)) <= 31);
12915
12916 ins_cost(INSN_COST);
12917 format %{ "ubfizw $dst, $src, $lshift, $mask" %}
12918 ins_encode %{
12919 int lshift = $lshift$$constant & 31;
12920 intptr_t mask = $mask$$constant;
12921 int width = exact_log2(mask+1);
12922 __ ubfizw(as_Register($dst$$reg),
12923 as_Register($src$$reg), lshift, width);
12924 %}
12925 ins_pipe(ialu_reg_shift);
12926 %}
12927
12928 // This pattern is automatically generated from aarch64_ad.m4
12929 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12930
12931 // We can use ubfiz when masking by a positive number and then left shifting the result.
12932 // We know that the mask is positive because immL_bitmask guarantees it.
12933 instruct ubfizLConvL2I(iRegINoSp dst, iRegL src, immI lshift, immL_positive_bitmaskI mask)
12934 %{
12935 match(Set dst (ConvL2I (LShiftL (AndL src mask) lshift)));
12936 predicate((exact_log2_long(n->in(1)->in(1)->in(2)->get_long() + 1) + (n->in(1)->in(2)->get_int() & 63)) <= 31);
12937
12938 ins_cost(INSN_COST);
12939 format %{ "ubfiz $dst, $src, $lshift, $mask" %}
12940 ins_encode %{
12941 int lshift = $lshift$$constant & 63;
12942 intptr_t mask = $mask$$constant;
12943 int width = exact_log2_long(mask+1);
12944 __ ubfiz(as_Register($dst$$reg),
12945 as_Register($src$$reg), lshift, width);
12946 %}
12947 ins_pipe(ialu_reg_shift);
12948 %}
12949
12950
12951 // This pattern is automatically generated from aarch64_ad.m4
12952 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12953
12954 // If there is a convert I to L block between and AndI and a LShiftL, we can also match ubfiz
12955 instruct ubfizIConvI2L(iRegLNoSp dst, iRegIorL2I src, immI lshift, immI_bitmask mask)
12956 %{
12957 match(Set dst (LShiftL (ConvI2L (AndI src mask)) lshift));
12958 predicate((exact_log2(n->in(1)->in(1)->in(2)->get_int() + 1) + (n->in(2)->get_int() & 63)) <= (63 + 1));
12959
12960 ins_cost(INSN_COST);
12961 format %{ "ubfiz $dst, $src, $lshift, $mask" %}
12962 ins_encode %{
12963 int lshift = $lshift$$constant & 63;
12964 intptr_t mask = $mask$$constant;
12965 int width = exact_log2(mask+1);
12966 __ ubfiz(as_Register($dst$$reg),
12967 as_Register($src$$reg), lshift, width);
12968 %}
12969 ins_pipe(ialu_reg_shift);
12970 %}
12971
12972 // This pattern is automatically generated from aarch64_ad.m4
12973 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12974
12975 // If there is a convert L to I block between and AndL and a LShiftI, we can also match ubfiz
12976 instruct ubfizLConvL2Ix(iRegINoSp dst, iRegL src, immI lshift, immL_positive_bitmaskI mask)
12977 %{
12978 match(Set dst (LShiftI (ConvL2I (AndL src mask)) lshift));
12979 predicate((exact_log2_long(n->in(1)->in(1)->in(2)->get_long() + 1) + (n->in(2)->get_int() & 31)) <= 31);
12980
12981 ins_cost(INSN_COST);
12982 format %{ "ubfiz $dst, $src, $lshift, $mask" %}
12983 ins_encode %{
12984 int lshift = $lshift$$constant & 31;
12985 intptr_t mask = $mask$$constant;
12986 int width = exact_log2(mask+1);
12987 __ ubfiz(as_Register($dst$$reg),
12988 as_Register($src$$reg), lshift, width);
12989 %}
12990 ins_pipe(ialu_reg_shift);
12991 %}
12992
12993 // This pattern is automatically generated from aarch64_ad.m4
12994 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12995
12996 // Can skip int2long conversions after AND with small bitmask
12997 instruct ubfizIConvI2LAndI(iRegLNoSp dst, iRegI src, immI_bitmask msk)
12998 %{
12999 match(Set dst (ConvI2L (AndI src msk)));
13000 ins_cost(INSN_COST);
13001 format %{ "ubfiz $dst, $src, 0, exact_log2($msk + 1) " %}
13002 ins_encode %{
13003 __ ubfiz(as_Register($dst$$reg), as_Register($src$$reg), 0, exact_log2($msk$$constant + 1));
13004 %}
13005 ins_pipe(ialu_reg_shift);
13006 %}
13007
13008
13009 // Rotations
13010 // This pattern is automatically generated from aarch64_ad.m4
13011 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13012 instruct extrOrL(iRegLNoSp dst, iRegL src1, iRegL src2, immI lshift, immI rshift, rFlagsReg cr)
13013 %{
13014 match(Set dst (OrL (LShiftL src1 lshift) (URShiftL src2 rshift)));
13015 predicate(0 == (((n->in(1)->in(2)->get_int() & 63) + (n->in(2)->in(2)->get_int() & 63)) & 63));
13016
13017 ins_cost(INSN_COST);
13018 format %{ "extr $dst, $src1, $src2, #$rshift" %}
13019
13020 ins_encode %{
13021 __ extr(as_Register($dst$$reg), as_Register($src1$$reg), as_Register($src2$$reg),
13022 $rshift$$constant & 63);
13023 %}
13024 ins_pipe(ialu_reg_reg_extr);
13025 %}
13026
13027
13028 // This pattern is automatically generated from aarch64_ad.m4
13029 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13030 instruct extrOrI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI lshift, immI rshift, rFlagsReg cr)
13031 %{
13032 match(Set dst (OrI (LShiftI src1 lshift) (URShiftI src2 rshift)));
13033 predicate(0 == (((n->in(1)->in(2)->get_int() & 31) + (n->in(2)->in(2)->get_int() & 31)) & 31));
13034
13035 ins_cost(INSN_COST);
13036 format %{ "extr $dst, $src1, $src2, #$rshift" %}
13037
13038 ins_encode %{
13039 __ extrw(as_Register($dst$$reg), as_Register($src1$$reg), as_Register($src2$$reg),
13040 $rshift$$constant & 31);
13041 %}
13042 ins_pipe(ialu_reg_reg_extr);
13043 %}
13044
13045
13046 // This pattern is automatically generated from aarch64_ad.m4
13047 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13048 instruct extrAddL(iRegLNoSp dst, iRegL src1, iRegL src2, immI lshift, immI rshift, rFlagsReg cr)
13049 %{
13050 match(Set dst (AddL (LShiftL src1 lshift) (URShiftL src2 rshift)));
13051 predicate(0 == (((n->in(1)->in(2)->get_int() & 63) + (n->in(2)->in(2)->get_int() & 63)) & 63));
13052
13053 ins_cost(INSN_COST);
13054 format %{ "extr $dst, $src1, $src2, #$rshift" %}
13055
13056 ins_encode %{
13057 __ extr(as_Register($dst$$reg), as_Register($src1$$reg), as_Register($src2$$reg),
13058 $rshift$$constant & 63);
13059 %}
13060 ins_pipe(ialu_reg_reg_extr);
13061 %}
13062
13063
13064 // This pattern is automatically generated from aarch64_ad.m4
13065 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13066 instruct extrAddI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI lshift, immI rshift, rFlagsReg cr)
13067 %{
13068 match(Set dst (AddI (LShiftI src1 lshift) (URShiftI src2 rshift)));
13069 predicate(0 == (((n->in(1)->in(2)->get_int() & 31) + (n->in(2)->in(2)->get_int() & 31)) & 31));
13070
13071 ins_cost(INSN_COST);
13072 format %{ "extr $dst, $src1, $src2, #$rshift" %}
13073
13074 ins_encode %{
13075 __ extrw(as_Register($dst$$reg), as_Register($src1$$reg), as_Register($src2$$reg),
13076 $rshift$$constant & 31);
13077 %}
13078 ins_pipe(ialu_reg_reg_extr);
13079 %}
13080
13081
13082 // This pattern is automatically generated from aarch64_ad.m4
13083 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13084 instruct rorI_imm(iRegINoSp dst, iRegI src, immI shift)
13085 %{
13086 match(Set dst (RotateRight src shift));
13087
13088 ins_cost(INSN_COST);
13089 format %{ "ror $dst, $src, $shift" %}
13090
13091 ins_encode %{
13092 __ extrw(as_Register($dst$$reg), as_Register($src$$reg), as_Register($src$$reg),
13093 $shift$$constant & 0x1f);
13094 %}
13095 ins_pipe(ialu_reg_reg_vshift);
13096 %}
13097
13098 // This pattern is automatically generated from aarch64_ad.m4
13099 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13100 instruct rorL_imm(iRegLNoSp dst, iRegL src, immI shift)
13101 %{
13102 match(Set dst (RotateRight src shift));
13103
13104 ins_cost(INSN_COST);
13105 format %{ "ror $dst, $src, $shift" %}
13106
13107 ins_encode %{
13108 __ extr(as_Register($dst$$reg), as_Register($src$$reg), as_Register($src$$reg),
13109 $shift$$constant & 0x3f);
13110 %}
13111 ins_pipe(ialu_reg_reg_vshift);
13112 %}
13113
13114 // This pattern is automatically generated from aarch64_ad.m4
13115 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13116 instruct rorI_reg(iRegINoSp dst, iRegI src, iRegI shift)
13117 %{
13118 match(Set dst (RotateRight src shift));
13119
13120 ins_cost(INSN_COST);
13121 format %{ "ror $dst, $src, $shift" %}
13122
13123 ins_encode %{
13124 __ rorvw(as_Register($dst$$reg), as_Register($src$$reg), as_Register($shift$$reg));
13125 %}
13126 ins_pipe(ialu_reg_reg_vshift);
13127 %}
13128
13129 // This pattern is automatically generated from aarch64_ad.m4
13130 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13131 instruct rorL_reg(iRegLNoSp dst, iRegL src, iRegI shift)
13132 %{
13133 match(Set dst (RotateRight src shift));
13134
13135 ins_cost(INSN_COST);
13136 format %{ "ror $dst, $src, $shift" %}
13137
13138 ins_encode %{
13139 __ rorv(as_Register($dst$$reg), as_Register($src$$reg), as_Register($shift$$reg));
13140 %}
13141 ins_pipe(ialu_reg_reg_vshift);
13142 %}
13143
13144 // This pattern is automatically generated from aarch64_ad.m4
13145 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13146 instruct rolI_reg(iRegINoSp dst, iRegI src, iRegI shift)
13147 %{
13148 match(Set dst (RotateLeft src shift));
13149
13150 ins_cost(INSN_COST);
13151 format %{ "rol $dst, $src, $shift" %}
13152
13153 ins_encode %{
13154 __ subw(rscratch1, zr, as_Register($shift$$reg));
13155 __ rorvw(as_Register($dst$$reg), as_Register($src$$reg), rscratch1);
13156 %}
13157 ins_pipe(ialu_reg_reg_vshift);
13158 %}
13159
13160 // This pattern is automatically generated from aarch64_ad.m4
13161 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13162 instruct rolL_reg(iRegLNoSp dst, iRegL src, iRegI shift)
13163 %{
13164 match(Set dst (RotateLeft src shift));
13165
13166 ins_cost(INSN_COST);
13167 format %{ "rol $dst, $src, $shift" %}
13168
13169 ins_encode %{
13170 __ subw(rscratch1, zr, as_Register($shift$$reg));
13171 __ rorv(as_Register($dst$$reg), as_Register($src$$reg), rscratch1);
13172 %}
13173 ins_pipe(ialu_reg_reg_vshift);
13174 %}
13175
13176
13177 // Add/subtract (extended)
13178
13179 // This pattern is automatically generated from aarch64_ad.m4
13180 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13181 instruct AddExtI(iRegLNoSp dst, iRegL src1, iRegIorL2I src2, rFlagsReg cr)
13182 %{
13183 match(Set dst (AddL src1 (ConvI2L src2)));
13184 ins_cost(INSN_COST);
13185 format %{ "add $dst, $src1, $src2, sxtw" %}
13186
13187 ins_encode %{
13188 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13189 as_Register($src2$$reg), ext::sxtw);
13190 %}
13191 ins_pipe(ialu_reg_reg);
13192 %}
13193
13194 // This pattern is automatically generated from aarch64_ad.m4
13195 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13196 instruct SubExtI(iRegLNoSp dst, iRegL src1, iRegIorL2I src2, rFlagsReg cr)
13197 %{
13198 match(Set dst (SubL src1 (ConvI2L src2)));
13199 ins_cost(INSN_COST);
13200 format %{ "sub $dst, $src1, $src2, sxtw" %}
13201
13202 ins_encode %{
13203 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13204 as_Register($src2$$reg), ext::sxtw);
13205 %}
13206 ins_pipe(ialu_reg_reg);
13207 %}
13208
13209 // This pattern is automatically generated from aarch64_ad.m4
13210 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13211 instruct AddExtI_sxth(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_16 lshift, immI_16 rshift, rFlagsReg cr)
13212 %{
13213 match(Set dst (AddI src1 (RShiftI (LShiftI src2 lshift) rshift)));
13214 ins_cost(INSN_COST);
13215 format %{ "add $dst, $src1, $src2, sxth" %}
13216
13217 ins_encode %{
13218 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13219 as_Register($src2$$reg), ext::sxth);
13220 %}
13221 ins_pipe(ialu_reg_reg);
13222 %}
13223
13224 // This pattern is automatically generated from aarch64_ad.m4
13225 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13226 instruct AddExtI_sxtb(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_24 lshift, immI_24 rshift, rFlagsReg cr)
13227 %{
13228 match(Set dst (AddI src1 (RShiftI (LShiftI src2 lshift) rshift)));
13229 ins_cost(INSN_COST);
13230 format %{ "add $dst, $src1, $src2, sxtb" %}
13231
13232 ins_encode %{
13233 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13234 as_Register($src2$$reg), ext::sxtb);
13235 %}
13236 ins_pipe(ialu_reg_reg);
13237 %}
13238
13239 // This pattern is automatically generated from aarch64_ad.m4
13240 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13241 instruct AddExtI_uxtb(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_24 lshift, immI_24 rshift, rFlagsReg cr)
13242 %{
13243 match(Set dst (AddI src1 (URShiftI (LShiftI src2 lshift) rshift)));
13244 ins_cost(INSN_COST);
13245 format %{ "add $dst, $src1, $src2, uxtb" %}
13246
13247 ins_encode %{
13248 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13249 as_Register($src2$$reg), ext::uxtb);
13250 %}
13251 ins_pipe(ialu_reg_reg);
13252 %}
13253
13254 // This pattern is automatically generated from aarch64_ad.m4
13255 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13256 instruct AddExtL_sxth(iRegLNoSp dst, iRegL src1, iRegL src2, immI_48 lshift, immI_48 rshift, rFlagsReg cr)
13257 %{
13258 match(Set dst (AddL src1 (RShiftL (LShiftL src2 lshift) rshift)));
13259 ins_cost(INSN_COST);
13260 format %{ "add $dst, $src1, $src2, sxth" %}
13261
13262 ins_encode %{
13263 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13264 as_Register($src2$$reg), ext::sxth);
13265 %}
13266 ins_pipe(ialu_reg_reg);
13267 %}
13268
13269 // This pattern is automatically generated from aarch64_ad.m4
13270 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13271 instruct AddExtL_sxtw(iRegLNoSp dst, iRegL src1, iRegL src2, immI_32 lshift, immI_32 rshift, rFlagsReg cr)
13272 %{
13273 match(Set dst (AddL src1 (RShiftL (LShiftL src2 lshift) rshift)));
13274 ins_cost(INSN_COST);
13275 format %{ "add $dst, $src1, $src2, sxtw" %}
13276
13277 ins_encode %{
13278 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13279 as_Register($src2$$reg), ext::sxtw);
13280 %}
13281 ins_pipe(ialu_reg_reg);
13282 %}
13283
13284 // This pattern is automatically generated from aarch64_ad.m4
13285 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13286 instruct AddExtL_sxtb(iRegLNoSp dst, iRegL src1, iRegL src2, immI_56 lshift, immI_56 rshift, rFlagsReg cr)
13287 %{
13288 match(Set dst (AddL src1 (RShiftL (LShiftL src2 lshift) rshift)));
13289 ins_cost(INSN_COST);
13290 format %{ "add $dst, $src1, $src2, sxtb" %}
13291
13292 ins_encode %{
13293 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13294 as_Register($src2$$reg), ext::sxtb);
13295 %}
13296 ins_pipe(ialu_reg_reg);
13297 %}
13298
13299 // This pattern is automatically generated from aarch64_ad.m4
13300 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13301 instruct AddExtL_uxtb(iRegLNoSp dst, iRegL src1, iRegL src2, immI_56 lshift, immI_56 rshift, rFlagsReg cr)
13302 %{
13303 match(Set dst (AddL src1 (URShiftL (LShiftL src2 lshift) rshift)));
13304 ins_cost(INSN_COST);
13305 format %{ "add $dst, $src1, $src2, uxtb" %}
13306
13307 ins_encode %{
13308 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13309 as_Register($src2$$reg), ext::uxtb);
13310 %}
13311 ins_pipe(ialu_reg_reg);
13312 %}
13313
13314 // This pattern is automatically generated from aarch64_ad.m4
13315 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13316 instruct AddExtI_uxtb_and(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_255 mask, rFlagsReg cr)
13317 %{
13318 match(Set dst (AddI src1 (AndI src2 mask)));
13319 ins_cost(INSN_COST);
13320 format %{ "addw $dst, $src1, $src2, uxtb" %}
13321
13322 ins_encode %{
13323 __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
13324 as_Register($src2$$reg), ext::uxtb);
13325 %}
13326 ins_pipe(ialu_reg_reg);
13327 %}
13328
13329 // This pattern is automatically generated from aarch64_ad.m4
13330 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13331 instruct AddExtI_uxth_and(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_65535 mask, rFlagsReg cr)
13332 %{
13333 match(Set dst (AddI src1 (AndI src2 mask)));
13334 ins_cost(INSN_COST);
13335 format %{ "addw $dst, $src1, $src2, uxth" %}
13336
13337 ins_encode %{
13338 __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
13339 as_Register($src2$$reg), ext::uxth);
13340 %}
13341 ins_pipe(ialu_reg_reg);
13342 %}
13343
13344 // This pattern is automatically generated from aarch64_ad.m4
13345 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13346 instruct AddExtL_uxtb_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_255 mask, rFlagsReg cr)
13347 %{
13348 match(Set dst (AddL src1 (AndL src2 mask)));
13349 ins_cost(INSN_COST);
13350 format %{ "add $dst, $src1, $src2, uxtb" %}
13351
13352 ins_encode %{
13353 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13354 as_Register($src2$$reg), ext::uxtb);
13355 %}
13356 ins_pipe(ialu_reg_reg);
13357 %}
13358
13359 // This pattern is automatically generated from aarch64_ad.m4
13360 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13361 instruct AddExtL_uxth_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_65535 mask, rFlagsReg cr)
13362 %{
13363 match(Set dst (AddL src1 (AndL src2 mask)));
13364 ins_cost(INSN_COST);
13365 format %{ "add $dst, $src1, $src2, uxth" %}
13366
13367 ins_encode %{
13368 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13369 as_Register($src2$$reg), ext::uxth);
13370 %}
13371 ins_pipe(ialu_reg_reg);
13372 %}
13373
13374 // This pattern is automatically generated from aarch64_ad.m4
13375 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13376 instruct AddExtL_uxtw_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_4294967295 mask, rFlagsReg cr)
13377 %{
13378 match(Set dst (AddL src1 (AndL src2 mask)));
13379 ins_cost(INSN_COST);
13380 format %{ "add $dst, $src1, $src2, uxtw" %}
13381
13382 ins_encode %{
13383 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13384 as_Register($src2$$reg), ext::uxtw);
13385 %}
13386 ins_pipe(ialu_reg_reg);
13387 %}
13388
13389 // This pattern is automatically generated from aarch64_ad.m4
13390 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13391 instruct SubExtI_uxtb_and(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_255 mask, rFlagsReg cr)
13392 %{
13393 match(Set dst (SubI src1 (AndI src2 mask)));
13394 ins_cost(INSN_COST);
13395 format %{ "subw $dst, $src1, $src2, uxtb" %}
13396
13397 ins_encode %{
13398 __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
13399 as_Register($src2$$reg), ext::uxtb);
13400 %}
13401 ins_pipe(ialu_reg_reg);
13402 %}
13403
13404 // This pattern is automatically generated from aarch64_ad.m4
13405 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13406 instruct SubExtI_uxth_and(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_65535 mask, rFlagsReg cr)
13407 %{
13408 match(Set dst (SubI src1 (AndI src2 mask)));
13409 ins_cost(INSN_COST);
13410 format %{ "subw $dst, $src1, $src2, uxth" %}
13411
13412 ins_encode %{
13413 __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
13414 as_Register($src2$$reg), ext::uxth);
13415 %}
13416 ins_pipe(ialu_reg_reg);
13417 %}
13418
13419 // This pattern is automatically generated from aarch64_ad.m4
13420 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13421 instruct SubExtL_uxtb_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_255 mask, rFlagsReg cr)
13422 %{
13423 match(Set dst (SubL src1 (AndL src2 mask)));
13424 ins_cost(INSN_COST);
13425 format %{ "sub $dst, $src1, $src2, uxtb" %}
13426
13427 ins_encode %{
13428 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13429 as_Register($src2$$reg), ext::uxtb);
13430 %}
13431 ins_pipe(ialu_reg_reg);
13432 %}
13433
13434 // This pattern is automatically generated from aarch64_ad.m4
13435 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13436 instruct SubExtL_uxth_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_65535 mask, rFlagsReg cr)
13437 %{
13438 match(Set dst (SubL src1 (AndL src2 mask)));
13439 ins_cost(INSN_COST);
13440 format %{ "sub $dst, $src1, $src2, uxth" %}
13441
13442 ins_encode %{
13443 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13444 as_Register($src2$$reg), ext::uxth);
13445 %}
13446 ins_pipe(ialu_reg_reg);
13447 %}
13448
13449 // This pattern is automatically generated from aarch64_ad.m4
13450 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13451 instruct SubExtL_uxtw_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_4294967295 mask, rFlagsReg cr)
13452 %{
13453 match(Set dst (SubL src1 (AndL src2 mask)));
13454 ins_cost(INSN_COST);
13455 format %{ "sub $dst, $src1, $src2, uxtw" %}
13456
13457 ins_encode %{
13458 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13459 as_Register($src2$$reg), ext::uxtw);
13460 %}
13461 ins_pipe(ialu_reg_reg);
13462 %}
13463
13464
13465 // This pattern is automatically generated from aarch64_ad.m4
13466 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13467 instruct AddExtL_sxtb_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_56 lshift1, immI_56 rshift1, rFlagsReg cr)
13468 %{
13469 match(Set dst (AddL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
13470 ins_cost(1.9 * INSN_COST);
13471 format %{ "add $dst, $src1, $src2, sxtb #lshift2" %}
13472
13473 ins_encode %{
13474 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13475 as_Register($src2$$reg), ext::sxtb, ($lshift2$$constant));
13476 %}
13477 ins_pipe(ialu_reg_reg_shift);
13478 %}
13479
13480 // This pattern is automatically generated from aarch64_ad.m4
13481 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13482 instruct AddExtL_sxth_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_48 lshift1, immI_48 rshift1, rFlagsReg cr)
13483 %{
13484 match(Set dst (AddL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
13485 ins_cost(1.9 * INSN_COST);
13486 format %{ "add $dst, $src1, $src2, sxth #lshift2" %}
13487
13488 ins_encode %{
13489 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13490 as_Register($src2$$reg), ext::sxth, ($lshift2$$constant));
13491 %}
13492 ins_pipe(ialu_reg_reg_shift);
13493 %}
13494
13495 // This pattern is automatically generated from aarch64_ad.m4
13496 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13497 instruct AddExtL_sxtw_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_32 lshift1, immI_32 rshift1, rFlagsReg cr)
13498 %{
13499 match(Set dst (AddL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
13500 ins_cost(1.9 * INSN_COST);
13501 format %{ "add $dst, $src1, $src2, sxtw #lshift2" %}
13502
13503 ins_encode %{
13504 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13505 as_Register($src2$$reg), ext::sxtw, ($lshift2$$constant));
13506 %}
13507 ins_pipe(ialu_reg_reg_shift);
13508 %}
13509
13510 // This pattern is automatically generated from aarch64_ad.m4
13511 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13512 instruct SubExtL_sxtb_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_56 lshift1, immI_56 rshift1, rFlagsReg cr)
13513 %{
13514 match(Set dst (SubL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
13515 ins_cost(1.9 * INSN_COST);
13516 format %{ "sub $dst, $src1, $src2, sxtb #lshift2" %}
13517
13518 ins_encode %{
13519 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13520 as_Register($src2$$reg), ext::sxtb, ($lshift2$$constant));
13521 %}
13522 ins_pipe(ialu_reg_reg_shift);
13523 %}
13524
13525 // This pattern is automatically generated from aarch64_ad.m4
13526 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13527 instruct SubExtL_sxth_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_48 lshift1, immI_48 rshift1, rFlagsReg cr)
13528 %{
13529 match(Set dst (SubL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
13530 ins_cost(1.9 * INSN_COST);
13531 format %{ "sub $dst, $src1, $src2, sxth #lshift2" %}
13532
13533 ins_encode %{
13534 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13535 as_Register($src2$$reg), ext::sxth, ($lshift2$$constant));
13536 %}
13537 ins_pipe(ialu_reg_reg_shift);
13538 %}
13539
13540 // This pattern is automatically generated from aarch64_ad.m4
13541 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13542 instruct SubExtL_sxtw_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_32 lshift1, immI_32 rshift1, rFlagsReg cr)
13543 %{
13544 match(Set dst (SubL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
13545 ins_cost(1.9 * INSN_COST);
13546 format %{ "sub $dst, $src1, $src2, sxtw #lshift2" %}
13547
13548 ins_encode %{
13549 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13550 as_Register($src2$$reg), ext::sxtw, ($lshift2$$constant));
13551 %}
13552 ins_pipe(ialu_reg_reg_shift);
13553 %}
13554
13555 // This pattern is automatically generated from aarch64_ad.m4
13556 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13557 instruct AddExtI_sxtb_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immIExt lshift2, immI_24 lshift1, immI_24 rshift1, rFlagsReg cr)
13558 %{
13559 match(Set dst (AddI src1 (LShiftI (RShiftI (LShiftI src2 lshift1) rshift1) lshift2)));
13560 ins_cost(1.9 * INSN_COST);
13561 format %{ "addw $dst, $src1, $src2, sxtb #lshift2" %}
13562
13563 ins_encode %{
13564 __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
13565 as_Register($src2$$reg), ext::sxtb, ($lshift2$$constant));
13566 %}
13567 ins_pipe(ialu_reg_reg_shift);
13568 %}
13569
13570 // This pattern is automatically generated from aarch64_ad.m4
13571 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13572 instruct AddExtI_sxth_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immIExt lshift2, immI_16 lshift1, immI_16 rshift1, rFlagsReg cr)
13573 %{
13574 match(Set dst (AddI src1 (LShiftI (RShiftI (LShiftI src2 lshift1) rshift1) lshift2)));
13575 ins_cost(1.9 * INSN_COST);
13576 format %{ "addw $dst, $src1, $src2, sxth #lshift2" %}
13577
13578 ins_encode %{
13579 __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
13580 as_Register($src2$$reg), ext::sxth, ($lshift2$$constant));
13581 %}
13582 ins_pipe(ialu_reg_reg_shift);
13583 %}
13584
13585 // This pattern is automatically generated from aarch64_ad.m4
13586 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13587 instruct SubExtI_sxtb_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immIExt lshift2, immI_24 lshift1, immI_24 rshift1, rFlagsReg cr)
13588 %{
13589 match(Set dst (SubI src1 (LShiftI (RShiftI (LShiftI src2 lshift1) rshift1) lshift2)));
13590 ins_cost(1.9 * INSN_COST);
13591 format %{ "subw $dst, $src1, $src2, sxtb #lshift2" %}
13592
13593 ins_encode %{
13594 __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
13595 as_Register($src2$$reg), ext::sxtb, ($lshift2$$constant));
13596 %}
13597 ins_pipe(ialu_reg_reg_shift);
13598 %}
13599
13600 // This pattern is automatically generated from aarch64_ad.m4
13601 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13602 instruct SubExtI_sxth_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immIExt lshift2, immI_16 lshift1, immI_16 rshift1, rFlagsReg cr)
13603 %{
13604 match(Set dst (SubI src1 (LShiftI (RShiftI (LShiftI src2 lshift1) rshift1) lshift2)));
13605 ins_cost(1.9 * INSN_COST);
13606 format %{ "subw $dst, $src1, $src2, sxth #lshift2" %}
13607
13608 ins_encode %{
13609 __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
13610 as_Register($src2$$reg), ext::sxth, ($lshift2$$constant));
13611 %}
13612 ins_pipe(ialu_reg_reg_shift);
13613 %}
13614
13615 // This pattern is automatically generated from aarch64_ad.m4
13616 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13617 instruct AddExtI_shift(iRegLNoSp dst, iRegL src1, iRegIorL2I src2, immIExt lshift, rFlagsReg cr)
13618 %{
13619 match(Set dst (AddL src1 (LShiftL (ConvI2L src2) lshift)));
13620 ins_cost(1.9 * INSN_COST);
13621 format %{ "add $dst, $src1, $src2, sxtw #lshift" %}
13622
13623 ins_encode %{
13624 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13625 as_Register($src2$$reg), ext::sxtw, ($lshift$$constant));
13626 %}
13627 ins_pipe(ialu_reg_reg_shift);
13628 %}
13629
13630 // This pattern is automatically generated from aarch64_ad.m4
13631 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13632 instruct SubExtI_shift(iRegLNoSp dst, iRegL src1, iRegIorL2I src2, immIExt lshift, rFlagsReg cr)
13633 %{
13634 match(Set dst (SubL src1 (LShiftL (ConvI2L src2) lshift)));
13635 ins_cost(1.9 * INSN_COST);
13636 format %{ "sub $dst, $src1, $src2, sxtw #lshift" %}
13637
13638 ins_encode %{
13639 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13640 as_Register($src2$$reg), ext::sxtw, ($lshift$$constant));
13641 %}
13642 ins_pipe(ialu_reg_reg_shift);
13643 %}
13644
13645 // This pattern is automatically generated from aarch64_ad.m4
13646 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13647 instruct AddExtL_uxtb_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_255 mask, immIExt lshift, rFlagsReg cr)
13648 %{
13649 match(Set dst (AddL src1 (LShiftL (AndL src2 mask) lshift)));
13650 ins_cost(1.9 * INSN_COST);
13651 format %{ "add $dst, $src1, $src2, uxtb #lshift" %}
13652
13653 ins_encode %{
13654 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13655 as_Register($src2$$reg), ext::uxtb, ($lshift$$constant));
13656 %}
13657 ins_pipe(ialu_reg_reg_shift);
13658 %}
13659
13660 // This pattern is automatically generated from aarch64_ad.m4
13661 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13662 instruct AddExtL_uxth_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_65535 mask, immIExt lshift, rFlagsReg cr)
13663 %{
13664 match(Set dst (AddL src1 (LShiftL (AndL src2 mask) lshift)));
13665 ins_cost(1.9 * INSN_COST);
13666 format %{ "add $dst, $src1, $src2, uxth #lshift" %}
13667
13668 ins_encode %{
13669 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13670 as_Register($src2$$reg), ext::uxth, ($lshift$$constant));
13671 %}
13672 ins_pipe(ialu_reg_reg_shift);
13673 %}
13674
13675 // This pattern is automatically generated from aarch64_ad.m4
13676 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13677 instruct AddExtL_uxtw_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_4294967295 mask, immIExt lshift, rFlagsReg cr)
13678 %{
13679 match(Set dst (AddL src1 (LShiftL (AndL src2 mask) lshift)));
13680 ins_cost(1.9 * INSN_COST);
13681 format %{ "add $dst, $src1, $src2, uxtw #lshift" %}
13682
13683 ins_encode %{
13684 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13685 as_Register($src2$$reg), ext::uxtw, ($lshift$$constant));
13686 %}
13687 ins_pipe(ialu_reg_reg_shift);
13688 %}
13689
13690 // This pattern is automatically generated from aarch64_ad.m4
13691 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13692 instruct SubExtL_uxtb_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_255 mask, immIExt lshift, rFlagsReg cr)
13693 %{
13694 match(Set dst (SubL src1 (LShiftL (AndL src2 mask) lshift)));
13695 ins_cost(1.9 * INSN_COST);
13696 format %{ "sub $dst, $src1, $src2, uxtb #lshift" %}
13697
13698 ins_encode %{
13699 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13700 as_Register($src2$$reg), ext::uxtb, ($lshift$$constant));
13701 %}
13702 ins_pipe(ialu_reg_reg_shift);
13703 %}
13704
13705 // This pattern is automatically generated from aarch64_ad.m4
13706 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13707 instruct SubExtL_uxth_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_65535 mask, immIExt lshift, rFlagsReg cr)
13708 %{
13709 match(Set dst (SubL src1 (LShiftL (AndL src2 mask) lshift)));
13710 ins_cost(1.9 * INSN_COST);
13711 format %{ "sub $dst, $src1, $src2, uxth #lshift" %}
13712
13713 ins_encode %{
13714 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13715 as_Register($src2$$reg), ext::uxth, ($lshift$$constant));
13716 %}
13717 ins_pipe(ialu_reg_reg_shift);
13718 %}
13719
13720 // This pattern is automatically generated from aarch64_ad.m4
13721 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13722 instruct SubExtL_uxtw_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_4294967295 mask, immIExt lshift, rFlagsReg cr)
13723 %{
13724 match(Set dst (SubL src1 (LShiftL (AndL src2 mask) lshift)));
13725 ins_cost(1.9 * INSN_COST);
13726 format %{ "sub $dst, $src1, $src2, uxtw #lshift" %}
13727
13728 ins_encode %{
13729 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13730 as_Register($src2$$reg), ext::uxtw, ($lshift$$constant));
13731 %}
13732 ins_pipe(ialu_reg_reg_shift);
13733 %}
13734
13735 // This pattern is automatically generated from aarch64_ad.m4
13736 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13737 instruct AddExtI_uxtb_and_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_255 mask, immIExt lshift, rFlagsReg cr)
13738 %{
13739 match(Set dst (AddI src1 (LShiftI (AndI src2 mask) lshift)));
13740 ins_cost(1.9 * INSN_COST);
13741 format %{ "addw $dst, $src1, $src2, uxtb #lshift" %}
13742
13743 ins_encode %{
13744 __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
13745 as_Register($src2$$reg), ext::uxtb, ($lshift$$constant));
13746 %}
13747 ins_pipe(ialu_reg_reg_shift);
13748 %}
13749
13750 // This pattern is automatically generated from aarch64_ad.m4
13751 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13752 instruct AddExtI_uxth_and_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_65535 mask, immIExt lshift, rFlagsReg cr)
13753 %{
13754 match(Set dst (AddI src1 (LShiftI (AndI src2 mask) lshift)));
13755 ins_cost(1.9 * INSN_COST);
13756 format %{ "addw $dst, $src1, $src2, uxth #lshift" %}
13757
13758 ins_encode %{
13759 __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
13760 as_Register($src2$$reg), ext::uxth, ($lshift$$constant));
13761 %}
13762 ins_pipe(ialu_reg_reg_shift);
13763 %}
13764
13765 // This pattern is automatically generated from aarch64_ad.m4
13766 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13767 instruct SubExtI_uxtb_and_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_255 mask, immIExt lshift, rFlagsReg cr)
13768 %{
13769 match(Set dst (SubI src1 (LShiftI (AndI src2 mask) lshift)));
13770 ins_cost(1.9 * INSN_COST);
13771 format %{ "subw $dst, $src1, $src2, uxtb #lshift" %}
13772
13773 ins_encode %{
13774 __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
13775 as_Register($src2$$reg), ext::uxtb, ($lshift$$constant));
13776 %}
13777 ins_pipe(ialu_reg_reg_shift);
13778 %}
13779
13780 // This pattern is automatically generated from aarch64_ad.m4
13781 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13782 instruct SubExtI_uxth_and_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_65535 mask, immIExt lshift, rFlagsReg cr)
13783 %{
13784 match(Set dst (SubI src1 (LShiftI (AndI src2 mask) lshift)));
13785 ins_cost(1.9 * INSN_COST);
13786 format %{ "subw $dst, $src1, $src2, uxth #lshift" %}
13787
13788 ins_encode %{
13789 __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
13790 as_Register($src2$$reg), ext::uxth, ($lshift$$constant));
13791 %}
13792 ins_pipe(ialu_reg_reg_shift);
13793 %}
13794
13795
13796
13797 // END This section of the file is automatically generated. Do not edit --------------
13798
13799
13800 // ============================================================================
13801 // Floating Point Arithmetic Instructions
13802
13803 instruct addF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
13804 match(Set dst (AddF src1 src2));
13805
13806 ins_cost(INSN_COST * 5);
13807 format %{ "fadds $dst, $src1, $src2" %}
13808
13809 ins_encode %{
13810 __ fadds(as_FloatRegister($dst$$reg),
13811 as_FloatRegister($src1$$reg),
13812 as_FloatRegister($src2$$reg));
13813 %}
13814
13815 ins_pipe(fp_dop_reg_reg_s);
13816 %}
13817
13818 instruct addD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
13819 match(Set dst (AddD src1 src2));
13820
13821 ins_cost(INSN_COST * 5);
13822 format %{ "faddd $dst, $src1, $src2" %}
13823
13824 ins_encode %{
13825 __ faddd(as_FloatRegister($dst$$reg),
13826 as_FloatRegister($src1$$reg),
13827 as_FloatRegister($src2$$reg));
13828 %}
13829
13830 ins_pipe(fp_dop_reg_reg_d);
13831 %}
13832
13833 instruct subF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
13834 match(Set dst (SubF src1 src2));
13835
13836 ins_cost(INSN_COST * 5);
13837 format %{ "fsubs $dst, $src1, $src2" %}
13838
13839 ins_encode %{
13840 __ fsubs(as_FloatRegister($dst$$reg),
13841 as_FloatRegister($src1$$reg),
13842 as_FloatRegister($src2$$reg));
13843 %}
13844
13845 ins_pipe(fp_dop_reg_reg_s);
13846 %}
13847
13848 instruct subD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
13849 match(Set dst (SubD src1 src2));
13850
13851 ins_cost(INSN_COST * 5);
13852 format %{ "fsubd $dst, $src1, $src2" %}
13853
13854 ins_encode %{
13855 __ fsubd(as_FloatRegister($dst$$reg),
13856 as_FloatRegister($src1$$reg),
13857 as_FloatRegister($src2$$reg));
13858 %}
13859
13860 ins_pipe(fp_dop_reg_reg_d);
13861 %}
13862
13863 instruct mulF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
13864 match(Set dst (MulF src1 src2));
13865
13866 ins_cost(INSN_COST * 6);
13867 format %{ "fmuls $dst, $src1, $src2" %}
13868
13869 ins_encode %{
13870 __ fmuls(as_FloatRegister($dst$$reg),
13871 as_FloatRegister($src1$$reg),
13872 as_FloatRegister($src2$$reg));
13873 %}
13874
13875 ins_pipe(fp_dop_reg_reg_s);
13876 %}
13877
13878 instruct mulD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
13879 match(Set dst (MulD src1 src2));
13880
13881 ins_cost(INSN_COST * 6);
13882 format %{ "fmuld $dst, $src1, $src2" %}
13883
13884 ins_encode %{
13885 __ fmuld(as_FloatRegister($dst$$reg),
13886 as_FloatRegister($src1$$reg),
13887 as_FloatRegister($src2$$reg));
13888 %}
13889
13890 ins_pipe(fp_dop_reg_reg_d);
13891 %}
13892
13893 // src1 * src2 + src3
13894 instruct maddF_reg_reg(vRegF dst, vRegF src1, vRegF src2, vRegF src3) %{
13895 predicate(UseFMA);
13896 match(Set dst (FmaF src3 (Binary src1 src2)));
13897
13898 format %{ "fmadds $dst, $src1, $src2, $src3" %}
13899
13900 ins_encode %{
13901 __ fmadds(as_FloatRegister($dst$$reg),
13902 as_FloatRegister($src1$$reg),
13903 as_FloatRegister($src2$$reg),
13904 as_FloatRegister($src3$$reg));
13905 %}
13906
13907 ins_pipe(pipe_class_default);
13908 %}
13909
13910 // src1 * src2 + src3
13911 instruct maddD_reg_reg(vRegD dst, vRegD src1, vRegD src2, vRegD src3) %{
13912 predicate(UseFMA);
13913 match(Set dst (FmaD src3 (Binary src1 src2)));
13914
13915 format %{ "fmaddd $dst, $src1, $src2, $src3" %}
13916
13917 ins_encode %{
13918 __ fmaddd(as_FloatRegister($dst$$reg),
13919 as_FloatRegister($src1$$reg),
13920 as_FloatRegister($src2$$reg),
13921 as_FloatRegister($src3$$reg));
13922 %}
13923
13924 ins_pipe(pipe_class_default);
13925 %}
13926
13927 // -src1 * src2 + src3
13928 instruct msubF_reg_reg(vRegF dst, vRegF src1, vRegF src2, vRegF src3) %{
13929 predicate(UseFMA);
13930 match(Set dst (FmaF src3 (Binary (NegF src1) src2)));
13931 match(Set dst (FmaF src3 (Binary src1 (NegF src2))));
13932
13933 format %{ "fmsubs $dst, $src1, $src2, $src3" %}
13934
13935 ins_encode %{
13936 __ fmsubs(as_FloatRegister($dst$$reg),
13937 as_FloatRegister($src1$$reg),
13938 as_FloatRegister($src2$$reg),
13939 as_FloatRegister($src3$$reg));
13940 %}
13941
13942 ins_pipe(pipe_class_default);
13943 %}
13944
13945 // -src1 * src2 + src3
13946 instruct msubD_reg_reg(vRegD dst, vRegD src1, vRegD src2, vRegD src3) %{
13947 predicate(UseFMA);
13948 match(Set dst (FmaD src3 (Binary (NegD src1) src2)));
13949 match(Set dst (FmaD src3 (Binary src1 (NegD src2))));
13950
13951 format %{ "fmsubd $dst, $src1, $src2, $src3" %}
13952
13953 ins_encode %{
13954 __ fmsubd(as_FloatRegister($dst$$reg),
13955 as_FloatRegister($src1$$reg),
13956 as_FloatRegister($src2$$reg),
13957 as_FloatRegister($src3$$reg));
13958 %}
13959
13960 ins_pipe(pipe_class_default);
13961 %}
13962
13963 // -src1 * src2 - src3
13964 instruct mnaddF_reg_reg(vRegF dst, vRegF src1, vRegF src2, vRegF src3) %{
13965 predicate(UseFMA);
13966 match(Set dst (FmaF (NegF src3) (Binary (NegF src1) src2)));
13967 match(Set dst (FmaF (NegF src3) (Binary src1 (NegF src2))));
13968
13969 format %{ "fnmadds $dst, $src1, $src2, $src3" %}
13970
13971 ins_encode %{
13972 __ fnmadds(as_FloatRegister($dst$$reg),
13973 as_FloatRegister($src1$$reg),
13974 as_FloatRegister($src2$$reg),
13975 as_FloatRegister($src3$$reg));
13976 %}
13977
13978 ins_pipe(pipe_class_default);
13979 %}
13980
13981 // -src1 * src2 - src3
13982 instruct mnaddD_reg_reg(vRegD dst, vRegD src1, vRegD src2, vRegD src3) %{
13983 predicate(UseFMA);
13984 match(Set dst (FmaD (NegD src3) (Binary (NegD src1) src2)));
13985 match(Set dst (FmaD (NegD src3) (Binary src1 (NegD src2))));
13986
13987 format %{ "fnmaddd $dst, $src1, $src2, $src3" %}
13988
13989 ins_encode %{
13990 __ fnmaddd(as_FloatRegister($dst$$reg),
13991 as_FloatRegister($src1$$reg),
13992 as_FloatRegister($src2$$reg),
13993 as_FloatRegister($src3$$reg));
13994 %}
13995
13996 ins_pipe(pipe_class_default);
13997 %}
13998
13999 // src1 * src2 - src3
14000 instruct mnsubF_reg_reg(vRegF dst, vRegF src1, vRegF src2, vRegF src3, immF0 zero) %{
14001 predicate(UseFMA);
14002 match(Set dst (FmaF (NegF src3) (Binary src1 src2)));
14003
14004 format %{ "fnmsubs $dst, $src1, $src2, $src3" %}
14005
14006 ins_encode %{
14007 __ fnmsubs(as_FloatRegister($dst$$reg),
14008 as_FloatRegister($src1$$reg),
14009 as_FloatRegister($src2$$reg),
14010 as_FloatRegister($src3$$reg));
14011 %}
14012
14013 ins_pipe(pipe_class_default);
14014 %}
14015
14016 // src1 * src2 - src3
14017 instruct mnsubD_reg_reg(vRegD dst, vRegD src1, vRegD src2, vRegD src3, immD0 zero) %{
14018 predicate(UseFMA);
14019 match(Set dst (FmaD (NegD src3) (Binary src1 src2)));
14020
14021 format %{ "fnmsubd $dst, $src1, $src2, $src3" %}
14022
14023 ins_encode %{
14024 // n.b. insn name should be fnmsubd
14025 __ fnmsub(as_FloatRegister($dst$$reg),
14026 as_FloatRegister($src1$$reg),
14027 as_FloatRegister($src2$$reg),
14028 as_FloatRegister($src3$$reg));
14029 %}
14030
14031 ins_pipe(pipe_class_default);
14032 %}
14033
14034
14035 // Math.max(FF)F
14036 instruct maxF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
14037 match(Set dst (MaxF src1 src2));
14038
14039 format %{ "fmaxs $dst, $src1, $src2" %}
14040 ins_encode %{
14041 __ fmaxs(as_FloatRegister($dst$$reg),
14042 as_FloatRegister($src1$$reg),
14043 as_FloatRegister($src2$$reg));
14044 %}
14045
14046 ins_pipe(fp_dop_reg_reg_s);
14047 %}
14048
14049 // Math.min(FF)F
14050 instruct minF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
14051 match(Set dst (MinF src1 src2));
14052
14053 format %{ "fmins $dst, $src1, $src2" %}
14054 ins_encode %{
14055 __ fmins(as_FloatRegister($dst$$reg),
14056 as_FloatRegister($src1$$reg),
14057 as_FloatRegister($src2$$reg));
14058 %}
14059
14060 ins_pipe(fp_dop_reg_reg_s);
14061 %}
14062
14063 // Math.max(DD)D
14064 instruct maxD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
14065 match(Set dst (MaxD src1 src2));
14066
14067 format %{ "fmaxd $dst, $src1, $src2" %}
14068 ins_encode %{
14069 __ fmaxd(as_FloatRegister($dst$$reg),
14070 as_FloatRegister($src1$$reg),
14071 as_FloatRegister($src2$$reg));
14072 %}
14073
14074 ins_pipe(fp_dop_reg_reg_d);
14075 %}
14076
14077 // Math.min(DD)D
14078 instruct minD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
14079 match(Set dst (MinD src1 src2));
14080
14081 format %{ "fmind $dst, $src1, $src2" %}
14082 ins_encode %{
14083 __ fmind(as_FloatRegister($dst$$reg),
14084 as_FloatRegister($src1$$reg),
14085 as_FloatRegister($src2$$reg));
14086 %}
14087
14088 ins_pipe(fp_dop_reg_reg_d);
14089 %}
14090
14091
14092 instruct divF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
14093 match(Set dst (DivF src1 src2));
14094
14095 ins_cost(INSN_COST * 18);
14096 format %{ "fdivs $dst, $src1, $src2" %}
14097
14098 ins_encode %{
14099 __ fdivs(as_FloatRegister($dst$$reg),
14100 as_FloatRegister($src1$$reg),
14101 as_FloatRegister($src2$$reg));
14102 %}
14103
14104 ins_pipe(fp_div_s);
14105 %}
14106
14107 instruct divD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
14108 match(Set dst (DivD src1 src2));
14109
14110 ins_cost(INSN_COST * 32);
14111 format %{ "fdivd $dst, $src1, $src2" %}
14112
14113 ins_encode %{
14114 __ fdivd(as_FloatRegister($dst$$reg),
14115 as_FloatRegister($src1$$reg),
14116 as_FloatRegister($src2$$reg));
14117 %}
14118
14119 ins_pipe(fp_div_d);
14120 %}
14121
14122 instruct negF_reg_reg(vRegF dst, vRegF src) %{
14123 match(Set dst (NegF src));
14124
14125 ins_cost(INSN_COST * 3);
14126 format %{ "fneg $dst, $src" %}
14127
14128 ins_encode %{
14129 __ fnegs(as_FloatRegister($dst$$reg),
14130 as_FloatRegister($src$$reg));
14131 %}
14132
14133 ins_pipe(fp_uop_s);
14134 %}
14135
14136 instruct negD_reg_reg(vRegD dst, vRegD src) %{
14137 match(Set dst (NegD src));
14138
14139 ins_cost(INSN_COST * 3);
14140 format %{ "fnegd $dst, $src" %}
14141
14142 ins_encode %{
14143 __ fnegd(as_FloatRegister($dst$$reg),
14144 as_FloatRegister($src$$reg));
14145 %}
14146
14147 ins_pipe(fp_uop_d);
14148 %}
14149
14150 instruct absI_reg(iRegINoSp dst, iRegIorL2I src, rFlagsReg cr)
14151 %{
14152 match(Set dst (AbsI src));
14153
14154 effect(KILL cr);
14155 ins_cost(INSN_COST * 2);
14156 format %{ "cmpw $src, zr\n\t"
14157 "cnegw $dst, $src, Assembler::LT\t# int abs"
14158 %}
14159
14160 ins_encode %{
14161 __ cmpw(as_Register($src$$reg), zr);
14162 __ cnegw(as_Register($dst$$reg), as_Register($src$$reg), Assembler::LT);
14163 %}
14164 ins_pipe(pipe_class_default);
14165 %}
14166
14167 instruct absL_reg(iRegLNoSp dst, iRegL src, rFlagsReg cr)
14168 %{
14169 match(Set dst (AbsL src));
14170
14171 effect(KILL cr);
14172 ins_cost(INSN_COST * 2);
14173 format %{ "cmp $src, zr\n\t"
14174 "cneg $dst, $src, Assembler::LT\t# long abs"
14175 %}
14176
14177 ins_encode %{
14178 __ cmp(as_Register($src$$reg), zr);
14179 __ cneg(as_Register($dst$$reg), as_Register($src$$reg), Assembler::LT);
14180 %}
14181 ins_pipe(pipe_class_default);
14182 %}
14183
14184 instruct absF_reg(vRegF dst, vRegF src) %{
14185 match(Set dst (AbsF src));
14186
14187 ins_cost(INSN_COST * 3);
14188 format %{ "fabss $dst, $src" %}
14189 ins_encode %{
14190 __ fabss(as_FloatRegister($dst$$reg),
14191 as_FloatRegister($src$$reg));
14192 %}
14193
14194 ins_pipe(fp_uop_s);
14195 %}
14196
14197 instruct absD_reg(vRegD dst, vRegD src) %{
14198 match(Set dst (AbsD src));
14199
14200 ins_cost(INSN_COST * 3);
14201 format %{ "fabsd $dst, $src" %}
14202 ins_encode %{
14203 __ fabsd(as_FloatRegister($dst$$reg),
14204 as_FloatRegister($src$$reg));
14205 %}
14206
14207 ins_pipe(fp_uop_d);
14208 %}
14209
14210 instruct absdF_reg(vRegF dst, vRegF src1, vRegF src2) %{
14211 match(Set dst (AbsF (SubF src1 src2)));
14212
14213 ins_cost(INSN_COST * 3);
14214 format %{ "fabds $dst, $src1, $src2" %}
14215 ins_encode %{
14216 __ fabds(as_FloatRegister($dst$$reg),
14217 as_FloatRegister($src1$$reg),
14218 as_FloatRegister($src2$$reg));
14219 %}
14220
14221 ins_pipe(fp_uop_s);
14222 %}
14223
14224 instruct absdD_reg(vRegD dst, vRegD src1, vRegD src2) %{
14225 match(Set dst (AbsD (SubD src1 src2)));
14226
14227 ins_cost(INSN_COST * 3);
14228 format %{ "fabdd $dst, $src1, $src2" %}
14229 ins_encode %{
14230 __ fabdd(as_FloatRegister($dst$$reg),
14231 as_FloatRegister($src1$$reg),
14232 as_FloatRegister($src2$$reg));
14233 %}
14234
14235 ins_pipe(fp_uop_d);
14236 %}
14237
14238 instruct sqrtD_reg(vRegD dst, vRegD src) %{
14239 match(Set dst (SqrtD src));
14240
14241 ins_cost(INSN_COST * 50);
14242 format %{ "fsqrtd $dst, $src" %}
14243 ins_encode %{
14244 __ fsqrtd(as_FloatRegister($dst$$reg),
14245 as_FloatRegister($src$$reg));
14246 %}
14247
14248 ins_pipe(fp_div_s);
14249 %}
14250
14251 instruct sqrtF_reg(vRegF dst, vRegF src) %{
14252 match(Set dst (SqrtF src));
14253
14254 ins_cost(INSN_COST * 50);
14255 format %{ "fsqrts $dst, $src" %}
14256 ins_encode %{
14257 __ fsqrts(as_FloatRegister($dst$$reg),
14258 as_FloatRegister($src$$reg));
14259 %}
14260
14261 ins_pipe(fp_div_d);
14262 %}
14263
14264 // Math.rint, floor, ceil
14265 instruct roundD_reg(vRegD dst, vRegD src, immI rmode) %{
14266 match(Set dst (RoundDoubleMode src rmode));
14267 format %{ "frint $dst, $src, $rmode" %}
14268 ins_encode %{
14269 switch ($rmode$$constant) {
14270 case RoundDoubleModeNode::rmode_rint:
14271 __ frintnd(as_FloatRegister($dst$$reg),
14272 as_FloatRegister($src$$reg));
14273 break;
14274 case RoundDoubleModeNode::rmode_floor:
14275 __ frintmd(as_FloatRegister($dst$$reg),
14276 as_FloatRegister($src$$reg));
14277 break;
14278 case RoundDoubleModeNode::rmode_ceil:
14279 __ frintpd(as_FloatRegister($dst$$reg),
14280 as_FloatRegister($src$$reg));
14281 break;
14282 }
14283 %}
14284 ins_pipe(fp_uop_d);
14285 %}
14286
14287 instruct copySignD_reg(vRegD dst, vRegD src1, vRegD src2, vRegD zero) %{
14288 match(Set dst (CopySignD src1 (Binary src2 zero)));
14289 effect(TEMP_DEF dst, USE src1, USE src2, USE zero);
14290 format %{ "CopySignD $dst $src1 $src2" %}
14291 ins_encode %{
14292 FloatRegister dst = as_FloatRegister($dst$$reg),
14293 src1 = as_FloatRegister($src1$$reg),
14294 src2 = as_FloatRegister($src2$$reg),
14295 zero = as_FloatRegister($zero$$reg);
14296 __ fnegd(dst, zero);
14297 __ bsl(dst, __ T8B, src2, src1);
14298 %}
14299 ins_pipe(fp_uop_d);
14300 %}
14301
14302 instruct copySignF_reg(vRegF dst, vRegF src1, vRegF src2) %{
14303 match(Set dst (CopySignF src1 src2));
14304 effect(TEMP_DEF dst, USE src1, USE src2);
14305 format %{ "CopySignF $dst $src1 $src2" %}
14306 ins_encode %{
14307 FloatRegister dst = as_FloatRegister($dst$$reg),
14308 src1 = as_FloatRegister($src1$$reg),
14309 src2 = as_FloatRegister($src2$$reg);
14310 __ movi(dst, __ T2S, 0x80, 24);
14311 __ bsl(dst, __ T8B, src2, src1);
14312 %}
14313 ins_pipe(fp_uop_d);
14314 %}
14315
14316 instruct signumD_reg(vRegD dst, vRegD src, vRegD zero, vRegD one) %{
14317 match(Set dst (SignumD src (Binary zero one)));
14318 effect(TEMP_DEF dst, USE src, USE zero, USE one);
14319 format %{ "signumD $dst, $src" %}
14320 ins_encode %{
14321 FloatRegister src = as_FloatRegister($src$$reg),
14322 dst = as_FloatRegister($dst$$reg),
14323 zero = as_FloatRegister($zero$$reg),
14324 one = as_FloatRegister($one$$reg);
14325 __ facgtd(dst, src, zero); // dst=0 for +-0.0 and NaN. 0xFFF..F otherwise
14326 __ ushrd(dst, dst, 1); // dst=0 for +-0.0 and NaN. 0x7FF..F otherwise
14327 // Bit selection instruction gets bit from "one" for each enabled bit in
14328 // "dst", otherwise gets a bit from "src". For "src" that contains +-0.0 or
14329 // NaN the whole "src" will be copied because "dst" is zero. For all other
14330 // "src" values dst is 0x7FF..F, which means only the sign bit is copied
14331 // from "src", and all other bits are copied from 1.0.
14332 __ bsl(dst, __ T8B, one, src);
14333 %}
14334 ins_pipe(fp_uop_d);
14335 %}
14336
14337 instruct signumF_reg(vRegF dst, vRegF src, vRegF zero, vRegF one) %{
14338 match(Set dst (SignumF src (Binary zero one)));
14339 effect(TEMP_DEF dst, USE src, USE zero, USE one);
14340 format %{ "signumF $dst, $src" %}
14341 ins_encode %{
14342 FloatRegister src = as_FloatRegister($src$$reg),
14343 dst = as_FloatRegister($dst$$reg),
14344 zero = as_FloatRegister($zero$$reg),
14345 one = as_FloatRegister($one$$reg);
14346 __ facgts(dst, src, zero); // dst=0 for +-0.0 and NaN. 0xFFF..F otherwise
14347 __ ushr(dst, __ T2S, dst, 1); // dst=0 for +-0.0 and NaN. 0x7FF..F otherwise
14348 // Bit selection instruction gets bit from "one" for each enabled bit in
14349 // "dst", otherwise gets a bit from "src". For "src" that contains +-0.0 or
14350 // NaN the whole "src" will be copied because "dst" is zero. For all other
14351 // "src" values dst is 0x7FF..F, which means only the sign bit is copied
14352 // from "src", and all other bits are copied from 1.0.
14353 __ bsl(dst, __ T8B, one, src);
14354 %}
14355 ins_pipe(fp_uop_d);
14356 %}
14357
14358 // ============================================================================
14359 // Logical Instructions
14360
14361 // Integer Logical Instructions
14362
14363 // And Instructions
14364
14365
14366 instruct andI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, rFlagsReg cr) %{
14367 match(Set dst (AndI src1 src2));
14368
14369 format %{ "andw $dst, $src1, $src2\t# int" %}
14370
14371 ins_cost(INSN_COST);
14372 ins_encode %{
14373 __ andw(as_Register($dst$$reg),
14374 as_Register($src1$$reg),
14375 as_Register($src2$$reg));
14376 %}
14377
14378 ins_pipe(ialu_reg_reg);
14379 %}
14380
14381 instruct andI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immILog src2, rFlagsReg cr) %{
14382 match(Set dst (AndI src1 src2));
14383
14384 format %{ "andsw $dst, $src1, $src2\t# int" %}
14385
14386 ins_cost(INSN_COST);
14387 ins_encode %{
14388 __ andw(as_Register($dst$$reg),
14389 as_Register($src1$$reg),
14390 (uint64_t)($src2$$constant));
14391 %}
14392
14393 ins_pipe(ialu_reg_imm);
14394 %}
14395
14396 // Or Instructions
14397
14398 instruct orI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
14399 match(Set dst (OrI src1 src2));
14400
14401 format %{ "orrw $dst, $src1, $src2\t# int" %}
14402
14403 ins_cost(INSN_COST);
14404 ins_encode %{
14405 __ orrw(as_Register($dst$$reg),
14406 as_Register($src1$$reg),
14407 as_Register($src2$$reg));
14408 %}
14409
14410 ins_pipe(ialu_reg_reg);
14411 %}
14412
14413 instruct orI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immILog src2) %{
14414 match(Set dst (OrI src1 src2));
14415
14416 format %{ "orrw $dst, $src1, $src2\t# int" %}
14417
14418 ins_cost(INSN_COST);
14419 ins_encode %{
14420 __ orrw(as_Register($dst$$reg),
14421 as_Register($src1$$reg),
14422 (uint64_t)($src2$$constant));
14423 %}
14424
14425 ins_pipe(ialu_reg_imm);
14426 %}
14427
14428 // Xor Instructions
14429
14430 instruct xorI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
14431 match(Set dst (XorI src1 src2));
14432
14433 format %{ "eorw $dst, $src1, $src2\t# int" %}
14434
14435 ins_cost(INSN_COST);
14436 ins_encode %{
14437 __ eorw(as_Register($dst$$reg),
14438 as_Register($src1$$reg),
14439 as_Register($src2$$reg));
14440 %}
14441
14442 ins_pipe(ialu_reg_reg);
14443 %}
14444
14445 instruct xorI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immILog src2) %{
14446 match(Set dst (XorI src1 src2));
14447
14448 format %{ "eorw $dst, $src1, $src2\t# int" %}
14449
14450 ins_cost(INSN_COST);
14451 ins_encode %{
14452 __ eorw(as_Register($dst$$reg),
14453 as_Register($src1$$reg),
14454 (uint64_t)($src2$$constant));
14455 %}
14456
14457 ins_pipe(ialu_reg_imm);
14458 %}
14459
14460 // Long Logical Instructions
14461 // TODO
14462
14463 instruct andL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2, rFlagsReg cr) %{
14464 match(Set dst (AndL src1 src2));
14465
14466 format %{ "and $dst, $src1, $src2\t# int" %}
14467
14468 ins_cost(INSN_COST);
14469 ins_encode %{
14470 __ andr(as_Register($dst$$reg),
14471 as_Register($src1$$reg),
14472 as_Register($src2$$reg));
14473 %}
14474
14475 ins_pipe(ialu_reg_reg);
14476 %}
14477
14478 instruct andL_reg_imm(iRegLNoSp dst, iRegL src1, immLLog src2, rFlagsReg cr) %{
14479 match(Set dst (AndL src1 src2));
14480
14481 format %{ "and $dst, $src1, $src2\t# int" %}
14482
14483 ins_cost(INSN_COST);
14484 ins_encode %{
14485 __ andr(as_Register($dst$$reg),
14486 as_Register($src1$$reg),
14487 (uint64_t)($src2$$constant));
14488 %}
14489
14490 ins_pipe(ialu_reg_imm);
14491 %}
14492
14493 // Or Instructions
14494
14495 instruct orL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
14496 match(Set dst (OrL src1 src2));
14497
14498 format %{ "orr $dst, $src1, $src2\t# int" %}
14499
14500 ins_cost(INSN_COST);
14501 ins_encode %{
14502 __ orr(as_Register($dst$$reg),
14503 as_Register($src1$$reg),
14504 as_Register($src2$$reg));
14505 %}
14506
14507 ins_pipe(ialu_reg_reg);
14508 %}
14509
14510 instruct orL_reg_imm(iRegLNoSp dst, iRegL src1, immLLog src2) %{
14511 match(Set dst (OrL src1 src2));
14512
14513 format %{ "orr $dst, $src1, $src2\t# int" %}
14514
14515 ins_cost(INSN_COST);
14516 ins_encode %{
14517 __ orr(as_Register($dst$$reg),
14518 as_Register($src1$$reg),
14519 (uint64_t)($src2$$constant));
14520 %}
14521
14522 ins_pipe(ialu_reg_imm);
14523 %}
14524
14525 // Xor Instructions
14526
14527 instruct xorL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
14528 match(Set dst (XorL src1 src2));
14529
14530 format %{ "eor $dst, $src1, $src2\t# int" %}
14531
14532 ins_cost(INSN_COST);
14533 ins_encode %{
14534 __ eor(as_Register($dst$$reg),
14535 as_Register($src1$$reg),
14536 as_Register($src2$$reg));
14537 %}
14538
14539 ins_pipe(ialu_reg_reg);
14540 %}
14541
14542 instruct xorL_reg_imm(iRegLNoSp dst, iRegL src1, immLLog src2) %{
14543 match(Set dst (XorL src1 src2));
14544
14545 ins_cost(INSN_COST);
14546 format %{ "eor $dst, $src1, $src2\t# int" %}
14547
14548 ins_encode %{
14549 __ eor(as_Register($dst$$reg),
14550 as_Register($src1$$reg),
14551 (uint64_t)($src2$$constant));
14552 %}
14553
14554 ins_pipe(ialu_reg_imm);
14555 %}
14556
14557 instruct convI2L_reg_reg(iRegLNoSp dst, iRegIorL2I src)
14558 %{
14559 match(Set dst (ConvI2L src));
14560
14561 ins_cost(INSN_COST);
14562 format %{ "sxtw $dst, $src\t# i2l" %}
14563 ins_encode %{
14564 __ sbfm($dst$$Register, $src$$Register, 0, 31);
14565 %}
14566 ins_pipe(ialu_reg_shift);
14567 %}
14568
14569 // this pattern occurs in bigmath arithmetic
14570 instruct convUI2L_reg_reg(iRegLNoSp dst, iRegIorL2I src, immL_32bits mask)
14571 %{
14572 match(Set dst (AndL (ConvI2L src) mask));
14573
14574 ins_cost(INSN_COST);
14575 format %{ "ubfm $dst, $src, 0, 31\t# ui2l" %}
14576 ins_encode %{
14577 __ ubfm($dst$$Register, $src$$Register, 0, 31);
14578 %}
14579
14580 ins_pipe(ialu_reg_shift);
14581 %}
14582
14583 instruct convL2I_reg(iRegINoSp dst, iRegL src) %{
14584 match(Set dst (ConvL2I src));
14585
14586 ins_cost(INSN_COST);
14587 format %{ "movw $dst, $src \t// l2i" %}
14588
14589 ins_encode %{
14590 __ movw(as_Register($dst$$reg), as_Register($src$$reg));
14591 %}
14592
14593 ins_pipe(ialu_reg);
14594 %}
14595
14596 instruct convI2B(iRegINoSp dst, iRegIorL2I src, rFlagsReg cr)
14597 %{
14598 match(Set dst (Conv2B src));
14599 effect(KILL cr);
14600
14601 format %{
14602 "cmpw $src, zr\n\t"
14603 "cset $dst, ne"
14604 %}
14605
14606 ins_encode %{
14607 __ cmpw(as_Register($src$$reg), zr);
14608 __ cset(as_Register($dst$$reg), Assembler::NE);
14609 %}
14610
14611 ins_pipe(ialu_reg);
14612 %}
14613
14614 instruct convP2B(iRegINoSp dst, iRegP src, rFlagsReg cr)
14615 %{
14616 match(Set dst (Conv2B src));
14617 effect(KILL cr);
14618
14619 format %{
14620 "cmp $src, zr\n\t"
14621 "cset $dst, ne"
14622 %}
14623
14624 ins_encode %{
14625 __ cmp(as_Register($src$$reg), zr);
14626 __ cset(as_Register($dst$$reg), Assembler::NE);
14627 %}
14628
14629 ins_pipe(ialu_reg);
14630 %}
14631
14632 instruct convD2F_reg(vRegF dst, vRegD src) %{
14633 match(Set dst (ConvD2F src));
14634
14635 ins_cost(INSN_COST * 5);
14636 format %{ "fcvtd $dst, $src \t// d2f" %}
14637
14638 ins_encode %{
14639 __ fcvtd(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg));
14640 %}
14641
14642 ins_pipe(fp_d2f);
14643 %}
14644
14645 instruct convF2D_reg(vRegD dst, vRegF src) %{
14646 match(Set dst (ConvF2D src));
14647
14648 ins_cost(INSN_COST * 5);
14649 format %{ "fcvts $dst, $src \t// f2d" %}
14650
14651 ins_encode %{
14652 __ fcvts(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg));
14653 %}
14654
14655 ins_pipe(fp_f2d);
14656 %}
14657
14658 instruct convF2I_reg_reg(iRegINoSp dst, vRegF src) %{
14659 match(Set dst (ConvF2I src));
14660
14661 ins_cost(INSN_COST * 5);
14662 format %{ "fcvtzsw $dst, $src \t// f2i" %}
14663
14664 ins_encode %{
14665 __ fcvtzsw(as_Register($dst$$reg), as_FloatRegister($src$$reg));
14666 %}
14667
14668 ins_pipe(fp_f2i);
14669 %}
14670
14671 instruct convF2L_reg_reg(iRegLNoSp dst, vRegF src) %{
14672 match(Set dst (ConvF2L src));
14673
14674 ins_cost(INSN_COST * 5);
14675 format %{ "fcvtzs $dst, $src \t// f2l" %}
14676
14677 ins_encode %{
14678 __ fcvtzs(as_Register($dst$$reg), as_FloatRegister($src$$reg));
14679 %}
14680
14681 ins_pipe(fp_f2l);
14682 %}
14683
14684 instruct convI2F_reg_reg(vRegF dst, iRegIorL2I src) %{
14685 match(Set dst (ConvI2F src));
14686
14687 ins_cost(INSN_COST * 5);
14688 format %{ "scvtfws $dst, $src \t// i2f" %}
14689
14690 ins_encode %{
14691 __ scvtfws(as_FloatRegister($dst$$reg), as_Register($src$$reg));
14692 %}
14693
14694 ins_pipe(fp_i2f);
14695 %}
14696
14697 instruct convL2F_reg_reg(vRegF dst, iRegL src) %{
14698 match(Set dst (ConvL2F src));
14699
14700 ins_cost(INSN_COST * 5);
14701 format %{ "scvtfs $dst, $src \t// l2f" %}
14702
14703 ins_encode %{
14704 __ scvtfs(as_FloatRegister($dst$$reg), as_Register($src$$reg));
14705 %}
14706
14707 ins_pipe(fp_l2f);
14708 %}
14709
14710 instruct convD2I_reg_reg(iRegINoSp dst, vRegD src) %{
14711 match(Set dst (ConvD2I src));
14712
14713 ins_cost(INSN_COST * 5);
14714 format %{ "fcvtzdw $dst, $src \t// d2i" %}
14715
14716 ins_encode %{
14717 __ fcvtzdw(as_Register($dst$$reg), as_FloatRegister($src$$reg));
14718 %}
14719
14720 ins_pipe(fp_d2i);
14721 %}
14722
14723 instruct convD2L_reg_reg(iRegLNoSp dst, vRegD src) %{
14724 match(Set dst (ConvD2L src));
14725
14726 ins_cost(INSN_COST * 5);
14727 format %{ "fcvtzd $dst, $src \t// d2l" %}
14728
14729 ins_encode %{
14730 __ fcvtzd(as_Register($dst$$reg), as_FloatRegister($src$$reg));
14731 %}
14732
14733 ins_pipe(fp_d2l);
14734 %}
14735
14736 instruct convI2D_reg_reg(vRegD dst, iRegIorL2I src) %{
14737 match(Set dst (ConvI2D src));
14738
14739 ins_cost(INSN_COST * 5);
14740 format %{ "scvtfwd $dst, $src \t// i2d" %}
14741
14742 ins_encode %{
14743 __ scvtfwd(as_FloatRegister($dst$$reg), as_Register($src$$reg));
14744 %}
14745
14746 ins_pipe(fp_i2d);
14747 %}
14748
14749 instruct convL2D_reg_reg(vRegD dst, iRegL src) %{
14750 match(Set dst (ConvL2D src));
14751
14752 ins_cost(INSN_COST * 5);
14753 format %{ "scvtfd $dst, $src \t// l2d" %}
14754
14755 ins_encode %{
14756 __ scvtfd(as_FloatRegister($dst$$reg), as_Register($src$$reg));
14757 %}
14758
14759 ins_pipe(fp_l2d);
14760 %}
14761
14762 // stack <-> reg and reg <-> reg shuffles with no conversion
14763
14764 instruct MoveF2I_stack_reg(iRegINoSp dst, stackSlotF src) %{
14765
14766 match(Set dst (MoveF2I src));
14767
14768 effect(DEF dst, USE src);
14769
14770 ins_cost(4 * INSN_COST);
14771
14772 format %{ "ldrw $dst, $src\t# MoveF2I_stack_reg" %}
14773
14774 ins_encode %{
14775 __ ldrw($dst$$Register, Address(sp, $src$$disp));
14776 %}
14777
14778 ins_pipe(iload_reg_reg);
14779
14780 %}
14781
14782 instruct MoveI2F_stack_reg(vRegF dst, stackSlotI src) %{
14783
14784 match(Set dst (MoveI2F src));
14785
14786 effect(DEF dst, USE src);
14787
14788 ins_cost(4 * INSN_COST);
14789
14790 format %{ "ldrs $dst, $src\t# MoveI2F_stack_reg" %}
14791
14792 ins_encode %{
14793 __ ldrs(as_FloatRegister($dst$$reg), Address(sp, $src$$disp));
14794 %}
14795
14796 ins_pipe(pipe_class_memory);
14797
14798 %}
14799
14800 instruct MoveD2L_stack_reg(iRegLNoSp dst, stackSlotD src) %{
14801
14802 match(Set dst (MoveD2L src));
14803
14804 effect(DEF dst, USE src);
14805
14806 ins_cost(4 * INSN_COST);
14807
14808 format %{ "ldr $dst, $src\t# MoveD2L_stack_reg" %}
14809
14810 ins_encode %{
14811 __ ldr($dst$$Register, Address(sp, $src$$disp));
14812 %}
14813
14814 ins_pipe(iload_reg_reg);
14815
14816 %}
14817
14818 instruct MoveL2D_stack_reg(vRegD dst, stackSlotL src) %{
14819
14820 match(Set dst (MoveL2D src));
14821
14822 effect(DEF dst, USE src);
14823
14824 ins_cost(4 * INSN_COST);
14825
14826 format %{ "ldrd $dst, $src\t# MoveL2D_stack_reg" %}
14827
14828 ins_encode %{
14829 __ ldrd(as_FloatRegister($dst$$reg), Address(sp, $src$$disp));
14830 %}
14831
14832 ins_pipe(pipe_class_memory);
14833
14834 %}
14835
14836 instruct MoveF2I_reg_stack(stackSlotI dst, vRegF src) %{
14837
14838 match(Set dst (MoveF2I src));
14839
14840 effect(DEF dst, USE src);
14841
14842 ins_cost(INSN_COST);
14843
14844 format %{ "strs $src, $dst\t# MoveF2I_reg_stack" %}
14845
14846 ins_encode %{
14847 __ strs(as_FloatRegister($src$$reg), Address(sp, $dst$$disp));
14848 %}
14849
14850 ins_pipe(pipe_class_memory);
14851
14852 %}
14853
14854 instruct MoveI2F_reg_stack(stackSlotF dst, iRegI src) %{
14855
14856 match(Set dst (MoveI2F src));
14857
14858 effect(DEF dst, USE src);
14859
14860 ins_cost(INSN_COST);
14861
14862 format %{ "strw $src, $dst\t# MoveI2F_reg_stack" %}
14863
14864 ins_encode %{
14865 __ strw($src$$Register, Address(sp, $dst$$disp));
14866 %}
14867
14868 ins_pipe(istore_reg_reg);
14869
14870 %}
14871
14872 instruct MoveD2L_reg_stack(stackSlotL dst, vRegD src) %{
14873
14874 match(Set dst (MoveD2L src));
14875
14876 effect(DEF dst, USE src);
14877
14878 ins_cost(INSN_COST);
14879
14880 format %{ "strd $dst, $src\t# MoveD2L_reg_stack" %}
14881
14882 ins_encode %{
14883 __ strd(as_FloatRegister($src$$reg), Address(sp, $dst$$disp));
14884 %}
14885
14886 ins_pipe(pipe_class_memory);
14887
14888 %}
14889
14890 instruct MoveL2D_reg_stack(stackSlotD dst, iRegL src) %{
14891
14892 match(Set dst (MoveL2D src));
14893
14894 effect(DEF dst, USE src);
14895
14896 ins_cost(INSN_COST);
14897
14898 format %{ "str $src, $dst\t# MoveL2D_reg_stack" %}
14899
14900 ins_encode %{
14901 __ str($src$$Register, Address(sp, $dst$$disp));
14902 %}
14903
14904 ins_pipe(istore_reg_reg);
14905
14906 %}
14907
14908 instruct MoveF2I_reg_reg(iRegINoSp dst, vRegF src) %{
14909
14910 match(Set dst (MoveF2I src));
14911
14912 effect(DEF dst, USE src);
14913
14914 ins_cost(INSN_COST);
14915
14916 format %{ "fmovs $dst, $src\t# MoveF2I_reg_reg" %}
14917
14918 ins_encode %{
14919 __ fmovs($dst$$Register, as_FloatRegister($src$$reg));
14920 %}
14921
14922 ins_pipe(fp_f2i);
14923
14924 %}
14925
14926 instruct MoveI2F_reg_reg(vRegF dst, iRegI src) %{
14927
14928 match(Set dst (MoveI2F src));
14929
14930 effect(DEF dst, USE src);
14931
14932 ins_cost(INSN_COST);
14933
14934 format %{ "fmovs $dst, $src\t# MoveI2F_reg_reg" %}
14935
14936 ins_encode %{
14937 __ fmovs(as_FloatRegister($dst$$reg), $src$$Register);
14938 %}
14939
14940 ins_pipe(fp_i2f);
14941
14942 %}
14943
14944 instruct MoveD2L_reg_reg(iRegLNoSp dst, vRegD src) %{
14945
14946 match(Set dst (MoveD2L src));
14947
14948 effect(DEF dst, USE src);
14949
14950 ins_cost(INSN_COST);
14951
14952 format %{ "fmovd $dst, $src\t# MoveD2L_reg_reg" %}
14953
14954 ins_encode %{
14955 __ fmovd($dst$$Register, as_FloatRegister($src$$reg));
14956 %}
14957
14958 ins_pipe(fp_d2l);
14959
14960 %}
14961
14962 instruct MoveL2D_reg_reg(vRegD dst, iRegL src) %{
14963
14964 match(Set dst (MoveL2D src));
14965
14966 effect(DEF dst, USE src);
14967
14968 ins_cost(INSN_COST);
14969
14970 format %{ "fmovd $dst, $src\t# MoveL2D_reg_reg" %}
14971
14972 ins_encode %{
14973 __ fmovd(as_FloatRegister($dst$$reg), $src$$Register);
14974 %}
14975
14976 ins_pipe(fp_l2d);
14977
14978 %}
14979
14980 // ============================================================================
14981 // clearing of an array
14982
14983 instruct clearArray_reg_reg(iRegL_R11 cnt, iRegP_R10 base, Universe dummy, rFlagsReg cr)
14984 %{
14985 match(Set dummy (ClearArray cnt base));
14986 effect(USE_KILL cnt, USE_KILL base, KILL cr);
14987
14988 ins_cost(4 * INSN_COST);
14989 format %{ "ClearArray $cnt, $base" %}
14990
14991 ins_encode %{
14992 address tpc = __ zero_words($base$$Register, $cnt$$Register);
14993 if (tpc == NULL) {
14994 ciEnv::current()->record_failure("CodeCache is full");
14995 return;
14996 }
14997 %}
14998
14999 ins_pipe(pipe_class_memory);
15000 %}
15001
15002 instruct clearArray_imm_reg(immL cnt, iRegP_R10 base, iRegL_R11 temp, Universe dummy, rFlagsReg cr)
15003 %{
15004 predicate((uint64_t)n->in(2)->get_long()
15005 < (uint64_t)(BlockZeroingLowLimit >> LogBytesPerWord));
15006 match(Set dummy (ClearArray cnt base));
15007 effect(TEMP temp, USE_KILL base, KILL cr);
15008
15009 ins_cost(4 * INSN_COST);
15010 format %{ "ClearArray $cnt, $base" %}
15011
15012 ins_encode %{
15013 __ zero_words($base$$Register, (uint64_t)$cnt$$constant);
15014 %}
15015
15016 ins_pipe(pipe_class_memory);
15017 %}
15018
15019 // ============================================================================
15020 // Overflow Math Instructions
15021
15022 instruct overflowAddI_reg_reg(rFlagsReg cr, iRegIorL2I op1, iRegIorL2I op2)
15023 %{
15024 match(Set cr (OverflowAddI op1 op2));
15025
15026 format %{ "cmnw $op1, $op2\t# overflow check int" %}
15027 ins_cost(INSN_COST);
15028 ins_encode %{
15029 __ cmnw($op1$$Register, $op2$$Register);
15030 %}
15031
15032 ins_pipe(icmp_reg_reg);
15033 %}
15034
15035 instruct overflowAddI_reg_imm(rFlagsReg cr, iRegIorL2I op1, immIAddSub op2)
15036 %{
15037 match(Set cr (OverflowAddI op1 op2));
15038
15039 format %{ "cmnw $op1, $op2\t# overflow check int" %}
15040 ins_cost(INSN_COST);
15041 ins_encode %{
15042 __ cmnw($op1$$Register, $op2$$constant);
15043 %}
15044
15045 ins_pipe(icmp_reg_imm);
15046 %}
15047
15048 instruct overflowAddL_reg_reg(rFlagsReg cr, iRegL op1, iRegL op2)
15049 %{
15050 match(Set cr (OverflowAddL op1 op2));
15051
15052 format %{ "cmn $op1, $op2\t# overflow check long" %}
15053 ins_cost(INSN_COST);
15054 ins_encode %{
15055 __ cmn($op1$$Register, $op2$$Register);
15056 %}
15057
15058 ins_pipe(icmp_reg_reg);
15059 %}
15060
15061 instruct overflowAddL_reg_imm(rFlagsReg cr, iRegL op1, immLAddSub op2)
15062 %{
15063 match(Set cr (OverflowAddL op1 op2));
15064
15065 format %{ "cmn $op1, $op2\t# overflow check long" %}
15066 ins_cost(INSN_COST);
15067 ins_encode %{
15068 __ cmn($op1$$Register, $op2$$constant);
15069 %}
15070
15071 ins_pipe(icmp_reg_imm);
15072 %}
15073
15074 instruct overflowSubI_reg_reg(rFlagsReg cr, iRegIorL2I op1, iRegIorL2I op2)
15075 %{
15076 match(Set cr (OverflowSubI op1 op2));
15077
15078 format %{ "cmpw $op1, $op2\t# overflow check int" %}
15079 ins_cost(INSN_COST);
15080 ins_encode %{
15081 __ cmpw($op1$$Register, $op2$$Register);
15082 %}
15083
15084 ins_pipe(icmp_reg_reg);
15085 %}
15086
15087 instruct overflowSubI_reg_imm(rFlagsReg cr, iRegIorL2I op1, immIAddSub op2)
15088 %{
15089 match(Set cr (OverflowSubI op1 op2));
15090
15091 format %{ "cmpw $op1, $op2\t# overflow check int" %}
15092 ins_cost(INSN_COST);
15093 ins_encode %{
15094 __ cmpw($op1$$Register, $op2$$constant);
15095 %}
15096
15097 ins_pipe(icmp_reg_imm);
15098 %}
15099
15100 instruct overflowSubL_reg_reg(rFlagsReg cr, iRegL op1, iRegL op2)
15101 %{
15102 match(Set cr (OverflowSubL op1 op2));
15103
15104 format %{ "cmp $op1, $op2\t# overflow check long" %}
15105 ins_cost(INSN_COST);
15106 ins_encode %{
15107 __ cmp($op1$$Register, $op2$$Register);
15108 %}
15109
15110 ins_pipe(icmp_reg_reg);
15111 %}
15112
15113 instruct overflowSubL_reg_imm(rFlagsReg cr, iRegL op1, immLAddSub op2)
15114 %{
15115 match(Set cr (OverflowSubL op1 op2));
15116
15117 format %{ "cmp $op1, $op2\t# overflow check long" %}
15118 ins_cost(INSN_COST);
15119 ins_encode %{
15120 __ subs(zr, $op1$$Register, $op2$$constant);
15121 %}
15122
15123 ins_pipe(icmp_reg_imm);
15124 %}
15125
15126 instruct overflowNegI_reg(rFlagsReg cr, immI0 zero, iRegIorL2I op1)
15127 %{
15128 match(Set cr (OverflowSubI zero op1));
15129
15130 format %{ "cmpw zr, $op1\t# overflow check int" %}
15131 ins_cost(INSN_COST);
15132 ins_encode %{
15133 __ cmpw(zr, $op1$$Register);
15134 %}
15135
15136 ins_pipe(icmp_reg_imm);
15137 %}
15138
15139 instruct overflowNegL_reg(rFlagsReg cr, immI0 zero, iRegL op1)
15140 %{
15141 match(Set cr (OverflowSubL zero op1));
15142
15143 format %{ "cmp zr, $op1\t# overflow check long" %}
15144 ins_cost(INSN_COST);
15145 ins_encode %{
15146 __ cmp(zr, $op1$$Register);
15147 %}
15148
15149 ins_pipe(icmp_reg_imm);
15150 %}
15151
15152 instruct overflowMulI_reg(rFlagsReg cr, iRegIorL2I op1, iRegIorL2I op2)
15153 %{
15154 match(Set cr (OverflowMulI op1 op2));
15155
15156 format %{ "smull rscratch1, $op1, $op2\t# overflow check int\n\t"
15157 "cmp rscratch1, rscratch1, sxtw\n\t"
15158 "movw rscratch1, #0x80000000\n\t"
15159 "cselw rscratch1, rscratch1, zr, NE\n\t"
15160 "cmpw rscratch1, #1" %}
15161 ins_cost(5 * INSN_COST);
15162 ins_encode %{
15163 __ smull(rscratch1, $op1$$Register, $op2$$Register);
15164 __ subs(zr, rscratch1, rscratch1, ext::sxtw); // NE => overflow
15165 __ movw(rscratch1, 0x80000000); // Develop 0 (EQ),
15166 __ cselw(rscratch1, rscratch1, zr, Assembler::NE); // or 0x80000000 (NE)
15167 __ cmpw(rscratch1, 1); // 0x80000000 - 1 => VS
15168 %}
15169
15170 ins_pipe(pipe_slow);
15171 %}
15172
15173 instruct overflowMulI_reg_branch(cmpOp cmp, iRegIorL2I op1, iRegIorL2I op2, label labl, rFlagsReg cr)
15174 %{
15175 match(If cmp (OverflowMulI op1 op2));
15176 predicate(n->in(1)->as_Bool()->_test._test == BoolTest::overflow
15177 || n->in(1)->as_Bool()->_test._test == BoolTest::no_overflow);
15178 effect(USE labl, KILL cr);
15179
15180 format %{ "smull rscratch1, $op1, $op2\t# overflow check int\n\t"
15181 "cmp rscratch1, rscratch1, sxtw\n\t"
15182 "b$cmp $labl" %}
15183 ins_cost(3 * INSN_COST); // Branch is rare so treat as INSN_COST
15184 ins_encode %{
15185 Label* L = $labl$$label;
15186 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15187 __ smull(rscratch1, $op1$$Register, $op2$$Register);
15188 __ subs(zr, rscratch1, rscratch1, ext::sxtw); // NE => overflow
15189 __ br(cond == Assembler::VS ? Assembler::NE : Assembler::EQ, *L);
15190 %}
15191
15192 ins_pipe(pipe_serial);
15193 %}
15194
15195 instruct overflowMulL_reg(rFlagsReg cr, iRegL op1, iRegL op2)
15196 %{
15197 match(Set cr (OverflowMulL op1 op2));
15198
15199 format %{ "mul rscratch1, $op1, $op2\t#overflow check long\n\t"
15200 "smulh rscratch2, $op1, $op2\n\t"
15201 "cmp rscratch2, rscratch1, ASR #63\n\t"
15202 "movw rscratch1, #0x80000000\n\t"
15203 "cselw rscratch1, rscratch1, zr, NE\n\t"
15204 "cmpw rscratch1, #1" %}
15205 ins_cost(6 * INSN_COST);
15206 ins_encode %{
15207 __ mul(rscratch1, $op1$$Register, $op2$$Register); // Result bits 0..63
15208 __ smulh(rscratch2, $op1$$Register, $op2$$Register); // Result bits 64..127
15209 __ cmp(rscratch2, rscratch1, Assembler::ASR, 63); // Top is pure sign ext
15210 __ movw(rscratch1, 0x80000000); // Develop 0 (EQ),
15211 __ cselw(rscratch1, rscratch1, zr, Assembler::NE); // or 0x80000000 (NE)
15212 __ cmpw(rscratch1, 1); // 0x80000000 - 1 => VS
15213 %}
15214
15215 ins_pipe(pipe_slow);
15216 %}
15217
15218 instruct overflowMulL_reg_branch(cmpOp cmp, iRegL op1, iRegL op2, label labl, rFlagsReg cr)
15219 %{
15220 match(If cmp (OverflowMulL op1 op2));
15221 predicate(n->in(1)->as_Bool()->_test._test == BoolTest::overflow
15222 || n->in(1)->as_Bool()->_test._test == BoolTest::no_overflow);
15223 effect(USE labl, KILL cr);
15224
15225 format %{ "mul rscratch1, $op1, $op2\t#overflow check long\n\t"
15226 "smulh rscratch2, $op1, $op2\n\t"
15227 "cmp rscratch2, rscratch1, ASR #63\n\t"
15228 "b$cmp $labl" %}
15229 ins_cost(4 * INSN_COST); // Branch is rare so treat as INSN_COST
15230 ins_encode %{
15231 Label* L = $labl$$label;
15232 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15233 __ mul(rscratch1, $op1$$Register, $op2$$Register); // Result bits 0..63
15234 __ smulh(rscratch2, $op1$$Register, $op2$$Register); // Result bits 64..127
15235 __ cmp(rscratch2, rscratch1, Assembler::ASR, 63); // Top is pure sign ext
15236 __ br(cond == Assembler::VS ? Assembler::NE : Assembler::EQ, *L);
15237 %}
15238
15239 ins_pipe(pipe_serial);
15240 %}
15241
15242 // ============================================================================
15243 // Compare Instructions
15244
15245 instruct compI_reg_reg(rFlagsReg cr, iRegI op1, iRegI op2)
15246 %{
15247 match(Set cr (CmpI op1 op2));
15248
15249 effect(DEF cr, USE op1, USE op2);
15250
15251 ins_cost(INSN_COST);
15252 format %{ "cmpw $op1, $op2" %}
15253
15254 ins_encode(aarch64_enc_cmpw(op1, op2));
15255
15256 ins_pipe(icmp_reg_reg);
15257 %}
15258
15259 instruct compI_reg_immI0(rFlagsReg cr, iRegI op1, immI0 zero)
15260 %{
15261 match(Set cr (CmpI op1 zero));
15262
15263 effect(DEF cr, USE op1);
15264
15265 ins_cost(INSN_COST);
15266 format %{ "cmpw $op1, 0" %}
15267
15268 ins_encode(aarch64_enc_cmpw_imm_addsub(op1, zero));
15269
15270 ins_pipe(icmp_reg_imm);
15271 %}
15272
15273 instruct compI_reg_immIAddSub(rFlagsReg cr, iRegI op1, immIAddSub op2)
15274 %{
15275 match(Set cr (CmpI op1 op2));
15276
15277 effect(DEF cr, USE op1);
15278
15279 ins_cost(INSN_COST);
15280 format %{ "cmpw $op1, $op2" %}
15281
15282 ins_encode(aarch64_enc_cmpw_imm_addsub(op1, op2));
15283
15284 ins_pipe(icmp_reg_imm);
15285 %}
15286
15287 instruct compI_reg_immI(rFlagsReg cr, iRegI op1, immI op2)
15288 %{
15289 match(Set cr (CmpI op1 op2));
15290
15291 effect(DEF cr, USE op1);
15292
15293 ins_cost(INSN_COST * 2);
15294 format %{ "cmpw $op1, $op2" %}
15295
15296 ins_encode(aarch64_enc_cmpw_imm(op1, op2));
15297
15298 ins_pipe(icmp_reg_imm);
15299 %}
15300
15301 // Unsigned compare Instructions; really, same as signed compare
15302 // except it should only be used to feed an If or a CMovI which takes a
15303 // cmpOpU.
15304
15305 instruct compU_reg_reg(rFlagsRegU cr, iRegI op1, iRegI op2)
15306 %{
15307 match(Set cr (CmpU op1 op2));
15308
15309 effect(DEF cr, USE op1, USE op2);
15310
15311 ins_cost(INSN_COST);
15312 format %{ "cmpw $op1, $op2\t# unsigned" %}
15313
15314 ins_encode(aarch64_enc_cmpw(op1, op2));
15315
15316 ins_pipe(icmp_reg_reg);
15317 %}
15318
15319 instruct compU_reg_immI0(rFlagsRegU cr, iRegI op1, immI0 zero)
15320 %{
15321 match(Set cr (CmpU op1 zero));
15322
15323 effect(DEF cr, USE op1);
15324
15325 ins_cost(INSN_COST);
15326 format %{ "cmpw $op1, #0\t# unsigned" %}
15327
15328 ins_encode(aarch64_enc_cmpw_imm_addsub(op1, zero));
15329
15330 ins_pipe(icmp_reg_imm);
15331 %}
15332
15333 instruct compU_reg_immIAddSub(rFlagsRegU cr, iRegI op1, immIAddSub op2)
15334 %{
15335 match(Set cr (CmpU op1 op2));
15336
15337 effect(DEF cr, USE op1);
15338
15339 ins_cost(INSN_COST);
15340 format %{ "cmpw $op1, $op2\t# unsigned" %}
15341
15342 ins_encode(aarch64_enc_cmpw_imm_addsub(op1, op2));
15343
15344 ins_pipe(icmp_reg_imm);
15345 %}
15346
15347 instruct compU_reg_immI(rFlagsRegU cr, iRegI op1, immI op2)
15348 %{
15349 match(Set cr (CmpU op1 op2));
15350
15351 effect(DEF cr, USE op1);
15352
15353 ins_cost(INSN_COST * 2);
15354 format %{ "cmpw $op1, $op2\t# unsigned" %}
15355
15356 ins_encode(aarch64_enc_cmpw_imm(op1, op2));
15357
15358 ins_pipe(icmp_reg_imm);
15359 %}
15360
15361 instruct compL_reg_reg(rFlagsReg cr, iRegL op1, iRegL op2)
15362 %{
15363 match(Set cr (CmpL op1 op2));
15364
15365 effect(DEF cr, USE op1, USE op2);
15366
15367 ins_cost(INSN_COST);
15368 format %{ "cmp $op1, $op2" %}
15369
15370 ins_encode(aarch64_enc_cmp(op1, op2));
15371
15372 ins_pipe(icmp_reg_reg);
15373 %}
15374
15375 instruct compL_reg_immL0(rFlagsReg cr, iRegL op1, immL0 zero)
15376 %{
15377 match(Set cr (CmpL op1 zero));
15378
15379 effect(DEF cr, USE op1);
15380
15381 ins_cost(INSN_COST);
15382 format %{ "tst $op1" %}
15383
15384 ins_encode(aarch64_enc_cmp_imm_addsub(op1, zero));
15385
15386 ins_pipe(icmp_reg_imm);
15387 %}
15388
15389 instruct compL_reg_immLAddSub(rFlagsReg cr, iRegL op1, immLAddSub op2)
15390 %{
15391 match(Set cr (CmpL op1 op2));
15392
15393 effect(DEF cr, USE op1);
15394
15395 ins_cost(INSN_COST);
15396 format %{ "cmp $op1, $op2" %}
15397
15398 ins_encode(aarch64_enc_cmp_imm_addsub(op1, op2));
15399
15400 ins_pipe(icmp_reg_imm);
15401 %}
15402
15403 instruct compL_reg_immL(rFlagsReg cr, iRegL op1, immL op2)
15404 %{
15405 match(Set cr (CmpL op1 op2));
15406
15407 effect(DEF cr, USE op1);
15408
15409 ins_cost(INSN_COST * 2);
15410 format %{ "cmp $op1, $op2" %}
15411
15412 ins_encode(aarch64_enc_cmp_imm(op1, op2));
15413
15414 ins_pipe(icmp_reg_imm);
15415 %}
15416
15417 instruct compUL_reg_reg(rFlagsRegU cr, iRegL op1, iRegL op2)
15418 %{
15419 match(Set cr (CmpUL op1 op2));
15420
15421 effect(DEF cr, USE op1, USE op2);
15422
15423 ins_cost(INSN_COST);
15424 format %{ "cmp $op1, $op2" %}
15425
15426 ins_encode(aarch64_enc_cmp(op1, op2));
15427
15428 ins_pipe(icmp_reg_reg);
15429 %}
15430
15431 instruct compUL_reg_immL0(rFlagsRegU cr, iRegL op1, immL0 zero)
15432 %{
15433 match(Set cr (CmpUL op1 zero));
15434
15435 effect(DEF cr, USE op1);
15436
15437 ins_cost(INSN_COST);
15438 format %{ "tst $op1" %}
15439
15440 ins_encode(aarch64_enc_cmp_imm_addsub(op1, zero));
15441
15442 ins_pipe(icmp_reg_imm);
15443 %}
15444
15445 instruct compUL_reg_immLAddSub(rFlagsRegU cr, iRegL op1, immLAddSub op2)
15446 %{
15447 match(Set cr (CmpUL op1 op2));
15448
15449 effect(DEF cr, USE op1);
15450
15451 ins_cost(INSN_COST);
15452 format %{ "cmp $op1, $op2" %}
15453
15454 ins_encode(aarch64_enc_cmp_imm_addsub(op1, op2));
15455
15456 ins_pipe(icmp_reg_imm);
15457 %}
15458
15459 instruct compUL_reg_immL(rFlagsRegU cr, iRegL op1, immL op2)
15460 %{
15461 match(Set cr (CmpUL op1 op2));
15462
15463 effect(DEF cr, USE op1);
15464
15465 ins_cost(INSN_COST * 2);
15466 format %{ "cmp $op1, $op2" %}
15467
15468 ins_encode(aarch64_enc_cmp_imm(op1, op2));
15469
15470 ins_pipe(icmp_reg_imm);
15471 %}
15472
15473 instruct compP_reg_reg(rFlagsRegU cr, iRegP op1, iRegP op2)
15474 %{
15475 match(Set cr (CmpP op1 op2));
15476
15477 effect(DEF cr, USE op1, USE op2);
15478
15479 ins_cost(INSN_COST);
15480 format %{ "cmp $op1, $op2\t // ptr" %}
15481
15482 ins_encode(aarch64_enc_cmpp(op1, op2));
15483
15484 ins_pipe(icmp_reg_reg);
15485 %}
15486
15487 instruct compN_reg_reg(rFlagsRegU cr, iRegN op1, iRegN op2)
15488 %{
15489 match(Set cr (CmpN op1 op2));
15490
15491 effect(DEF cr, USE op1, USE op2);
15492
15493 ins_cost(INSN_COST);
15494 format %{ "cmp $op1, $op2\t // compressed ptr" %}
15495
15496 ins_encode(aarch64_enc_cmpn(op1, op2));
15497
15498 ins_pipe(icmp_reg_reg);
15499 %}
15500
15501 instruct testP_reg(rFlagsRegU cr, iRegP op1, immP0 zero)
15502 %{
15503 match(Set cr (CmpP op1 zero));
15504
15505 effect(DEF cr, USE op1, USE zero);
15506
15507 ins_cost(INSN_COST);
15508 format %{ "cmp $op1, 0\t // ptr" %}
15509
15510 ins_encode(aarch64_enc_testp(op1));
15511
15512 ins_pipe(icmp_reg_imm);
15513 %}
15514
15515 instruct testN_reg(rFlagsRegU cr, iRegN op1, immN0 zero)
15516 %{
15517 match(Set cr (CmpN op1 zero));
15518
15519 effect(DEF cr, USE op1, USE zero);
15520
15521 ins_cost(INSN_COST);
15522 format %{ "cmp $op1, 0\t // compressed ptr" %}
15523
15524 ins_encode(aarch64_enc_testn(op1));
15525
15526 ins_pipe(icmp_reg_imm);
15527 %}
15528
15529 // FP comparisons
15530 //
15531 // n.b. CmpF/CmpD set a normal flags reg which then gets compared
15532 // using normal cmpOp. See declaration of rFlagsReg for details.
15533
15534 instruct compF_reg_reg(rFlagsReg cr, vRegF src1, vRegF src2)
15535 %{
15536 match(Set cr (CmpF src1 src2));
15537
15538 ins_cost(3 * INSN_COST);
15539 format %{ "fcmps $src1, $src2" %}
15540
15541 ins_encode %{
15542 __ fcmps(as_FloatRegister($src1$$reg), as_FloatRegister($src2$$reg));
15543 %}
15544
15545 ins_pipe(pipe_class_compare);
15546 %}
15547
15548 instruct compF_reg_zero(rFlagsReg cr, vRegF src1, immF0 src2)
15549 %{
15550 match(Set cr (CmpF src1 src2));
15551
15552 ins_cost(3 * INSN_COST);
15553 format %{ "fcmps $src1, 0.0" %}
15554
15555 ins_encode %{
15556 __ fcmps(as_FloatRegister($src1$$reg), 0.0);
15557 %}
15558
15559 ins_pipe(pipe_class_compare);
15560 %}
15561 // FROM HERE
15562
15563 instruct compD_reg_reg(rFlagsReg cr, vRegD src1, vRegD src2)
15564 %{
15565 match(Set cr (CmpD src1 src2));
15566
15567 ins_cost(3 * INSN_COST);
15568 format %{ "fcmpd $src1, $src2" %}
15569
15570 ins_encode %{
15571 __ fcmpd(as_FloatRegister($src1$$reg), as_FloatRegister($src2$$reg));
15572 %}
15573
15574 ins_pipe(pipe_class_compare);
15575 %}
15576
15577 instruct compD_reg_zero(rFlagsReg cr, vRegD src1, immD0 src2)
15578 %{
15579 match(Set cr (CmpD src1 src2));
15580
15581 ins_cost(3 * INSN_COST);
15582 format %{ "fcmpd $src1, 0.0" %}
15583
15584 ins_encode %{
15585 __ fcmpd(as_FloatRegister($src1$$reg), 0.0);
15586 %}
15587
15588 ins_pipe(pipe_class_compare);
15589 %}
15590
15591 instruct compF3_reg_reg(iRegINoSp dst, vRegF src1, vRegF src2, rFlagsReg cr)
15592 %{
15593 match(Set dst (CmpF3 src1 src2));
15594 effect(KILL cr);
15595
15596 ins_cost(5 * INSN_COST);
15597 format %{ "fcmps $src1, $src2\n\t"
15598 "csinvw($dst, zr, zr, eq\n\t"
15599 "csnegw($dst, $dst, $dst, lt)"
15600 %}
15601
15602 ins_encode %{
15603 Label done;
15604 FloatRegister s1 = as_FloatRegister($src1$$reg);
15605 FloatRegister s2 = as_FloatRegister($src2$$reg);
15606 Register d = as_Register($dst$$reg);
15607 __ fcmps(s1, s2);
15608 // installs 0 if EQ else -1
15609 __ csinvw(d, zr, zr, Assembler::EQ);
15610 // keeps -1 if less or unordered else installs 1
15611 __ csnegw(d, d, d, Assembler::LT);
15612 __ bind(done);
15613 %}
15614
15615 ins_pipe(pipe_class_default);
15616
15617 %}
15618
15619 instruct compD3_reg_reg(iRegINoSp dst, vRegD src1, vRegD src2, rFlagsReg cr)
15620 %{
15621 match(Set dst (CmpD3 src1 src2));
15622 effect(KILL cr);
15623
15624 ins_cost(5 * INSN_COST);
15625 format %{ "fcmpd $src1, $src2\n\t"
15626 "csinvw($dst, zr, zr, eq\n\t"
15627 "csnegw($dst, $dst, $dst, lt)"
15628 %}
15629
15630 ins_encode %{
15631 Label done;
15632 FloatRegister s1 = as_FloatRegister($src1$$reg);
15633 FloatRegister s2 = as_FloatRegister($src2$$reg);
15634 Register d = as_Register($dst$$reg);
15635 __ fcmpd(s1, s2);
15636 // installs 0 if EQ else -1
15637 __ csinvw(d, zr, zr, Assembler::EQ);
15638 // keeps -1 if less or unordered else installs 1
15639 __ csnegw(d, d, d, Assembler::LT);
15640 __ bind(done);
15641 %}
15642 ins_pipe(pipe_class_default);
15643
15644 %}
15645
15646 instruct compF3_reg_immF0(iRegINoSp dst, vRegF src1, immF0 zero, rFlagsReg cr)
15647 %{
15648 match(Set dst (CmpF3 src1 zero));
15649 effect(KILL cr);
15650
15651 ins_cost(5 * INSN_COST);
15652 format %{ "fcmps $src1, 0.0\n\t"
15653 "csinvw($dst, zr, zr, eq\n\t"
15654 "csnegw($dst, $dst, $dst, lt)"
15655 %}
15656
15657 ins_encode %{
15658 Label done;
15659 FloatRegister s1 = as_FloatRegister($src1$$reg);
15660 Register d = as_Register($dst$$reg);
15661 __ fcmps(s1, 0.0);
15662 // installs 0 if EQ else -1
15663 __ csinvw(d, zr, zr, Assembler::EQ);
15664 // keeps -1 if less or unordered else installs 1
15665 __ csnegw(d, d, d, Assembler::LT);
15666 __ bind(done);
15667 %}
15668
15669 ins_pipe(pipe_class_default);
15670
15671 %}
15672
15673 instruct compD3_reg_immD0(iRegINoSp dst, vRegD src1, immD0 zero, rFlagsReg cr)
15674 %{
15675 match(Set dst (CmpD3 src1 zero));
15676 effect(KILL cr);
15677
15678 ins_cost(5 * INSN_COST);
15679 format %{ "fcmpd $src1, 0.0\n\t"
15680 "csinvw($dst, zr, zr, eq\n\t"
15681 "csnegw($dst, $dst, $dst, lt)"
15682 %}
15683
15684 ins_encode %{
15685 Label done;
15686 FloatRegister s1 = as_FloatRegister($src1$$reg);
15687 Register d = as_Register($dst$$reg);
15688 __ fcmpd(s1, 0.0);
15689 // installs 0 if EQ else -1
15690 __ csinvw(d, zr, zr, Assembler::EQ);
15691 // keeps -1 if less or unordered else installs 1
15692 __ csnegw(d, d, d, Assembler::LT);
15693 __ bind(done);
15694 %}
15695 ins_pipe(pipe_class_default);
15696
15697 %}
15698
15699 instruct cmpLTMask_reg_reg(iRegINoSp dst, iRegIorL2I p, iRegIorL2I q, rFlagsReg cr)
15700 %{
15701 match(Set dst (CmpLTMask p q));
15702 effect(KILL cr);
15703
15704 ins_cost(3 * INSN_COST);
15705
15706 format %{ "cmpw $p, $q\t# cmpLTMask\n\t"
15707 "csetw $dst, lt\n\t"
15708 "subw $dst, zr, $dst"
15709 %}
15710
15711 ins_encode %{
15712 __ cmpw(as_Register($p$$reg), as_Register($q$$reg));
15713 __ csetw(as_Register($dst$$reg), Assembler::LT);
15714 __ subw(as_Register($dst$$reg), zr, as_Register($dst$$reg));
15715 %}
15716
15717 ins_pipe(ialu_reg_reg);
15718 %}
15719
15720 instruct cmpLTMask_reg_zero(iRegINoSp dst, iRegIorL2I src, immI0 zero, rFlagsReg cr)
15721 %{
15722 match(Set dst (CmpLTMask src zero));
15723 effect(KILL cr);
15724
15725 ins_cost(INSN_COST);
15726
15727 format %{ "asrw $dst, $src, #31\t# cmpLTMask0" %}
15728
15729 ins_encode %{
15730 __ asrw(as_Register($dst$$reg), as_Register($src$$reg), 31);
15731 %}
15732
15733 ins_pipe(ialu_reg_shift);
15734 %}
15735
15736 // ============================================================================
15737 // Max and Min
15738
15739 instruct cmovI_reg_reg_lt(iRegINoSp dst, iRegI src1, iRegI src2, rFlagsReg cr)
15740 %{
15741 effect( DEF dst, USE src1, USE src2, USE cr );
15742
15743 ins_cost(INSN_COST * 2);
15744 format %{ "cselw $dst, $src1, $src2 lt\t" %}
15745
15746 ins_encode %{
15747 __ cselw(as_Register($dst$$reg),
15748 as_Register($src1$$reg),
15749 as_Register($src2$$reg),
15750 Assembler::LT);
15751 %}
15752
15753 ins_pipe(icond_reg_reg);
15754 %}
15755
15756 instruct minI_rReg(iRegINoSp dst, iRegI src1, iRegI src2)
15757 %{
15758 match(Set dst (MinI src1 src2));
15759 ins_cost(INSN_COST * 3);
15760
15761 expand %{
15762 rFlagsReg cr;
15763 compI_reg_reg(cr, src1, src2);
15764 cmovI_reg_reg_lt(dst, src1, src2, cr);
15765 %}
15766
15767 %}
15768 // FROM HERE
15769
15770 instruct cmovI_reg_reg_gt(iRegINoSp dst, iRegI src1, iRegI src2, rFlagsReg cr)
15771 %{
15772 effect( DEF dst, USE src1, USE src2, USE cr );
15773
15774 ins_cost(INSN_COST * 2);
15775 format %{ "cselw $dst, $src1, $src2 gt\t" %}
15776
15777 ins_encode %{
15778 __ cselw(as_Register($dst$$reg),
15779 as_Register($src1$$reg),
15780 as_Register($src2$$reg),
15781 Assembler::GT);
15782 %}
15783
15784 ins_pipe(icond_reg_reg);
15785 %}
15786
15787 instruct maxI_rReg(iRegINoSp dst, iRegI src1, iRegI src2)
15788 %{
15789 match(Set dst (MaxI src1 src2));
15790 ins_cost(INSN_COST * 3);
15791 expand %{
15792 rFlagsReg cr;
15793 compI_reg_reg(cr, src1, src2);
15794 cmovI_reg_reg_gt(dst, src1, src2, cr);
15795 %}
15796 %}
15797
15798 // ============================================================================
15799 // Branch Instructions
15800
15801 // Direct Branch.
15802 instruct branch(label lbl)
15803 %{
15804 match(Goto);
15805
15806 effect(USE lbl);
15807
15808 ins_cost(BRANCH_COST);
15809 format %{ "b $lbl" %}
15810
15811 ins_encode(aarch64_enc_b(lbl));
15812
15813 ins_pipe(pipe_branch);
15814 %}
15815
15816 // Conditional Near Branch
15817 instruct branchCon(cmpOp cmp, rFlagsReg cr, label lbl)
15818 %{
15819 // Same match rule as `branchConFar'.
15820 match(If cmp cr);
15821
15822 effect(USE lbl);
15823
15824 ins_cost(BRANCH_COST);
15825 // If set to 1 this indicates that the current instruction is a
15826 // short variant of a long branch. This avoids using this
15827 // instruction in first-pass matching. It will then only be used in
15828 // the `Shorten_branches' pass.
15829 // ins_short_branch(1);
15830 format %{ "b$cmp $lbl" %}
15831
15832 ins_encode(aarch64_enc_br_con(cmp, lbl));
15833
15834 ins_pipe(pipe_branch_cond);
15835 %}
15836
15837 // Conditional Near Branch Unsigned
15838 instruct branchConU(cmpOpU cmp, rFlagsRegU cr, label lbl)
15839 %{
15840 // Same match rule as `branchConFar'.
15841 match(If cmp cr);
15842
15843 effect(USE lbl);
15844
15845 ins_cost(BRANCH_COST);
15846 // If set to 1 this indicates that the current instruction is a
15847 // short variant of a long branch. This avoids using this
15848 // instruction in first-pass matching. It will then only be used in
15849 // the `Shorten_branches' pass.
15850 // ins_short_branch(1);
15851 format %{ "b$cmp $lbl\t# unsigned" %}
15852
15853 ins_encode(aarch64_enc_br_conU(cmp, lbl));
15854
15855 ins_pipe(pipe_branch_cond);
15856 %}
15857
15858 // Make use of CBZ and CBNZ. These instructions, as well as being
15859 // shorter than (cmp; branch), have the additional benefit of not
15860 // killing the flags.
15861
15862 instruct cmpI_imm0_branch(cmpOpEqNe cmp, iRegIorL2I op1, immI0 op2, label labl, rFlagsReg cr) %{
15863 match(If cmp (CmpI op1 op2));
15864 effect(USE labl);
15865
15866 ins_cost(BRANCH_COST);
15867 format %{ "cbw$cmp $op1, $labl" %}
15868 ins_encode %{
15869 Label* L = $labl$$label;
15870 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15871 if (cond == Assembler::EQ)
15872 __ cbzw($op1$$Register, *L);
15873 else
15874 __ cbnzw($op1$$Register, *L);
15875 %}
15876 ins_pipe(pipe_cmp_branch);
15877 %}
15878
15879 instruct cmpL_imm0_branch(cmpOpEqNe cmp, iRegL op1, immL0 op2, label labl, rFlagsReg cr) %{
15880 match(If cmp (CmpL op1 op2));
15881 effect(USE labl);
15882
15883 ins_cost(BRANCH_COST);
15884 format %{ "cb$cmp $op1, $labl" %}
15885 ins_encode %{
15886 Label* L = $labl$$label;
15887 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15888 if (cond == Assembler::EQ)
15889 __ cbz($op1$$Register, *L);
15890 else
15891 __ cbnz($op1$$Register, *L);
15892 %}
15893 ins_pipe(pipe_cmp_branch);
15894 %}
15895
15896 instruct cmpP_imm0_branch(cmpOpEqNe cmp, iRegP op1, immP0 op2, label labl, rFlagsReg cr) %{
15897 match(If cmp (CmpP op1 op2));
15898 effect(USE labl);
15899
15900 ins_cost(BRANCH_COST);
15901 format %{ "cb$cmp $op1, $labl" %}
15902 ins_encode %{
15903 Label* L = $labl$$label;
15904 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15905 if (cond == Assembler::EQ)
15906 __ cbz($op1$$Register, *L);
15907 else
15908 __ cbnz($op1$$Register, *L);
15909 %}
15910 ins_pipe(pipe_cmp_branch);
15911 %}
15912
15913 instruct cmpN_imm0_branch(cmpOpEqNe cmp, iRegN op1, immN0 op2, label labl, rFlagsReg cr) %{
15914 match(If cmp (CmpN op1 op2));
15915 effect(USE labl);
15916
15917 ins_cost(BRANCH_COST);
15918 format %{ "cbw$cmp $op1, $labl" %}
15919 ins_encode %{
15920 Label* L = $labl$$label;
15921 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15922 if (cond == Assembler::EQ)
15923 __ cbzw($op1$$Register, *L);
15924 else
15925 __ cbnzw($op1$$Register, *L);
15926 %}
15927 ins_pipe(pipe_cmp_branch);
15928 %}
15929
15930 instruct cmpP_narrowOop_imm0_branch(cmpOpEqNe cmp, iRegN oop, immP0 zero, label labl, rFlagsReg cr) %{
15931 match(If cmp (CmpP (DecodeN oop) zero));
15932 effect(USE labl);
15933
15934 ins_cost(BRANCH_COST);
15935 format %{ "cb$cmp $oop, $labl" %}
15936 ins_encode %{
15937 Label* L = $labl$$label;
15938 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15939 if (cond == Assembler::EQ)
15940 __ cbzw($oop$$Register, *L);
15941 else
15942 __ cbnzw($oop$$Register, *L);
15943 %}
15944 ins_pipe(pipe_cmp_branch);
15945 %}
15946
15947 instruct cmpUI_imm0_branch(cmpOpUEqNeLtGe cmp, iRegIorL2I op1, immI0 op2, label labl, rFlagsRegU cr) %{
15948 match(If cmp (CmpU op1 op2));
15949 effect(USE labl);
15950
15951 ins_cost(BRANCH_COST);
15952 format %{ "cbw$cmp $op1, $labl" %}
15953 ins_encode %{
15954 Label* L = $labl$$label;
15955 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15956 if (cond == Assembler::EQ || cond == Assembler::LS)
15957 __ cbzw($op1$$Register, *L);
15958 else
15959 __ cbnzw($op1$$Register, *L);
15960 %}
15961 ins_pipe(pipe_cmp_branch);
15962 %}
15963
15964 instruct cmpUL_imm0_branch(cmpOpUEqNeLtGe cmp, iRegL op1, immL0 op2, label labl, rFlagsRegU cr) %{
15965 match(If cmp (CmpUL op1 op2));
15966 effect(USE labl);
15967
15968 ins_cost(BRANCH_COST);
15969 format %{ "cb$cmp $op1, $labl" %}
15970 ins_encode %{
15971 Label* L = $labl$$label;
15972 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15973 if (cond == Assembler::EQ || cond == Assembler::LS)
15974 __ cbz($op1$$Register, *L);
15975 else
15976 __ cbnz($op1$$Register, *L);
15977 %}
15978 ins_pipe(pipe_cmp_branch);
15979 %}
15980
15981 // Test bit and Branch
15982
15983 // Patterns for short (< 32KiB) variants
15984 instruct cmpL_branch_sign(cmpOpLtGe cmp, iRegL op1, immL0 op2, label labl) %{
15985 match(If cmp (CmpL op1 op2));
15986 effect(USE labl);
15987
15988 ins_cost(BRANCH_COST);
15989 format %{ "cb$cmp $op1, $labl # long" %}
15990 ins_encode %{
15991 Label* L = $labl$$label;
15992 Assembler::Condition cond =
15993 ((Assembler::Condition)$cmp$$cmpcode == Assembler::LT) ? Assembler::NE : Assembler::EQ;
15994 __ tbr(cond, $op1$$Register, 63, *L);
15995 %}
15996 ins_pipe(pipe_cmp_branch);
15997 ins_short_branch(1);
15998 %}
15999
16000 instruct cmpI_branch_sign(cmpOpLtGe cmp, iRegIorL2I op1, immI0 op2, label labl) %{
16001 match(If cmp (CmpI op1 op2));
16002 effect(USE labl);
16003
16004 ins_cost(BRANCH_COST);
16005 format %{ "cb$cmp $op1, $labl # int" %}
16006 ins_encode %{
16007 Label* L = $labl$$label;
16008 Assembler::Condition cond =
16009 ((Assembler::Condition)$cmp$$cmpcode == Assembler::LT) ? Assembler::NE : Assembler::EQ;
16010 __ tbr(cond, $op1$$Register, 31, *L);
16011 %}
16012 ins_pipe(pipe_cmp_branch);
16013 ins_short_branch(1);
16014 %}
16015
16016 instruct cmpL_branch_bit(cmpOpEqNe cmp, iRegL op1, immL op2, immL0 op3, label labl) %{
16017 match(If cmp (CmpL (AndL op1 op2) op3));
16018 predicate(is_power_of_2((julong)n->in(2)->in(1)->in(2)->get_long()));
16019 effect(USE labl);
16020
16021 ins_cost(BRANCH_COST);
16022 format %{ "tb$cmp $op1, $op2, $labl" %}
16023 ins_encode %{
16024 Label* L = $labl$$label;
16025 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
16026 int bit = exact_log2_long($op2$$constant);
16027 __ tbr(cond, $op1$$Register, bit, *L);
16028 %}
16029 ins_pipe(pipe_cmp_branch);
16030 ins_short_branch(1);
16031 %}
16032
16033 instruct cmpI_branch_bit(cmpOpEqNe cmp, iRegIorL2I op1, immI op2, immI0 op3, label labl) %{
16034 match(If cmp (CmpI (AndI op1 op2) op3));
16035 predicate(is_power_of_2((juint)n->in(2)->in(1)->in(2)->get_int()));
16036 effect(USE labl);
16037
16038 ins_cost(BRANCH_COST);
16039 format %{ "tb$cmp $op1, $op2, $labl" %}
16040 ins_encode %{
16041 Label* L = $labl$$label;
16042 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
16043 int bit = exact_log2((juint)$op2$$constant);
16044 __ tbr(cond, $op1$$Register, bit, *L);
16045 %}
16046 ins_pipe(pipe_cmp_branch);
16047 ins_short_branch(1);
16048 %}
16049
16050 // And far variants
16051 instruct far_cmpL_branch_sign(cmpOpLtGe cmp, iRegL op1, immL0 op2, label labl) %{
16052 match(If cmp (CmpL op1 op2));
16053 effect(USE labl);
16054
16055 ins_cost(BRANCH_COST);
16056 format %{ "cb$cmp $op1, $labl # long" %}
16057 ins_encode %{
16058 Label* L = $labl$$label;
16059 Assembler::Condition cond =
16060 ((Assembler::Condition)$cmp$$cmpcode == Assembler::LT) ? Assembler::NE : Assembler::EQ;
16061 __ tbr(cond, $op1$$Register, 63, *L, /*far*/true);
16062 %}
16063 ins_pipe(pipe_cmp_branch);
16064 %}
16065
16066 instruct far_cmpI_branch_sign(cmpOpLtGe cmp, iRegIorL2I op1, immI0 op2, label labl) %{
16067 match(If cmp (CmpI op1 op2));
16068 effect(USE labl);
16069
16070 ins_cost(BRANCH_COST);
16071 format %{ "cb$cmp $op1, $labl # int" %}
16072 ins_encode %{
16073 Label* L = $labl$$label;
16074 Assembler::Condition cond =
16075 ((Assembler::Condition)$cmp$$cmpcode == Assembler::LT) ? Assembler::NE : Assembler::EQ;
16076 __ tbr(cond, $op1$$Register, 31, *L, /*far*/true);
16077 %}
16078 ins_pipe(pipe_cmp_branch);
16079 %}
16080
16081 instruct far_cmpL_branch_bit(cmpOpEqNe cmp, iRegL op1, immL op2, immL0 op3, label labl) %{
16082 match(If cmp (CmpL (AndL op1 op2) op3));
16083 predicate(is_power_of_2((julong)n->in(2)->in(1)->in(2)->get_long()));
16084 effect(USE labl);
16085
16086 ins_cost(BRANCH_COST);
16087 format %{ "tb$cmp $op1, $op2, $labl" %}
16088 ins_encode %{
16089 Label* L = $labl$$label;
16090 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
16091 int bit = exact_log2_long($op2$$constant);
16092 __ tbr(cond, $op1$$Register, bit, *L, /*far*/true);
16093 %}
16094 ins_pipe(pipe_cmp_branch);
16095 %}
16096
16097 instruct far_cmpI_branch_bit(cmpOpEqNe cmp, iRegIorL2I op1, immI op2, immI0 op3, label labl) %{
16098 match(If cmp (CmpI (AndI op1 op2) op3));
16099 predicate(is_power_of_2((juint)n->in(2)->in(1)->in(2)->get_int()));
16100 effect(USE labl);
16101
16102 ins_cost(BRANCH_COST);
16103 format %{ "tb$cmp $op1, $op2, $labl" %}
16104 ins_encode %{
16105 Label* L = $labl$$label;
16106 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
16107 int bit = exact_log2((juint)$op2$$constant);
16108 __ tbr(cond, $op1$$Register, bit, *L, /*far*/true);
16109 %}
16110 ins_pipe(pipe_cmp_branch);
16111 %}
16112
16113 // Test bits
16114
16115 instruct cmpL_and(cmpOp cmp, iRegL op1, immL op2, immL0 op3, rFlagsReg cr) %{
16116 match(Set cr (CmpL (AndL op1 op2) op3));
16117 predicate(Assembler::operand_valid_for_logical_immediate
16118 (/*is_32*/false, n->in(1)->in(2)->get_long()));
16119
16120 ins_cost(INSN_COST);
16121 format %{ "tst $op1, $op2 # long" %}
16122 ins_encode %{
16123 __ tst($op1$$Register, $op2$$constant);
16124 %}
16125 ins_pipe(ialu_reg_reg);
16126 %}
16127
16128 instruct cmpI_and(cmpOp cmp, iRegIorL2I op1, immI op2, immI0 op3, rFlagsReg cr) %{
16129 match(Set cr (CmpI (AndI op1 op2) op3));
16130 predicate(Assembler::operand_valid_for_logical_immediate
16131 (/*is_32*/true, n->in(1)->in(2)->get_int()));
16132
16133 ins_cost(INSN_COST);
16134 format %{ "tst $op1, $op2 # int" %}
16135 ins_encode %{
16136 __ tstw($op1$$Register, $op2$$constant);
16137 %}
16138 ins_pipe(ialu_reg_reg);
16139 %}
16140
16141 instruct cmpL_and_reg(cmpOp cmp, iRegL op1, iRegL op2, immL0 op3, rFlagsReg cr) %{
16142 match(Set cr (CmpL (AndL op1 op2) op3));
16143
16144 ins_cost(INSN_COST);
16145 format %{ "tst $op1, $op2 # long" %}
16146 ins_encode %{
16147 __ tst($op1$$Register, $op2$$Register);
16148 %}
16149 ins_pipe(ialu_reg_reg);
16150 %}
16151
16152 instruct cmpI_and_reg(cmpOp cmp, iRegIorL2I op1, iRegIorL2I op2, immI0 op3, rFlagsReg cr) %{
16153 match(Set cr (CmpI (AndI op1 op2) op3));
16154
16155 ins_cost(INSN_COST);
16156 format %{ "tstw $op1, $op2 # int" %}
16157 ins_encode %{
16158 __ tstw($op1$$Register, $op2$$Register);
16159 %}
16160 ins_pipe(ialu_reg_reg);
16161 %}
16162
16163
16164 // Conditional Far Branch
16165 // Conditional Far Branch Unsigned
16166 // TODO: fixme
16167
16168 // counted loop end branch near
16169 instruct branchLoopEnd(cmpOp cmp, rFlagsReg cr, label lbl)
16170 %{
16171 match(CountedLoopEnd cmp cr);
16172
16173 effect(USE lbl);
16174
16175 ins_cost(BRANCH_COST);
16176 // short variant.
16177 // ins_short_branch(1);
16178 format %{ "b$cmp $lbl \t// counted loop end" %}
16179
16180 ins_encode(aarch64_enc_br_con(cmp, lbl));
16181
16182 ins_pipe(pipe_branch);
16183 %}
16184
16185 // counted loop end branch near Unsigned
16186 instruct branchLoopEndU(cmpOpU cmp, rFlagsRegU cr, label lbl)
16187 %{
16188 match(CountedLoopEnd cmp cr);
16189
16190 effect(USE lbl);
16191
16192 ins_cost(BRANCH_COST);
16193 // short variant.
16194 // ins_short_branch(1);
16195 format %{ "b$cmp $lbl \t// counted loop end unsigned" %}
16196
16197 ins_encode(aarch64_enc_br_conU(cmp, lbl));
16198
16199 ins_pipe(pipe_branch);
16200 %}
16201
16202 // counted loop end branch far
16203 // counted loop end branch far unsigned
16204 // TODO: fixme
16205
16206 // ============================================================================
16207 // inlined locking and unlocking
16208
16209 instruct cmpFastLock(rFlagsReg cr, iRegP object, iRegP box, iRegPNoSp tmp, iRegPNoSp tmp2)
16210 %{
16211 match(Set cr (FastLock object box));
16212 effect(TEMP tmp, TEMP tmp2);
16213
16214 // TODO
16215 // identify correct cost
16216 ins_cost(5 * INSN_COST);
16217 format %{ "fastlock $object,$box\t! kills $tmp,$tmp2" %}
16218
16219 ins_encode(aarch64_enc_fast_lock(object, box, tmp, tmp2));
16220
16221 ins_pipe(pipe_serial);
16222 %}
16223
16224 instruct cmpFastUnlock(rFlagsReg cr, iRegP object, iRegP box, iRegPNoSp tmp, iRegPNoSp tmp2)
16225 %{
16226 match(Set cr (FastUnlock object box));
16227 effect(TEMP tmp, TEMP tmp2);
16228
16229 ins_cost(5 * INSN_COST);
16230 format %{ "fastunlock $object,$box\t! kills $tmp, $tmp2" %}
16231
16232 ins_encode(aarch64_enc_fast_unlock(object, box, tmp, tmp2));
16233
16234 ins_pipe(pipe_serial);
16235 %}
16236
16237
16238 // ============================================================================
16239 // Safepoint Instructions
16240
16241 // TODO
16242 // provide a near and far version of this code
16243
16244 instruct safePoint(rFlagsReg cr, iRegP poll)
16245 %{
16246 match(SafePoint poll);
16247 effect(KILL cr);
16248
16249 format %{
16250 "ldrw zr, [$poll]\t# Safepoint: poll for GC"
16251 %}
16252 ins_encode %{
16253 __ read_polling_page(as_Register($poll$$reg), relocInfo::poll_type);
16254 %}
16255 ins_pipe(pipe_serial); // ins_pipe(iload_reg_mem);
16256 %}
16257
16258
16259 // ============================================================================
16260 // Procedure Call/Return Instructions
16261
16262 // Call Java Static Instruction
16263
16264 instruct CallStaticJavaDirect(method meth)
16265 %{
16266 match(CallStaticJava);
16267
16268 effect(USE meth);
16269
16270 ins_cost(CALL_COST);
16271
16272 format %{ "call,static $meth \t// ==> " %}
16273
16274 ins_encode(aarch64_enc_java_static_call(meth),
16275 aarch64_enc_call_epilog);
16276
16277 ins_pipe(pipe_class_call);
16278 %}
16279
16280 // TO HERE
16281
16282 // Call Java Dynamic Instruction
16283 instruct CallDynamicJavaDirect(method meth)
16284 %{
16285 match(CallDynamicJava);
16286
16287 effect(USE meth);
16288
16289 ins_cost(CALL_COST);
16290
16291 format %{ "CALL,dynamic $meth \t// ==> " %}
16292
16293 ins_encode(aarch64_enc_java_dynamic_call(meth),
16294 aarch64_enc_call_epilog);
16295
16296 ins_pipe(pipe_class_call);
16297 %}
16298
16299 // Call Runtime Instruction
16300
16301 instruct CallRuntimeDirect(method meth)
16302 %{
16303 match(CallRuntime);
16304
16305 effect(USE meth);
16306
16307 ins_cost(CALL_COST);
16308
16309 format %{ "CALL, runtime $meth" %}
16310
16311 ins_encode( aarch64_enc_java_to_runtime(meth) );
16312
16313 ins_pipe(pipe_class_call);
16314 %}
16315
16316 // Call Runtime Instruction
16317
16318 instruct CallLeafDirect(method meth)
16319 %{
16320 match(CallLeaf);
16321
16322 effect(USE meth);
16323
16324 ins_cost(CALL_COST);
16325
16326 format %{ "CALL, runtime leaf $meth" %}
16327
16328 ins_encode( aarch64_enc_java_to_runtime(meth) );
16329
16330 ins_pipe(pipe_class_call);
16331 %}
16332
16333 // Call Runtime Instruction
16334
16335 instruct CallLeafNoFPDirect(method meth)
16336 %{
16337 match(CallLeafNoFP);
16338
16339 effect(USE meth);
16340
16341 ins_cost(CALL_COST);
16342
16343 format %{ "CALL, runtime leaf nofp $meth" %}
16344
16345 ins_encode( aarch64_enc_java_to_runtime(meth) );
16346
16347 ins_pipe(pipe_class_call);
16348 %}
16349
16350 instruct CallNativeDirect(method meth)
16351 %{
16352 match(CallNative);
16353
16354 effect(USE meth);
16355
16356 ins_cost(CALL_COST);
16357
16358 format %{ "CALL, native $meth" %}
16359
16360 ins_encode( aarch64_enc_java_to_runtime(meth) );
16361
16362 ins_pipe(pipe_class_call);
16363 %}
16364
16365 // Tail Call; Jump from runtime stub to Java code.
16366 // Also known as an 'interprocedural jump'.
16367 // Target of jump will eventually return to caller.
16368 // TailJump below removes the return address.
16369 instruct TailCalljmpInd(iRegPNoSp jump_target, inline_cache_RegP method_ptr)
16370 %{
16371 match(TailCall jump_target method_ptr);
16372
16373 ins_cost(CALL_COST);
16374
16375 format %{ "br $jump_target\t# $method_ptr holds method" %}
16376
16377 ins_encode(aarch64_enc_tail_call(jump_target));
16378
16379 ins_pipe(pipe_class_call);
16380 %}
16381
16382 instruct TailjmpInd(iRegPNoSp jump_target, iRegP_R0 ex_oop)
16383 %{
16384 match(TailJump jump_target ex_oop);
16385
16386 ins_cost(CALL_COST);
16387
16388 format %{ "br $jump_target\t# $ex_oop holds exception oop" %}
16389
16390 ins_encode(aarch64_enc_tail_jmp(jump_target));
16391
16392 ins_pipe(pipe_class_call);
16393 %}
16394
16395 // Create exception oop: created by stack-crawling runtime code.
16396 // Created exception is now available to this handler, and is setup
16397 // just prior to jumping to this handler. No code emitted.
16398 // TODO check
16399 // should ex_oop be in r0? intel uses rax, ppc cannot use r0 so uses rarg1
16400 instruct CreateException(iRegP_R0 ex_oop)
16401 %{
16402 match(Set ex_oop (CreateEx));
16403
16404 format %{ " -- \t// exception oop; no code emitted" %}
16405
16406 size(0);
16407
16408 ins_encode( /*empty*/ );
16409
16410 ins_pipe(pipe_class_empty);
16411 %}
16412
16413 // Rethrow exception: The exception oop will come in the first
16414 // argument position. Then JUMP (not call) to the rethrow stub code.
16415 instruct RethrowException() %{
16416 match(Rethrow);
16417 ins_cost(CALL_COST);
16418
16419 format %{ "b rethrow_stub" %}
16420
16421 ins_encode( aarch64_enc_rethrow() );
16422
16423 ins_pipe(pipe_class_call);
16424 %}
16425
16426
16427 // Return Instruction
16428 // epilog node loads ret address into lr as part of frame pop
16429 instruct Ret()
16430 %{
16431 match(Return);
16432
16433 format %{ "ret\t// return register" %}
16434
16435 ins_encode( aarch64_enc_ret() );
16436
16437 ins_pipe(pipe_branch);
16438 %}
16439
16440 // Die now.
16441 instruct ShouldNotReachHere() %{
16442 match(Halt);
16443
16444 ins_cost(CALL_COST);
16445 format %{ "ShouldNotReachHere" %}
16446
16447 ins_encode %{
16448 if (is_reachable()) {
16449 __ stop(_halt_reason);
16450 }
16451 %}
16452
16453 ins_pipe(pipe_class_default);
16454 %}
16455
16456 // ============================================================================
16457 // Partial Subtype Check
16458 //
16459 // superklass array for an instance of the superklass. Set a hidden
16460 // internal cache on a hit (cache is checked with exposed code in
16461 // gen_subtype_check()). Return NZ for a miss or zero for a hit. The
16462 // encoding ALSO sets flags.
16463
16464 instruct partialSubtypeCheck(iRegP_R4 sub, iRegP_R0 super, iRegP_R2 temp, iRegP_R5 result, rFlagsReg cr)
16465 %{
16466 match(Set result (PartialSubtypeCheck sub super));
16467 effect(KILL cr, KILL temp);
16468
16469 ins_cost(1100); // slightly larger than the next version
16470 format %{ "partialSubtypeCheck $result, $sub, $super" %}
16471
16472 ins_encode(aarch64_enc_partial_subtype_check(sub, super, temp, result));
16473
16474 opcode(0x1); // Force zero of result reg on hit
16475
16476 ins_pipe(pipe_class_memory);
16477 %}
16478
16479 instruct partialSubtypeCheckVsZero(iRegP_R4 sub, iRegP_R0 super, iRegP_R2 temp, iRegP_R5 result, immP0 zero, rFlagsReg cr)
16480 %{
16481 match(Set cr (CmpP (PartialSubtypeCheck sub super) zero));
16482 effect(KILL temp, KILL result);
16483
16484 ins_cost(1100); // slightly larger than the next version
16485 format %{ "partialSubtypeCheck $result, $sub, $super == 0" %}
16486
16487 ins_encode(aarch64_enc_partial_subtype_check(sub, super, temp, result));
16488
16489 opcode(0x0); // Don't zero result reg on hit
16490
16491 ins_pipe(pipe_class_memory);
16492 %}
16493
16494 instruct string_compareU(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
16495 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2, rFlagsReg cr)
16496 %{
16497 predicate(((StrCompNode*)n)->encoding() == StrIntrinsicNode::UU);
16498 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
16499 effect(KILL tmp1, KILL tmp2, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
16500
16501 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # KILL $tmp1" %}
16502 ins_encode %{
16503 // Count is in 8-bit bytes; non-Compact chars are 16 bits.
16504 __ string_compare($str1$$Register, $str2$$Register,
16505 $cnt1$$Register, $cnt2$$Register, $result$$Register,
16506 $tmp1$$Register, $tmp2$$Register,
16507 fnoreg, fnoreg, fnoreg, StrIntrinsicNode::UU);
16508 %}
16509 ins_pipe(pipe_class_memory);
16510 %}
16511
16512 instruct string_compareL(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
16513 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2, rFlagsReg cr)
16514 %{
16515 predicate(((StrCompNode*)n)->encoding() == StrIntrinsicNode::LL);
16516 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
16517 effect(KILL tmp1, KILL tmp2, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
16518
16519 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # KILL $tmp1" %}
16520 ins_encode %{
16521 __ string_compare($str1$$Register, $str2$$Register,
16522 $cnt1$$Register, $cnt2$$Register, $result$$Register,
16523 $tmp1$$Register, $tmp2$$Register,
16524 fnoreg, fnoreg, fnoreg, StrIntrinsicNode::LL);
16525 %}
16526 ins_pipe(pipe_class_memory);
16527 %}
16528
16529 instruct string_compareUL(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
16530 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
16531 vRegD_V0 vtmp1, vRegD_V1 vtmp2, vRegD_V2 vtmp3, rFlagsReg cr)
16532 %{
16533 predicate(((StrCompNode*)n)->encoding() == StrIntrinsicNode::UL);
16534 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
16535 effect(KILL tmp1, KILL tmp2, KILL vtmp1, KILL vtmp2, KILL vtmp3,
16536 USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
16537
16538 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # KILL $tmp1, $tmp2, $vtmp1, $vtmp2, $vtmp3" %}
16539 ins_encode %{
16540 __ string_compare($str1$$Register, $str2$$Register,
16541 $cnt1$$Register, $cnt2$$Register, $result$$Register,
16542 $tmp1$$Register, $tmp2$$Register,
16543 $vtmp1$$FloatRegister, $vtmp2$$FloatRegister,
16544 $vtmp3$$FloatRegister, StrIntrinsicNode::UL);
16545 %}
16546 ins_pipe(pipe_class_memory);
16547 %}
16548
16549 instruct string_compareLU(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
16550 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
16551 vRegD_V0 vtmp1, vRegD_V1 vtmp2, vRegD_V2 vtmp3, rFlagsReg cr)
16552 %{
16553 predicate(((StrCompNode*)n)->encoding() == StrIntrinsicNode::LU);
16554 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
16555 effect(KILL tmp1, KILL tmp2, KILL vtmp1, KILL vtmp2, KILL vtmp3,
16556 USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
16557
16558 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # KILL $tmp1, $tmp2, $vtmp1, $vtmp2, $vtmp3" %}
16559 ins_encode %{
16560 __ string_compare($str1$$Register, $str2$$Register,
16561 $cnt1$$Register, $cnt2$$Register, $result$$Register,
16562 $tmp1$$Register, $tmp2$$Register,
16563 $vtmp1$$FloatRegister, $vtmp2$$FloatRegister,
16564 $vtmp3$$FloatRegister,StrIntrinsicNode::LU);
16565 %}
16566 ins_pipe(pipe_class_memory);
16567 %}
16568
16569 instruct string_indexofUU(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2, iRegI_R2 cnt2,
16570 iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2, iRegINoSp tmp3,
16571 iRegINoSp tmp4, iRegINoSp tmp5, iRegINoSp tmp6, rFlagsReg cr)
16572 %{
16573 predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UU);
16574 match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 cnt2)));
16575 effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2,
16576 TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, TEMP tmp5, TEMP tmp6, KILL cr);
16577 format %{ "String IndexOf $str1,$cnt1,$str2,$cnt2 -> $result (UU)" %}
16578
16579 ins_encode %{
16580 __ string_indexof($str1$$Register, $str2$$Register,
16581 $cnt1$$Register, $cnt2$$Register,
16582 $tmp1$$Register, $tmp2$$Register,
16583 $tmp3$$Register, $tmp4$$Register,
16584 $tmp5$$Register, $tmp6$$Register,
16585 -1, $result$$Register, StrIntrinsicNode::UU);
16586 %}
16587 ins_pipe(pipe_class_memory);
16588 %}
16589
16590 instruct string_indexofLL(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2, iRegI_R2 cnt2,
16591 iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2, iRegINoSp tmp3,
16592 iRegINoSp tmp4, iRegINoSp tmp5, iRegINoSp tmp6, rFlagsReg cr)
16593 %{
16594 predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::LL);
16595 match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 cnt2)));
16596 effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2,
16597 TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, TEMP tmp5, TEMP tmp6, KILL cr);
16598 format %{ "String IndexOf $str1,$cnt1,$str2,$cnt2 -> $result (LL)" %}
16599
16600 ins_encode %{
16601 __ string_indexof($str1$$Register, $str2$$Register,
16602 $cnt1$$Register, $cnt2$$Register,
16603 $tmp1$$Register, $tmp2$$Register,
16604 $tmp3$$Register, $tmp4$$Register,
16605 $tmp5$$Register, $tmp6$$Register,
16606 -1, $result$$Register, StrIntrinsicNode::LL);
16607 %}
16608 ins_pipe(pipe_class_memory);
16609 %}
16610
16611 instruct string_indexofUL(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2, iRegI_R2 cnt2,
16612 iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2, iRegINoSp tmp3,
16613 iRegINoSp tmp4, iRegINoSp tmp5, iRegINoSp tmp6, rFlagsReg cr)
16614 %{
16615 predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UL);
16616 match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 cnt2)));
16617 effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2,
16618 TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, TEMP tmp5, TEMP tmp6, KILL cr);
16619 format %{ "String IndexOf $str1,$cnt1,$str2,$cnt2 -> $result (UL)" %}
16620
16621 ins_encode %{
16622 __ string_indexof($str1$$Register, $str2$$Register,
16623 $cnt1$$Register, $cnt2$$Register,
16624 $tmp1$$Register, $tmp2$$Register,
16625 $tmp3$$Register, $tmp4$$Register,
16626 $tmp5$$Register, $tmp6$$Register,
16627 -1, $result$$Register, StrIntrinsicNode::UL);
16628 %}
16629 ins_pipe(pipe_class_memory);
16630 %}
16631
16632 instruct string_indexof_conUU(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2,
16633 immI_le_4 int_cnt2, iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2,
16634 iRegINoSp tmp3, iRegINoSp tmp4, rFlagsReg cr)
16635 %{
16636 predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UU);
16637 match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 int_cnt2)));
16638 effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1,
16639 TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, KILL cr);
16640 format %{ "String IndexOf $str1,$cnt1,$str2,$int_cnt2 -> $result (UU)" %}
16641
16642 ins_encode %{
16643 int icnt2 = (int)$int_cnt2$$constant;
16644 __ string_indexof($str1$$Register, $str2$$Register,
16645 $cnt1$$Register, zr,
16646 $tmp1$$Register, $tmp2$$Register,
16647 $tmp3$$Register, $tmp4$$Register, zr, zr,
16648 icnt2, $result$$Register, StrIntrinsicNode::UU);
16649 %}
16650 ins_pipe(pipe_class_memory);
16651 %}
16652
16653 instruct string_indexof_conLL(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2,
16654 immI_le_4 int_cnt2, iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2,
16655 iRegINoSp tmp3, iRegINoSp tmp4, rFlagsReg cr)
16656 %{
16657 predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::LL);
16658 match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 int_cnt2)));
16659 effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1,
16660 TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, KILL cr);
16661 format %{ "String IndexOf $str1,$cnt1,$str2,$int_cnt2 -> $result (LL)" %}
16662
16663 ins_encode %{
16664 int icnt2 = (int)$int_cnt2$$constant;
16665 __ string_indexof($str1$$Register, $str2$$Register,
16666 $cnt1$$Register, zr,
16667 $tmp1$$Register, $tmp2$$Register,
16668 $tmp3$$Register, $tmp4$$Register, zr, zr,
16669 icnt2, $result$$Register, StrIntrinsicNode::LL);
16670 %}
16671 ins_pipe(pipe_class_memory);
16672 %}
16673
16674 instruct string_indexof_conUL(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2,
16675 immI_1 int_cnt2, iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2,
16676 iRegINoSp tmp3, iRegINoSp tmp4, rFlagsReg cr)
16677 %{
16678 predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UL);
16679 match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 int_cnt2)));
16680 effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1,
16681 TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, KILL cr);
16682 format %{ "String IndexOf $str1,$cnt1,$str2,$int_cnt2 -> $result (UL)" %}
16683
16684 ins_encode %{
16685 int icnt2 = (int)$int_cnt2$$constant;
16686 __ string_indexof($str1$$Register, $str2$$Register,
16687 $cnt1$$Register, zr,
16688 $tmp1$$Register, $tmp2$$Register,
16689 $tmp3$$Register, $tmp4$$Register, zr, zr,
16690 icnt2, $result$$Register, StrIntrinsicNode::UL);
16691 %}
16692 ins_pipe(pipe_class_memory);
16693 %}
16694
16695 instruct string_indexof_char(iRegP_R1 str1, iRegI_R2 cnt1, iRegI_R3 ch,
16696 iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2,
16697 iRegINoSp tmp3, rFlagsReg cr)
16698 %{
16699 match(Set result (StrIndexOfChar (Binary str1 cnt1) ch));
16700 predicate((UseSVE == 0) && (((StrIndexOfCharNode*)n)->encoding() == StrIntrinsicNode::U));
16701 effect(USE_KILL str1, USE_KILL cnt1, USE_KILL ch,
16702 TEMP tmp1, TEMP tmp2, TEMP tmp3, KILL cr);
16703
16704 format %{ "StringUTF16 IndexOf char[] $str1,$cnt1,$ch -> $result" %}
16705
16706 ins_encode %{
16707 __ string_indexof_char($str1$$Register, $cnt1$$Register, $ch$$Register,
16708 $result$$Register, $tmp1$$Register, $tmp2$$Register,
16709 $tmp3$$Register);
16710 %}
16711 ins_pipe(pipe_class_memory);
16712 %}
16713
16714 instruct stringL_indexof_char(iRegP_R1 str1, iRegI_R2 cnt1, iRegI_R3 ch,
16715 iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2,
16716 iRegINoSp tmp3, rFlagsReg cr)
16717 %{
16718 match(Set result (StrIndexOfChar (Binary str1 cnt1) ch));
16719 predicate((UseSVE == 0) && (((StrIndexOfCharNode*)n)->encoding() == StrIntrinsicNode::L));
16720 effect(USE_KILL str1, USE_KILL cnt1, USE_KILL ch,
16721 TEMP tmp1, TEMP tmp2, TEMP tmp3, KILL cr);
16722
16723 format %{ "StringLatin1 IndexOf char[] $str1,$cnt1,$ch -> $result" %}
16724
16725 ins_encode %{
16726 __ stringL_indexof_char($str1$$Register, $cnt1$$Register, $ch$$Register,
16727 $result$$Register, $tmp1$$Register, $tmp2$$Register,
16728 $tmp3$$Register);
16729 %}
16730 ins_pipe(pipe_class_memory);
16731 %}
16732
16733 instruct string_equalsL(iRegP_R1 str1, iRegP_R3 str2, iRegI_R4 cnt,
16734 iRegI_R0 result, rFlagsReg cr)
16735 %{
16736 predicate(((StrEqualsNode*)n)->encoding() == StrIntrinsicNode::LL);
16737 match(Set result (StrEquals (Binary str1 str2) cnt));
16738 effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt, KILL cr);
16739
16740 format %{ "String Equals $str1,$str2,$cnt -> $result" %}
16741 ins_encode %{
16742 // Count is in 8-bit bytes; non-Compact chars are 16 bits.
16743 __ string_equals($str1$$Register, $str2$$Register,
16744 $result$$Register, $cnt$$Register, 1);
16745 %}
16746 ins_pipe(pipe_class_memory);
16747 %}
16748
16749 instruct string_equalsU(iRegP_R1 str1, iRegP_R3 str2, iRegI_R4 cnt,
16750 iRegI_R0 result, rFlagsReg cr)
16751 %{
16752 predicate(((StrEqualsNode*)n)->encoding() == StrIntrinsicNode::UU);
16753 match(Set result (StrEquals (Binary str1 str2) cnt));
16754 effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt, KILL cr);
16755
16756 format %{ "String Equals $str1,$str2,$cnt -> $result" %}
16757 ins_encode %{
16758 // Count is in 8-bit bytes; non-Compact chars are 16 bits.
16759 __ string_equals($str1$$Register, $str2$$Register,
16760 $result$$Register, $cnt$$Register, 2);
16761 %}
16762 ins_pipe(pipe_class_memory);
16763 %}
16764
16765 instruct array_equalsB(iRegP_R1 ary1, iRegP_R2 ary2, iRegI_R0 result,
16766 iRegP_R3 tmp1, iRegP_R4 tmp2, iRegP_R5 tmp3,
16767 iRegP_R10 tmp, rFlagsReg cr)
16768 %{
16769 predicate(((AryEqNode*)n)->encoding() == StrIntrinsicNode::LL);
16770 match(Set result (AryEq ary1 ary2));
16771 effect(KILL tmp, USE_KILL ary1, USE_KILL ary2, TEMP tmp1, TEMP tmp2, TEMP tmp3, KILL cr);
16772
16773 format %{ "Array Equals $ary1,ary2 -> $result // KILL $tmp" %}
16774 ins_encode %{
16775 address tpc = __ arrays_equals($ary1$$Register, $ary2$$Register,
16776 $tmp1$$Register, $tmp2$$Register, $tmp3$$Register,
16777 $result$$Register, $tmp$$Register, 1);
16778 if (tpc == NULL) {
16779 ciEnv::current()->record_failure("CodeCache is full");
16780 return;
16781 }
16782 %}
16783 ins_pipe(pipe_class_memory);
16784 %}
16785
16786 instruct array_equalsC(iRegP_R1 ary1, iRegP_R2 ary2, iRegI_R0 result,
16787 iRegP_R3 tmp1, iRegP_R4 tmp2, iRegP_R5 tmp3,
16788 iRegP_R10 tmp, rFlagsReg cr)
16789 %{
16790 predicate(((AryEqNode*)n)->encoding() == StrIntrinsicNode::UU);
16791 match(Set result (AryEq ary1 ary2));
16792 effect(KILL tmp, USE_KILL ary1, USE_KILL ary2, TEMP tmp1, TEMP tmp2, TEMP tmp3, KILL cr);
16793
16794 format %{ "Array Equals $ary1,ary2 -> $result // KILL $tmp" %}
16795 ins_encode %{
16796 address tpc = __ arrays_equals($ary1$$Register, $ary2$$Register,
16797 $tmp1$$Register, $tmp2$$Register, $tmp3$$Register,
16798 $result$$Register, $tmp$$Register, 2);
16799 if (tpc == NULL) {
16800 ciEnv::current()->record_failure("CodeCache is full");
16801 return;
16802 }
16803 %}
16804 ins_pipe(pipe_class_memory);
16805 %}
16806
16807 instruct has_negatives(iRegP_R1 ary1, iRegI_R2 len, iRegI_R0 result, rFlagsReg cr)
16808 %{
16809 match(Set result (HasNegatives ary1 len));
16810 effect(USE_KILL ary1, USE_KILL len, KILL cr);
16811 format %{ "has negatives byte[] $ary1,$len -> $result" %}
16812 ins_encode %{
16813 address tpc = __ has_negatives($ary1$$Register, $len$$Register, $result$$Register);
16814 if (tpc == NULL) {
16815 ciEnv::current()->record_failure("CodeCache is full");
16816 return;
16817 }
16818 %}
16819 ins_pipe( pipe_slow );
16820 %}
16821
16822 // fast char[] to byte[] compression
16823 instruct string_compress(iRegP_R2 src, iRegP_R1 dst, iRegI_R3 len,
16824 vRegD_V0 tmp1, vRegD_V1 tmp2,
16825 vRegD_V2 tmp3, vRegD_V3 tmp4,
16826 iRegI_R0 result, rFlagsReg cr)
16827 %{
16828 match(Set result (StrCompressedCopy src (Binary dst len)));
16829 effect(TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, USE_KILL src, USE_KILL dst, USE_KILL len, KILL cr);
16830
16831 format %{ "String Compress $src,$dst -> $result // KILL R1, R2, R3, R4" %}
16832 ins_encode %{
16833 __ char_array_compress($src$$Register, $dst$$Register, $len$$Register,
16834 $tmp1$$FloatRegister, $tmp2$$FloatRegister,
16835 $tmp3$$FloatRegister, $tmp4$$FloatRegister,
16836 $result$$Register);
16837 %}
16838 ins_pipe( pipe_slow );
16839 %}
16840
16841 // fast byte[] to char[] inflation
16842 instruct string_inflate(Universe dummy, iRegP_R0 src, iRegP_R1 dst, iRegI_R2 len,
16843 vRegD_V0 tmp1, vRegD_V1 tmp2, vRegD_V2 tmp3, iRegP_R3 tmp4, rFlagsReg cr)
16844 %{
16845 match(Set dummy (StrInflatedCopy src (Binary dst len)));
16846 effect(TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, USE_KILL src, USE_KILL dst, USE_KILL len, KILL cr);
16847
16848 format %{ "String Inflate $src,$dst // KILL $tmp1, $tmp2" %}
16849 ins_encode %{
16850 address tpc = __ byte_array_inflate($src$$Register, $dst$$Register, $len$$Register,
16851 $tmp1$$FloatRegister, $tmp2$$FloatRegister,
16852 $tmp3$$FloatRegister, $tmp4$$Register);
16853 if (tpc == NULL) {
16854 ciEnv::current()->record_failure("CodeCache is full");
16855 return;
16856 }
16857 %}
16858 ins_pipe(pipe_class_memory);
16859 %}
16860
16861 // encode char[] to byte[] in ISO_8859_1
16862 instruct encode_iso_array(iRegP_R2 src, iRegP_R1 dst, iRegI_R3 len,
16863 vRegD_V0 Vtmp1, vRegD_V1 Vtmp2,
16864 vRegD_V2 Vtmp3, vRegD_V3 Vtmp4,
16865 iRegI_R0 result, rFlagsReg cr)
16866 %{
16867 predicate(!((EncodeISOArrayNode*)n)->is_ascii());
16868 match(Set result (EncodeISOArray src (Binary dst len)));
16869 effect(USE_KILL src, USE_KILL dst, USE_KILL len,
16870 KILL Vtmp1, KILL Vtmp2, KILL Vtmp3, KILL Vtmp4, KILL cr);
16871
16872 format %{ "Encode array $src,$dst,$len -> $result" %}
16873 ins_encode %{
16874 __ encode_iso_array($src$$Register, $dst$$Register, $len$$Register,
16875 $result$$Register, $Vtmp1$$FloatRegister, $Vtmp2$$FloatRegister,
16876 $Vtmp3$$FloatRegister, $Vtmp4$$FloatRegister);
16877 %}
16878 ins_pipe( pipe_class_memory );
16879 %}
16880
16881 // ============================================================================
16882 // This name is KNOWN by the ADLC and cannot be changed.
16883 // The ADLC forces a 'TypeRawPtr::BOTTOM' output type
16884 // for this guy.
16885 instruct tlsLoadP(thread_RegP dst)
16886 %{
16887 match(Set dst (ThreadLocal));
16888
16889 ins_cost(0);
16890
16891 format %{ " -- \t// $dst=Thread::current(), empty" %}
16892
16893 size(0);
16894
16895 ins_encode( /*empty*/ );
16896
16897 ins_pipe(pipe_class_empty);
16898 %}
16899
16900 //----------PEEPHOLE RULES-----------------------------------------------------
16901 // These must follow all instruction definitions as they use the names
16902 // defined in the instructions definitions.
16903 //
16904 // peepmatch ( root_instr_name [preceding_instruction]* );
16905 //
16906 // peepconstraint %{
16907 // (instruction_number.operand_name relational_op instruction_number.operand_name
16908 // [, ...] );
16909 // // instruction numbers are zero-based using left to right order in peepmatch
16910 //
16911 // peepreplace ( instr_name ( [instruction_number.operand_name]* ) );
16912 // // provide an instruction_number.operand_name for each operand that appears
16913 // // in the replacement instruction's match rule
16914 //
16915 // ---------VM FLAGS---------------------------------------------------------
16916 //
16917 // All peephole optimizations can be turned off using -XX:-OptoPeephole
16918 //
16919 // Each peephole rule is given an identifying number starting with zero and
16920 // increasing by one in the order seen by the parser. An individual peephole
16921 // can be enabled, and all others disabled, by using -XX:OptoPeepholeAt=#
16922 // on the command-line.
16923 //
16924 // ---------CURRENT LIMITATIONS----------------------------------------------
16925 //
16926 // Only match adjacent instructions in same basic block
16927 // Only equality constraints
16928 // Only constraints between operands, not (0.dest_reg == RAX_enc)
16929 // Only one replacement instruction
16930 //
16931 // ---------EXAMPLE----------------------------------------------------------
16932 //
16933 // // pertinent parts of existing instructions in architecture description
16934 // instruct movI(iRegINoSp dst, iRegI src)
16935 // %{
16936 // match(Set dst (CopyI src));
16937 // %}
16938 //
16939 // instruct incI_iReg(iRegINoSp dst, immI1 src, rFlagsReg cr)
16940 // %{
16941 // match(Set dst (AddI dst src));
16942 // effect(KILL cr);
16943 // %}
16944 //
16945 // // Change (inc mov) to lea
16946 // peephole %{
16947 // // increment preceeded by register-register move
16948 // peepmatch ( incI_iReg movI );
16949 // // require that the destination register of the increment
16950 // // match the destination register of the move
16951 // peepconstraint ( 0.dst == 1.dst );
16952 // // construct a replacement instruction that sets
16953 // // the destination to ( move's source register + one )
16954 // peepreplace ( leaI_iReg_immI( 0.dst 1.src 0.src ) );
16955 // %}
16956 //
16957
16958 // Implementation no longer uses movX instructions since
16959 // machine-independent system no longer uses CopyX nodes.
16960 //
16961 // peephole
16962 // %{
16963 // peepmatch (incI_iReg movI);
16964 // peepconstraint (0.dst == 1.dst);
16965 // peepreplace (leaI_iReg_immI(0.dst 1.src 0.src));
16966 // %}
16967
16968 // peephole
16969 // %{
16970 // peepmatch (decI_iReg movI);
16971 // peepconstraint (0.dst == 1.dst);
16972 // peepreplace (leaI_iReg_immI(0.dst 1.src 0.src));
16973 // %}
16974
16975 // peephole
16976 // %{
16977 // peepmatch (addI_iReg_imm movI);
16978 // peepconstraint (0.dst == 1.dst);
16979 // peepreplace (leaI_iReg_immI(0.dst 1.src 0.src));
16980 // %}
16981
16982 // peephole
16983 // %{
16984 // peepmatch (incL_iReg movL);
16985 // peepconstraint (0.dst == 1.dst);
16986 // peepreplace (leaL_iReg_immL(0.dst 1.src 0.src));
16987 // %}
16988
16989 // peephole
16990 // %{
16991 // peepmatch (decL_iReg movL);
16992 // peepconstraint (0.dst == 1.dst);
16993 // peepreplace (leaL_iReg_immL(0.dst 1.src 0.src));
16994 // %}
16995
16996 // peephole
16997 // %{
16998 // peepmatch (addL_iReg_imm movL);
16999 // peepconstraint (0.dst == 1.dst);
17000 // peepreplace (leaL_iReg_immL(0.dst 1.src 0.src));
17001 // %}
17002
17003 // peephole
17004 // %{
17005 // peepmatch (addP_iReg_imm movP);
17006 // peepconstraint (0.dst == 1.dst);
17007 // peepreplace (leaP_iReg_imm(0.dst 1.src 0.src));
17008 // %}
17009
17010 // // Change load of spilled value to only a spill
17011 // instruct storeI(memory mem, iRegI src)
17012 // %{
17013 // match(Set mem (StoreI mem src));
17014 // %}
17015 //
17016 // instruct loadI(iRegINoSp dst, memory mem)
17017 // %{
17018 // match(Set dst (LoadI mem));
17019 // %}
17020 //
17021
17022 //----------SMARTSPILL RULES---------------------------------------------------
17023 // These must follow all instruction definitions as they use the names
17024 // defined in the instructions definitions.
17025
17026 // Local Variables:
17027 // mode: c++
17028 // End: