1 //
2 // Copyright (c) 2003, 2021, Oracle and/or its affiliates. All rights reserved.
3 // Copyright (c) 2014, 2021, Red Hat, Inc. All rights reserved.
4 // DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
5 //
6 // This code is free software; you can redistribute it and/or modify it
7 // under the terms of the GNU General Public License version 2 only, as
8 // published by the Free Software Foundation.
9 //
10 // This code is distributed in the hope that it will be useful, but WITHOUT
11 // ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
12 // FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
13 // version 2 for more details (a copy is included in the LICENSE file that
14 // accompanied this code).
15 //
16 // You should have received a copy of the GNU General Public License version
17 // 2 along with this work; if not, write to the Free Software Foundation,
18 // Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
19 //
20 // Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
21 // or visit www.oracle.com if you need additional information or have any
22 // questions.
23 //
24 //
25
26 // AArch64 Architecture Description File
27
28 //----------REGISTER DEFINITION BLOCK------------------------------------------
29 // This information is used by the matcher and the register allocator to
30 // describe individual registers and classes of registers within the target
31 // archtecture.
32
33 register %{
34 //----------Architecture Description Register Definitions----------------------
35 // General Registers
36 // "reg_def" name ( register save type, C convention save type,
37 // ideal register type, encoding );
38 // Register Save Types:
39 //
40 // NS = No-Save: The register allocator assumes that these registers
41 // can be used without saving upon entry to the method, &
42 // that they do not need to be saved at call sites.
43 //
44 // SOC = Save-On-Call: The register allocator assumes that these registers
45 // can be used without saving upon entry to the method,
46 // but that they must be saved at call sites.
47 //
48 // SOE = Save-On-Entry: The register allocator assumes that these registers
49 // must be saved before using them upon entry to the
50 // method, but they do not need to be saved at call
51 // sites.
52 //
53 // AS = Always-Save: The register allocator assumes that these registers
54 // must be saved before using them upon entry to the
55 // method, & that they must be saved at call sites.
56 //
57 // Ideal Register Type is used to determine how to save & restore a
58 // register. Op_RegI will get spilled with LoadI/StoreI, Op_RegP will get
59 // spilled with LoadP/StoreP. If the register supports both, use Op_RegI.
60 //
61 // The encoding number is the actual bit-pattern placed into the opcodes.
62
63 // We must define the 64 bit int registers in two 32 bit halves, the
64 // real lower register and a virtual upper half register. upper halves
65 // are used by the register allocator but are not actually supplied as
66 // operands to memory ops.
67 //
68 // follow the C1 compiler in making registers
69 //
70 // r0-r7,r10-r26 volatile (caller save)
71 // r27-r32 system (no save, no allocate)
72 // r8-r9 non-allocatable (so we can use them as scratch regs)
73 //
74 // as regards Java usage. we don't use any callee save registers
75 // because this makes it difficult to de-optimise a frame (see comment
76 // in x86 implementation of Deoptimization::unwind_callee_save_values)
77 //
78
79 // General Registers
80
81 reg_def R0 ( SOC, SOC, Op_RegI, 0, r0->as_VMReg() );
82 reg_def R0_H ( SOC, SOC, Op_RegI, 0, r0->as_VMReg()->next() );
83 reg_def R1 ( SOC, SOC, Op_RegI, 1, r1->as_VMReg() );
84 reg_def R1_H ( SOC, SOC, Op_RegI, 1, r1->as_VMReg()->next() );
85 reg_def R2 ( SOC, SOC, Op_RegI, 2, r2->as_VMReg() );
86 reg_def R2_H ( SOC, SOC, Op_RegI, 2, r2->as_VMReg()->next() );
87 reg_def R3 ( SOC, SOC, Op_RegI, 3, r3->as_VMReg() );
88 reg_def R3_H ( SOC, SOC, Op_RegI, 3, r3->as_VMReg()->next() );
89 reg_def R4 ( SOC, SOC, Op_RegI, 4, r4->as_VMReg() );
90 reg_def R4_H ( SOC, SOC, Op_RegI, 4, r4->as_VMReg()->next() );
91 reg_def R5 ( SOC, SOC, Op_RegI, 5, r5->as_VMReg() );
92 reg_def R5_H ( SOC, SOC, Op_RegI, 5, r5->as_VMReg()->next() );
93 reg_def R6 ( SOC, SOC, Op_RegI, 6, r6->as_VMReg() );
94 reg_def R6_H ( SOC, SOC, Op_RegI, 6, r6->as_VMReg()->next() );
95 reg_def R7 ( SOC, SOC, Op_RegI, 7, r7->as_VMReg() );
96 reg_def R7_H ( SOC, SOC, Op_RegI, 7, r7->as_VMReg()->next() );
97 reg_def R8 ( NS, SOC, Op_RegI, 8, r8->as_VMReg() ); // rscratch1, non-allocatable
98 reg_def R8_H ( NS, SOC, Op_RegI, 8, r8->as_VMReg()->next() );
99 reg_def R9 ( NS, SOC, Op_RegI, 9, r9->as_VMReg() ); // rscratch2, non-allocatable
100 reg_def R9_H ( NS, SOC, Op_RegI, 9, r9->as_VMReg()->next() );
101 reg_def R10 ( SOC, SOC, Op_RegI, 10, r10->as_VMReg() );
102 reg_def R10_H ( SOC, SOC, Op_RegI, 10, r10->as_VMReg()->next());
103 reg_def R11 ( SOC, SOC, Op_RegI, 11, r11->as_VMReg() );
104 reg_def R11_H ( SOC, SOC, Op_RegI, 11, r11->as_VMReg()->next());
105 reg_def R12 ( SOC, SOC, Op_RegI, 12, r12->as_VMReg() );
106 reg_def R12_H ( SOC, SOC, Op_RegI, 12, r12->as_VMReg()->next());
107 reg_def R13 ( SOC, SOC, Op_RegI, 13, r13->as_VMReg() );
108 reg_def R13_H ( SOC, SOC, Op_RegI, 13, r13->as_VMReg()->next());
109 reg_def R14 ( SOC, SOC, Op_RegI, 14, r14->as_VMReg() );
110 reg_def R14_H ( SOC, SOC, Op_RegI, 14, r14->as_VMReg()->next());
111 reg_def R15 ( SOC, SOC, Op_RegI, 15, r15->as_VMReg() );
112 reg_def R15_H ( SOC, SOC, Op_RegI, 15, r15->as_VMReg()->next());
113 reg_def R16 ( SOC, SOC, Op_RegI, 16, r16->as_VMReg() );
114 reg_def R16_H ( SOC, SOC, Op_RegI, 16, r16->as_VMReg()->next());
115 reg_def R17 ( SOC, SOC, Op_RegI, 17, r17->as_VMReg() );
116 reg_def R17_H ( SOC, SOC, Op_RegI, 17, r17->as_VMReg()->next());
117 reg_def R18 ( SOC, SOC, Op_RegI, 18, r18_tls->as_VMReg() );
118 reg_def R18_H ( SOC, SOC, Op_RegI, 18, r18_tls->as_VMReg()->next());
119 reg_def R19 ( SOC, SOE, Op_RegI, 19, r19->as_VMReg() );
120 reg_def R19_H ( SOC, SOE, Op_RegI, 19, r19->as_VMReg()->next());
121 reg_def R20 ( SOC, SOE, Op_RegI, 20, r20->as_VMReg() ); // caller esp
122 reg_def R20_H ( SOC, SOE, Op_RegI, 20, r20->as_VMReg()->next());
123 reg_def R21 ( SOC, SOE, Op_RegI, 21, r21->as_VMReg() );
124 reg_def R21_H ( SOC, SOE, Op_RegI, 21, r21->as_VMReg()->next());
125 reg_def R22 ( SOC, SOE, Op_RegI, 22, r22->as_VMReg() );
126 reg_def R22_H ( SOC, SOE, Op_RegI, 22, r22->as_VMReg()->next());
127 reg_def R23 ( SOC, SOE, Op_RegI, 23, r23->as_VMReg() );
128 reg_def R23_H ( SOC, SOE, Op_RegI, 23, r23->as_VMReg()->next());
129 reg_def R24 ( SOC, SOE, Op_RegI, 24, r24->as_VMReg() );
130 reg_def R24_H ( SOC, SOE, Op_RegI, 24, r24->as_VMReg()->next());
131 reg_def R25 ( SOC, SOE, Op_RegI, 25, r25->as_VMReg() );
132 reg_def R25_H ( SOC, SOE, Op_RegI, 25, r25->as_VMReg()->next());
133 reg_def R26 ( SOC, SOE, Op_RegI, 26, r26->as_VMReg() );
134 reg_def R26_H ( SOC, SOE, Op_RegI, 26, r26->as_VMReg()->next());
135 reg_def R27 ( SOC, SOE, Op_RegI, 27, r27->as_VMReg() ); // heapbase
136 reg_def R27_H ( SOC, SOE, Op_RegI, 27, r27->as_VMReg()->next());
137 reg_def R28 ( NS, SOE, Op_RegI, 28, r28->as_VMReg() ); // thread
138 reg_def R28_H ( NS, SOE, Op_RegI, 28, r28->as_VMReg()->next());
139 reg_def R29 ( NS, NS, Op_RegI, 29, r29->as_VMReg() ); // fp
140 reg_def R29_H ( NS, NS, Op_RegI, 29, r29->as_VMReg()->next());
141 reg_def R30 ( NS, NS, Op_RegI, 30, r30->as_VMReg() ); // lr
142 reg_def R30_H ( NS, NS, Op_RegI, 30, r30->as_VMReg()->next());
143 reg_def R31 ( NS, NS, Op_RegI, 31, r31_sp->as_VMReg() ); // sp
144 reg_def R31_H ( NS, NS, Op_RegI, 31, r31_sp->as_VMReg()->next());
145
146 // ----------------------------
147 // Float/Double/Vector Registers
148 // ----------------------------
149
150 // Double Registers
151
152 // The rules of ADL require that double registers be defined in pairs.
153 // Each pair must be two 32-bit values, but not necessarily a pair of
154 // single float registers. In each pair, ADLC-assigned register numbers
155 // must be adjacent, with the lower number even. Finally, when the
156 // CPU stores such a register pair to memory, the word associated with
157 // the lower ADLC-assigned number must be stored to the lower address.
158
159 // AArch64 has 32 floating-point registers. Each can store a vector of
160 // single or double precision floating-point values up to 8 * 32
161 // floats, 4 * 64 bit floats or 2 * 128 bit floats. We currently only
162 // use the first float or double element of the vector.
163
164 // for Java use float registers v0-v15 are always save on call whereas
165 // the platform ABI treats v8-v15 as callee save). float registers
166 // v16-v31 are SOC as per the platform spec
167
168 // For SVE vector registers, we simply extend vector register size to 8
169 // 'logical' slots. This is nominally 256 bits but it actually covers
170 // all possible 'physical' SVE vector register lengths from 128 ~ 2048
171 // bits. The 'physical' SVE vector register length is detected during
172 // startup, so the register allocator is able to identify the correct
173 // number of bytes needed for an SVE spill/unspill.
174 // Note that a vector register with 4 slots denotes a 128-bit NEON
175 // register allowing it to be distinguished from the corresponding SVE
176 // vector register when the SVE vector length is 128 bits.
177
178 reg_def V0 ( SOC, SOC, Op_RegF, 0, v0->as_VMReg() );
179 reg_def V0_H ( SOC, SOC, Op_RegF, 0, v0->as_VMReg()->next() );
180 reg_def V0_J ( SOC, SOC, Op_RegF, 0, v0->as_VMReg()->next(2) );
181 reg_def V0_K ( SOC, SOC, Op_RegF, 0, v0->as_VMReg()->next(3) );
182 reg_def V0_L ( SOC, SOC, Op_RegF, 0, v0->as_VMReg()->next(4) );
183 reg_def V0_M ( SOC, SOC, Op_RegF, 0, v0->as_VMReg()->next(5) );
184 reg_def V0_N ( SOC, SOC, Op_RegF, 0, v0->as_VMReg()->next(6) );
185 reg_def V0_O ( SOC, SOC, Op_RegF, 0, v0->as_VMReg()->next(7) );
186
187 reg_def V1 ( SOC, SOC, Op_RegF, 1, v1->as_VMReg() );
188 reg_def V1_H ( SOC, SOC, Op_RegF, 1, v1->as_VMReg()->next() );
189 reg_def V1_J ( SOC, SOC, Op_RegF, 1, v1->as_VMReg()->next(2) );
190 reg_def V1_K ( SOC, SOC, Op_RegF, 1, v1->as_VMReg()->next(3) );
191 reg_def V1_L ( SOC, SOC, Op_RegF, 1, v1->as_VMReg()->next(4) );
192 reg_def V1_M ( SOC, SOC, Op_RegF, 1, v1->as_VMReg()->next(5) );
193 reg_def V1_N ( SOC, SOC, Op_RegF, 1, v1->as_VMReg()->next(6) );
194 reg_def V1_O ( SOC, SOC, Op_RegF, 1, v1->as_VMReg()->next(7) );
195
196 reg_def V2 ( SOC, SOC, Op_RegF, 2, v2->as_VMReg() );
197 reg_def V2_H ( SOC, SOC, Op_RegF, 2, v2->as_VMReg()->next() );
198 reg_def V2_J ( SOC, SOC, Op_RegF, 2, v2->as_VMReg()->next(2) );
199 reg_def V2_K ( SOC, SOC, Op_RegF, 2, v2->as_VMReg()->next(3) );
200 reg_def V2_L ( SOC, SOC, Op_RegF, 2, v2->as_VMReg()->next(4) );
201 reg_def V2_M ( SOC, SOC, Op_RegF, 2, v2->as_VMReg()->next(5) );
202 reg_def V2_N ( SOC, SOC, Op_RegF, 2, v2->as_VMReg()->next(6) );
203 reg_def V2_O ( SOC, SOC, Op_RegF, 2, v2->as_VMReg()->next(7) );
204
205 reg_def V3 ( SOC, SOC, Op_RegF, 3, v3->as_VMReg() );
206 reg_def V3_H ( SOC, SOC, Op_RegF, 3, v3->as_VMReg()->next() );
207 reg_def V3_J ( SOC, SOC, Op_RegF, 3, v3->as_VMReg()->next(2) );
208 reg_def V3_K ( SOC, SOC, Op_RegF, 3, v3->as_VMReg()->next(3) );
209 reg_def V3_L ( SOC, SOC, Op_RegF, 3, v3->as_VMReg()->next(4) );
210 reg_def V3_M ( SOC, SOC, Op_RegF, 3, v3->as_VMReg()->next(5) );
211 reg_def V3_N ( SOC, SOC, Op_RegF, 3, v3->as_VMReg()->next(6) );
212 reg_def V3_O ( SOC, SOC, Op_RegF, 3, v3->as_VMReg()->next(7) );
213
214 reg_def V4 ( SOC, SOC, Op_RegF, 4, v4->as_VMReg() );
215 reg_def V4_H ( SOC, SOC, Op_RegF, 4, v4->as_VMReg()->next() );
216 reg_def V4_J ( SOC, SOC, Op_RegF, 4, v4->as_VMReg()->next(2) );
217 reg_def V4_K ( SOC, SOC, Op_RegF, 4, v4->as_VMReg()->next(3) );
218 reg_def V4_L ( SOC, SOC, Op_RegF, 4, v4->as_VMReg()->next(4) );
219 reg_def V4_M ( SOC, SOC, Op_RegF, 4, v4->as_VMReg()->next(5) );
220 reg_def V4_N ( SOC, SOC, Op_RegF, 4, v4->as_VMReg()->next(6) );
221 reg_def V4_O ( SOC, SOC, Op_RegF, 4, v4->as_VMReg()->next(7) );
222
223 reg_def V5 ( SOC, SOC, Op_RegF, 5, v5->as_VMReg() );
224 reg_def V5_H ( SOC, SOC, Op_RegF, 5, v5->as_VMReg()->next() );
225 reg_def V5_J ( SOC, SOC, Op_RegF, 5, v5->as_VMReg()->next(2) );
226 reg_def V5_K ( SOC, SOC, Op_RegF, 5, v5->as_VMReg()->next(3) );
227 reg_def V5_L ( SOC, SOC, Op_RegF, 5, v5->as_VMReg()->next(4) );
228 reg_def V5_M ( SOC, SOC, Op_RegF, 5, v5->as_VMReg()->next(5) );
229 reg_def V5_N ( SOC, SOC, Op_RegF, 5, v5->as_VMReg()->next(6) );
230 reg_def V5_O ( SOC, SOC, Op_RegF, 5, v5->as_VMReg()->next(7) );
231
232 reg_def V6 ( SOC, SOC, Op_RegF, 6, v6->as_VMReg() );
233 reg_def V6_H ( SOC, SOC, Op_RegF, 6, v6->as_VMReg()->next() );
234 reg_def V6_J ( SOC, SOC, Op_RegF, 6, v6->as_VMReg()->next(2) );
235 reg_def V6_K ( SOC, SOC, Op_RegF, 6, v6->as_VMReg()->next(3) );
236 reg_def V6_L ( SOC, SOC, Op_RegF, 6, v6->as_VMReg()->next(4) );
237 reg_def V6_M ( SOC, SOC, Op_RegF, 6, v6->as_VMReg()->next(5) );
238 reg_def V6_N ( SOC, SOC, Op_RegF, 6, v6->as_VMReg()->next(6) );
239 reg_def V6_O ( SOC, SOC, Op_RegF, 6, v6->as_VMReg()->next(7) );
240
241 reg_def V7 ( SOC, SOC, Op_RegF, 7, v7->as_VMReg() );
242 reg_def V7_H ( SOC, SOC, Op_RegF, 7, v7->as_VMReg()->next() );
243 reg_def V7_J ( SOC, SOC, Op_RegF, 7, v7->as_VMReg()->next(2) );
244 reg_def V7_K ( SOC, SOC, Op_RegF, 7, v7->as_VMReg()->next(3) );
245 reg_def V7_L ( SOC, SOC, Op_RegF, 7, v7->as_VMReg()->next(4) );
246 reg_def V7_M ( SOC, SOC, Op_RegF, 7, v7->as_VMReg()->next(5) );
247 reg_def V7_N ( SOC, SOC, Op_RegF, 7, v7->as_VMReg()->next(6) );
248 reg_def V7_O ( SOC, SOC, Op_RegF, 7, v7->as_VMReg()->next(7) );
249
250 reg_def V8 ( SOC, SOE, Op_RegF, 8, v8->as_VMReg() );
251 reg_def V8_H ( SOC, SOE, Op_RegF, 8, v8->as_VMReg()->next() );
252 reg_def V8_J ( SOC, SOC, Op_RegF, 8, v8->as_VMReg()->next(2) );
253 reg_def V8_K ( SOC, SOC, Op_RegF, 8, v8->as_VMReg()->next(3) );
254 reg_def V8_L ( SOC, SOC, Op_RegF, 8, v8->as_VMReg()->next(4) );
255 reg_def V8_M ( SOC, SOC, Op_RegF, 8, v8->as_VMReg()->next(5) );
256 reg_def V8_N ( SOC, SOC, Op_RegF, 8, v8->as_VMReg()->next(6) );
257 reg_def V8_O ( SOC, SOC, Op_RegF, 8, v8->as_VMReg()->next(7) );
258
259 reg_def V9 ( SOC, SOE, Op_RegF, 9, v9->as_VMReg() );
260 reg_def V9_H ( SOC, SOE, Op_RegF, 9, v9->as_VMReg()->next() );
261 reg_def V9_J ( SOC, SOC, Op_RegF, 9, v9->as_VMReg()->next(2) );
262 reg_def V9_K ( SOC, SOC, Op_RegF, 9, v9->as_VMReg()->next(3) );
263 reg_def V9_L ( SOC, SOC, Op_RegF, 9, v9->as_VMReg()->next(4) );
264 reg_def V9_M ( SOC, SOC, Op_RegF, 9, v9->as_VMReg()->next(5) );
265 reg_def V9_N ( SOC, SOC, Op_RegF, 9, v9->as_VMReg()->next(6) );
266 reg_def V9_O ( SOC, SOC, Op_RegF, 9, v9->as_VMReg()->next(7) );
267
268 reg_def V10 ( SOC, SOE, Op_RegF, 10, v10->as_VMReg() );
269 reg_def V10_H ( SOC, SOE, Op_RegF, 10, v10->as_VMReg()->next() );
270 reg_def V10_J ( SOC, SOC, Op_RegF, 10, v10->as_VMReg()->next(2) );
271 reg_def V10_K ( SOC, SOC, Op_RegF, 10, v10->as_VMReg()->next(3) );
272 reg_def V10_L ( SOC, SOC, Op_RegF, 10, v10->as_VMReg()->next(4) );
273 reg_def V10_M ( SOC, SOC, Op_RegF, 10, v10->as_VMReg()->next(5) );
274 reg_def V10_N ( SOC, SOC, Op_RegF, 10, v10->as_VMReg()->next(6) );
275 reg_def V10_O ( SOC, SOC, Op_RegF, 10, v10->as_VMReg()->next(7) );
276
277 reg_def V11 ( SOC, SOE, Op_RegF, 11, v11->as_VMReg() );
278 reg_def V11_H ( SOC, SOE, Op_RegF, 11, v11->as_VMReg()->next() );
279 reg_def V11_J ( SOC, SOC, Op_RegF, 11, v11->as_VMReg()->next(2) );
280 reg_def V11_K ( SOC, SOC, Op_RegF, 11, v11->as_VMReg()->next(3) );
281 reg_def V11_L ( SOC, SOC, Op_RegF, 11, v11->as_VMReg()->next(4) );
282 reg_def V11_M ( SOC, SOC, Op_RegF, 11, v11->as_VMReg()->next(5) );
283 reg_def V11_N ( SOC, SOC, Op_RegF, 11, v11->as_VMReg()->next(6) );
284 reg_def V11_O ( SOC, SOC, Op_RegF, 11, v11->as_VMReg()->next(7) );
285
286 reg_def V12 ( SOC, SOE, Op_RegF, 12, v12->as_VMReg() );
287 reg_def V12_H ( SOC, SOE, Op_RegF, 12, v12->as_VMReg()->next() );
288 reg_def V12_J ( SOC, SOC, Op_RegF, 12, v12->as_VMReg()->next(2) );
289 reg_def V12_K ( SOC, SOC, Op_RegF, 12, v12->as_VMReg()->next(3) );
290 reg_def V12_L ( SOC, SOC, Op_RegF, 12, v12->as_VMReg()->next(4) );
291 reg_def V12_M ( SOC, SOC, Op_RegF, 12, v12->as_VMReg()->next(5) );
292 reg_def V12_N ( SOC, SOC, Op_RegF, 12, v12->as_VMReg()->next(6) );
293 reg_def V12_O ( SOC, SOC, Op_RegF, 12, v12->as_VMReg()->next(7) );
294
295 reg_def V13 ( SOC, SOE, Op_RegF, 13, v13->as_VMReg() );
296 reg_def V13_H ( SOC, SOE, Op_RegF, 13, v13->as_VMReg()->next() );
297 reg_def V13_J ( SOC, SOC, Op_RegF, 13, v13->as_VMReg()->next(2) );
298 reg_def V13_K ( SOC, SOC, Op_RegF, 13, v13->as_VMReg()->next(3) );
299 reg_def V13_L ( SOC, SOC, Op_RegF, 13, v13->as_VMReg()->next(4) );
300 reg_def V13_M ( SOC, SOC, Op_RegF, 13, v13->as_VMReg()->next(5) );
301 reg_def V13_N ( SOC, SOC, Op_RegF, 13, v13->as_VMReg()->next(6) );
302 reg_def V13_O ( SOC, SOC, Op_RegF, 13, v13->as_VMReg()->next(7) );
303
304 reg_def V14 ( SOC, SOE, Op_RegF, 14, v14->as_VMReg() );
305 reg_def V14_H ( SOC, SOE, Op_RegF, 14, v14->as_VMReg()->next() );
306 reg_def V14_J ( SOC, SOC, Op_RegF, 14, v14->as_VMReg()->next(2) );
307 reg_def V14_K ( SOC, SOC, Op_RegF, 14, v14->as_VMReg()->next(3) );
308 reg_def V14_L ( SOC, SOC, Op_RegF, 14, v14->as_VMReg()->next(4) );
309 reg_def V14_M ( SOC, SOC, Op_RegF, 14, v14->as_VMReg()->next(5) );
310 reg_def V14_N ( SOC, SOC, Op_RegF, 14, v14->as_VMReg()->next(6) );
311 reg_def V14_O ( SOC, SOC, Op_RegF, 14, v14->as_VMReg()->next(7) );
312
313 reg_def V15 ( SOC, SOE, Op_RegF, 15, v15->as_VMReg() );
314 reg_def V15_H ( SOC, SOE, Op_RegF, 15, v15->as_VMReg()->next() );
315 reg_def V15_J ( SOC, SOC, Op_RegF, 15, v15->as_VMReg()->next(2) );
316 reg_def V15_K ( SOC, SOC, Op_RegF, 15, v15->as_VMReg()->next(3) );
317 reg_def V15_L ( SOC, SOC, Op_RegF, 15, v15->as_VMReg()->next(4) );
318 reg_def V15_M ( SOC, SOC, Op_RegF, 15, v15->as_VMReg()->next(5) );
319 reg_def V15_N ( SOC, SOC, Op_RegF, 15, v15->as_VMReg()->next(6) );
320 reg_def V15_O ( SOC, SOC, Op_RegF, 15, v15->as_VMReg()->next(7) );
321
322 reg_def V16 ( SOC, SOC, Op_RegF, 16, v16->as_VMReg() );
323 reg_def V16_H ( SOC, SOC, Op_RegF, 16, v16->as_VMReg()->next() );
324 reg_def V16_J ( SOC, SOC, Op_RegF, 16, v16->as_VMReg()->next(2) );
325 reg_def V16_K ( SOC, SOC, Op_RegF, 16, v16->as_VMReg()->next(3) );
326 reg_def V16_L ( SOC, SOC, Op_RegF, 16, v16->as_VMReg()->next(4) );
327 reg_def V16_M ( SOC, SOC, Op_RegF, 16, v16->as_VMReg()->next(5) );
328 reg_def V16_N ( SOC, SOC, Op_RegF, 16, v16->as_VMReg()->next(6) );
329 reg_def V16_O ( SOC, SOC, Op_RegF, 16, v16->as_VMReg()->next(7) );
330
331 reg_def V17 ( SOC, SOC, Op_RegF, 17, v17->as_VMReg() );
332 reg_def V17_H ( SOC, SOC, Op_RegF, 17, v17->as_VMReg()->next() );
333 reg_def V17_J ( SOC, SOC, Op_RegF, 17, v17->as_VMReg()->next(2) );
334 reg_def V17_K ( SOC, SOC, Op_RegF, 17, v17->as_VMReg()->next(3) );
335 reg_def V17_L ( SOC, SOC, Op_RegF, 17, v17->as_VMReg()->next(4) );
336 reg_def V17_M ( SOC, SOC, Op_RegF, 17, v17->as_VMReg()->next(5) );
337 reg_def V17_N ( SOC, SOC, Op_RegF, 17, v17->as_VMReg()->next(6) );
338 reg_def V17_O ( SOC, SOC, Op_RegF, 17, v17->as_VMReg()->next(7) );
339
340 reg_def V18 ( SOC, SOC, Op_RegF, 18, v18->as_VMReg() );
341 reg_def V18_H ( SOC, SOC, Op_RegF, 18, v18->as_VMReg()->next() );
342 reg_def V18_J ( SOC, SOC, Op_RegF, 18, v18->as_VMReg()->next(2) );
343 reg_def V18_K ( SOC, SOC, Op_RegF, 18, v18->as_VMReg()->next(3) );
344 reg_def V18_L ( SOC, SOC, Op_RegF, 18, v18->as_VMReg()->next(4) );
345 reg_def V18_M ( SOC, SOC, Op_RegF, 18, v18->as_VMReg()->next(5) );
346 reg_def V18_N ( SOC, SOC, Op_RegF, 18, v18->as_VMReg()->next(6) );
347 reg_def V18_O ( SOC, SOC, Op_RegF, 18, v18->as_VMReg()->next(7) );
348
349 reg_def V19 ( SOC, SOC, Op_RegF, 19, v19->as_VMReg() );
350 reg_def V19_H ( SOC, SOC, Op_RegF, 19, v19->as_VMReg()->next() );
351 reg_def V19_J ( SOC, SOC, Op_RegF, 19, v19->as_VMReg()->next(2) );
352 reg_def V19_K ( SOC, SOC, Op_RegF, 19, v19->as_VMReg()->next(3) );
353 reg_def V19_L ( SOC, SOC, Op_RegF, 19, v19->as_VMReg()->next(4) );
354 reg_def V19_M ( SOC, SOC, Op_RegF, 19, v19->as_VMReg()->next(5) );
355 reg_def V19_N ( SOC, SOC, Op_RegF, 19, v19->as_VMReg()->next(6) );
356 reg_def V19_O ( SOC, SOC, Op_RegF, 19, v19->as_VMReg()->next(7) );
357
358 reg_def V20 ( SOC, SOC, Op_RegF, 20, v20->as_VMReg() );
359 reg_def V20_H ( SOC, SOC, Op_RegF, 20, v20->as_VMReg()->next() );
360 reg_def V20_J ( SOC, SOC, Op_RegF, 20, v20->as_VMReg()->next(2) );
361 reg_def V20_K ( SOC, SOC, Op_RegF, 20, v20->as_VMReg()->next(3) );
362 reg_def V20_L ( SOC, SOC, Op_RegF, 20, v20->as_VMReg()->next(4) );
363 reg_def V20_M ( SOC, SOC, Op_RegF, 20, v20->as_VMReg()->next(5) );
364 reg_def V20_N ( SOC, SOC, Op_RegF, 20, v20->as_VMReg()->next(6) );
365 reg_def V20_O ( SOC, SOC, Op_RegF, 20, v20->as_VMReg()->next(7) );
366
367 reg_def V21 ( SOC, SOC, Op_RegF, 21, v21->as_VMReg() );
368 reg_def V21_H ( SOC, SOC, Op_RegF, 21, v21->as_VMReg()->next() );
369 reg_def V21_J ( SOC, SOC, Op_RegF, 21, v21->as_VMReg()->next(2) );
370 reg_def V21_K ( SOC, SOC, Op_RegF, 21, v21->as_VMReg()->next(3) );
371 reg_def V21_L ( SOC, SOC, Op_RegF, 21, v21->as_VMReg()->next(4) );
372 reg_def V21_M ( SOC, SOC, Op_RegF, 21, v21->as_VMReg()->next(5) );
373 reg_def V21_N ( SOC, SOC, Op_RegF, 21, v21->as_VMReg()->next(6) );
374 reg_def V21_O ( SOC, SOC, Op_RegF, 21, v21->as_VMReg()->next(7) );
375
376 reg_def V22 ( SOC, SOC, Op_RegF, 22, v22->as_VMReg() );
377 reg_def V22_H ( SOC, SOC, Op_RegF, 22, v22->as_VMReg()->next() );
378 reg_def V22_J ( SOC, SOC, Op_RegF, 22, v22->as_VMReg()->next(2) );
379 reg_def V22_K ( SOC, SOC, Op_RegF, 22, v22->as_VMReg()->next(3) );
380 reg_def V22_L ( SOC, SOC, Op_RegF, 22, v22->as_VMReg()->next(4) );
381 reg_def V22_M ( SOC, SOC, Op_RegF, 22, v22->as_VMReg()->next(5) );
382 reg_def V22_N ( SOC, SOC, Op_RegF, 22, v22->as_VMReg()->next(6) );
383 reg_def V22_O ( SOC, SOC, Op_RegF, 22, v22->as_VMReg()->next(7) );
384
385 reg_def V23 ( SOC, SOC, Op_RegF, 23, v23->as_VMReg() );
386 reg_def V23_H ( SOC, SOC, Op_RegF, 23, v23->as_VMReg()->next() );
387 reg_def V23_J ( SOC, SOC, Op_RegF, 23, v23->as_VMReg()->next(2) );
388 reg_def V23_K ( SOC, SOC, Op_RegF, 23, v23->as_VMReg()->next(3) );
389 reg_def V23_L ( SOC, SOC, Op_RegF, 23, v23->as_VMReg()->next(4) );
390 reg_def V23_M ( SOC, SOC, Op_RegF, 23, v23->as_VMReg()->next(5) );
391 reg_def V23_N ( SOC, SOC, Op_RegF, 23, v23->as_VMReg()->next(6) );
392 reg_def V23_O ( SOC, SOC, Op_RegF, 23, v23->as_VMReg()->next(7) );
393
394 reg_def V24 ( SOC, SOC, Op_RegF, 24, v24->as_VMReg() );
395 reg_def V24_H ( SOC, SOC, Op_RegF, 24, v24->as_VMReg()->next() );
396 reg_def V24_J ( SOC, SOC, Op_RegF, 24, v24->as_VMReg()->next(2) );
397 reg_def V24_K ( SOC, SOC, Op_RegF, 24, v24->as_VMReg()->next(3) );
398 reg_def V24_L ( SOC, SOC, Op_RegF, 24, v24->as_VMReg()->next(4) );
399 reg_def V24_M ( SOC, SOC, Op_RegF, 24, v24->as_VMReg()->next(5) );
400 reg_def V24_N ( SOC, SOC, Op_RegF, 24, v24->as_VMReg()->next(6) );
401 reg_def V24_O ( SOC, SOC, Op_RegF, 24, v24->as_VMReg()->next(7) );
402
403 reg_def V25 ( SOC, SOC, Op_RegF, 25, v25->as_VMReg() );
404 reg_def V25_H ( SOC, SOC, Op_RegF, 25, v25->as_VMReg()->next() );
405 reg_def V25_J ( SOC, SOC, Op_RegF, 25, v25->as_VMReg()->next(2) );
406 reg_def V25_K ( SOC, SOC, Op_RegF, 25, v25->as_VMReg()->next(3) );
407 reg_def V25_L ( SOC, SOC, Op_RegF, 25, v25->as_VMReg()->next(4) );
408 reg_def V25_M ( SOC, SOC, Op_RegF, 25, v25->as_VMReg()->next(5) );
409 reg_def V25_N ( SOC, SOC, Op_RegF, 25, v25->as_VMReg()->next(6) );
410 reg_def V25_O ( SOC, SOC, Op_RegF, 25, v25->as_VMReg()->next(7) );
411
412 reg_def V26 ( SOC, SOC, Op_RegF, 26, v26->as_VMReg() );
413 reg_def V26_H ( SOC, SOC, Op_RegF, 26, v26->as_VMReg()->next() );
414 reg_def V26_J ( SOC, SOC, Op_RegF, 26, v26->as_VMReg()->next(2) );
415 reg_def V26_K ( SOC, SOC, Op_RegF, 26, v26->as_VMReg()->next(3) );
416 reg_def V26_L ( SOC, SOC, Op_RegF, 26, v26->as_VMReg()->next(4) );
417 reg_def V26_M ( SOC, SOC, Op_RegF, 26, v26->as_VMReg()->next(5) );
418 reg_def V26_N ( SOC, SOC, Op_RegF, 26, v26->as_VMReg()->next(6) );
419 reg_def V26_O ( SOC, SOC, Op_RegF, 26, v26->as_VMReg()->next(7) );
420
421 reg_def V27 ( SOC, SOC, Op_RegF, 27, v27->as_VMReg() );
422 reg_def V27_H ( SOC, SOC, Op_RegF, 27, v27->as_VMReg()->next() );
423 reg_def V27_J ( SOC, SOC, Op_RegF, 27, v27->as_VMReg()->next(2) );
424 reg_def V27_K ( SOC, SOC, Op_RegF, 27, v27->as_VMReg()->next(3) );
425 reg_def V27_L ( SOC, SOC, Op_RegF, 27, v27->as_VMReg()->next(4) );
426 reg_def V27_M ( SOC, SOC, Op_RegF, 27, v27->as_VMReg()->next(5) );
427 reg_def V27_N ( SOC, SOC, Op_RegF, 27, v27->as_VMReg()->next(6) );
428 reg_def V27_O ( SOC, SOC, Op_RegF, 27, v27->as_VMReg()->next(7) );
429
430 reg_def V28 ( SOC, SOC, Op_RegF, 28, v28->as_VMReg() );
431 reg_def V28_H ( SOC, SOC, Op_RegF, 28, v28->as_VMReg()->next() );
432 reg_def V28_J ( SOC, SOC, Op_RegF, 28, v28->as_VMReg()->next(2) );
433 reg_def V28_K ( SOC, SOC, Op_RegF, 28, v28->as_VMReg()->next(3) );
434 reg_def V28_L ( SOC, SOC, Op_RegF, 28, v28->as_VMReg()->next(4) );
435 reg_def V28_M ( SOC, SOC, Op_RegF, 28, v28->as_VMReg()->next(5) );
436 reg_def V28_N ( SOC, SOC, Op_RegF, 28, v28->as_VMReg()->next(6) );
437 reg_def V28_O ( SOC, SOC, Op_RegF, 28, v28->as_VMReg()->next(7) );
438
439 reg_def V29 ( SOC, SOC, Op_RegF, 29, v29->as_VMReg() );
440 reg_def V29_H ( SOC, SOC, Op_RegF, 29, v29->as_VMReg()->next() );
441 reg_def V29_J ( SOC, SOC, Op_RegF, 29, v29->as_VMReg()->next(2) );
442 reg_def V29_K ( SOC, SOC, Op_RegF, 29, v29->as_VMReg()->next(3) );
443 reg_def V29_L ( SOC, SOC, Op_RegF, 29, v29->as_VMReg()->next(4) );
444 reg_def V29_M ( SOC, SOC, Op_RegF, 29, v29->as_VMReg()->next(5) );
445 reg_def V29_N ( SOC, SOC, Op_RegF, 29, v29->as_VMReg()->next(6) );
446 reg_def V29_O ( SOC, SOC, Op_RegF, 29, v29->as_VMReg()->next(7) );
447
448 reg_def V30 ( SOC, SOC, Op_RegF, 30, v30->as_VMReg() );
449 reg_def V30_H ( SOC, SOC, Op_RegF, 30, v30->as_VMReg()->next() );
450 reg_def V30_J ( SOC, SOC, Op_RegF, 30, v30->as_VMReg()->next(2) );
451 reg_def V30_K ( SOC, SOC, Op_RegF, 30, v30->as_VMReg()->next(3) );
452 reg_def V30_L ( SOC, SOC, Op_RegF, 30, v30->as_VMReg()->next(4) );
453 reg_def V30_M ( SOC, SOC, Op_RegF, 30, v30->as_VMReg()->next(5) );
454 reg_def V30_N ( SOC, SOC, Op_RegF, 30, v30->as_VMReg()->next(6) );
455 reg_def V30_O ( SOC, SOC, Op_RegF, 30, v30->as_VMReg()->next(7) );
456
457 reg_def V31 ( SOC, SOC, Op_RegF, 31, v31->as_VMReg() );
458 reg_def V31_H ( SOC, SOC, Op_RegF, 31, v31->as_VMReg()->next() );
459 reg_def V31_J ( SOC, SOC, Op_RegF, 31, v31->as_VMReg()->next(2) );
460 reg_def V31_K ( SOC, SOC, Op_RegF, 31, v31->as_VMReg()->next(3) );
461 reg_def V31_L ( SOC, SOC, Op_RegF, 31, v31->as_VMReg()->next(4) );
462 reg_def V31_M ( SOC, SOC, Op_RegF, 31, v31->as_VMReg()->next(5) );
463 reg_def V31_N ( SOC, SOC, Op_RegF, 31, v31->as_VMReg()->next(6) );
464 reg_def V31_O ( SOC, SOC, Op_RegF, 31, v31->as_VMReg()->next(7) );
465
466
467 // ----------------------------
468 // SVE Predicate Registers
469 // ----------------------------
470 reg_def P0 (SOC, SOC, Op_RegVectMask, 0, p0->as_VMReg());
471 reg_def P1 (SOC, SOC, Op_RegVectMask, 1, p1->as_VMReg());
472 reg_def P2 (SOC, SOC, Op_RegVectMask, 2, p2->as_VMReg());
473 reg_def P3 (SOC, SOC, Op_RegVectMask, 3, p3->as_VMReg());
474 reg_def P4 (SOC, SOC, Op_RegVectMask, 4, p4->as_VMReg());
475 reg_def P5 (SOC, SOC, Op_RegVectMask, 5, p5->as_VMReg());
476 reg_def P6 (SOC, SOC, Op_RegVectMask, 6, p6->as_VMReg());
477 reg_def P7 (SOC, SOC, Op_RegVectMask, 7, p7->as_VMReg());
478 reg_def P8 (SOC, SOC, Op_RegVectMask, 8, p8->as_VMReg());
479 reg_def P9 (SOC, SOC, Op_RegVectMask, 9, p9->as_VMReg());
480 reg_def P10 (SOC, SOC, Op_RegVectMask, 10, p10->as_VMReg());
481 reg_def P11 (SOC, SOC, Op_RegVectMask, 11, p11->as_VMReg());
482 reg_def P12 (SOC, SOC, Op_RegVectMask, 12, p12->as_VMReg());
483 reg_def P13 (SOC, SOC, Op_RegVectMask, 13, p13->as_VMReg());
484 reg_def P14 (SOC, SOC, Op_RegVectMask, 14, p14->as_VMReg());
485 reg_def P15 (SOC, SOC, Op_RegVectMask, 15, p15->as_VMReg());
486
487 // ----------------------------
488 // Special Registers
489 // ----------------------------
490
491 // the AArch64 CSPR status flag register is not directly acessible as
492 // instruction operand. the FPSR status flag register is a system
493 // register which can be written/read using MSR/MRS but again does not
494 // appear as an operand (a code identifying the FSPR occurs as an
495 // immediate value in the instruction).
496
497 reg_def RFLAGS(SOC, SOC, 0, 32, VMRegImpl::Bad());
498
499 // Specify priority of register selection within phases of register
500 // allocation. Highest priority is first. A useful heuristic is to
501 // give registers a low priority when they are required by machine
502 // instructions, like EAX and EDX on I486, and choose no-save registers
503 // before save-on-call, & save-on-call before save-on-entry. Registers
504 // which participate in fixed calling sequences should come last.
505 // Registers which are used as pairs must fall on an even boundary.
506
507 alloc_class chunk0(
508 // volatiles
509 R10, R10_H,
510 R11, R11_H,
511 R12, R12_H,
512 R13, R13_H,
513 R14, R14_H,
514 R15, R15_H,
515 R16, R16_H,
516 R17, R17_H,
517 R18, R18_H,
518
519 // arg registers
520 R0, R0_H,
521 R1, R1_H,
522 R2, R2_H,
523 R3, R3_H,
524 R4, R4_H,
525 R5, R5_H,
526 R6, R6_H,
527 R7, R7_H,
528
529 // non-volatiles
530 R19, R19_H,
531 R20, R20_H,
532 R21, R21_H,
533 R22, R22_H,
534 R23, R23_H,
535 R24, R24_H,
536 R25, R25_H,
537 R26, R26_H,
538
539 // non-allocatable registers
540
541 R27, R27_H, // heapbase
542 R28, R28_H, // thread
543 R29, R29_H, // fp
544 R30, R30_H, // lr
545 R31, R31_H, // sp
546 R8, R8_H, // rscratch1
547 R9, R9_H, // rscratch2
548 );
549
550 alloc_class chunk1(
551
552 // no save
553 V16, V16_H, V16_J, V16_K, V16_L, V16_M, V16_N, V16_O,
554 V17, V17_H, V17_J, V17_K, V17_L, V17_M, V17_N, V17_O,
555 V18, V18_H, V18_J, V18_K, V18_L, V18_M, V18_N, V18_O,
556 V19, V19_H, V19_J, V19_K, V19_L, V19_M, V19_N, V19_O,
557 V20, V20_H, V20_J, V20_K, V20_L, V20_M, V20_N, V20_O,
558 V21, V21_H, V21_J, V21_K, V21_L, V21_M, V21_N, V21_O,
559 V22, V22_H, V22_J, V22_K, V22_L, V22_M, V22_N, V22_O,
560 V23, V23_H, V23_J, V23_K, V23_L, V23_M, V23_N, V23_O,
561 V24, V24_H, V24_J, V24_K, V24_L, V24_M, V24_N, V24_O,
562 V25, V25_H, V25_J, V25_K, V25_L, V25_M, V25_N, V25_O,
563 V26, V26_H, V26_J, V26_K, V26_L, V26_M, V26_N, V26_O,
564 V27, V27_H, V27_J, V27_K, V27_L, V27_M, V27_N, V27_O,
565 V28, V28_H, V28_J, V28_K, V28_L, V28_M, V28_N, V28_O,
566 V29, V29_H, V29_J, V29_K, V29_L, V29_M, V29_N, V29_O,
567 V30, V30_H, V30_J, V30_K, V30_L, V30_M, V30_N, V30_O,
568 V31, V31_H, V31_J, V31_K, V31_L, V31_M, V31_N, V31_O,
569
570 // arg registers
571 V0, V0_H, V0_J, V0_K, V0_L, V0_M, V0_N, V0_O,
572 V1, V1_H, V1_J, V1_K, V1_L, V1_M, V1_N, V1_O,
573 V2, V2_H, V2_J, V2_K, V2_L, V2_M, V2_N, V2_O,
574 V3, V3_H, V3_J, V3_K, V3_L, V3_M, V3_N, V3_O,
575 V4, V4_H, V4_J, V4_K, V4_L, V4_M, V4_N, V4_O,
576 V5, V5_H, V5_J, V5_K, V5_L, V5_M, V5_N, V5_O,
577 V6, V6_H, V6_J, V6_K, V6_L, V6_M, V6_N, V6_O,
578 V7, V7_H, V7_J, V7_K, V7_L, V7_M, V7_N, V7_O,
579
580 // non-volatiles
581 V8, V8_H, V8_J, V8_K, V8_L, V8_M, V8_N, V8_O,
582 V9, V9_H, V9_J, V9_K, V9_L, V9_M, V9_N, V9_O,
583 V10, V10_H, V10_J, V10_K, V10_L, V10_M, V10_N, V10_O,
584 V11, V11_H, V11_J, V11_K, V11_L, V11_M, V11_N, V11_O,
585 V12, V12_H, V12_J, V12_K, V12_L, V12_M, V12_N, V12_O,
586 V13, V13_H, V13_J, V13_K, V13_L, V13_M, V13_N, V13_O,
587 V14, V14_H, V14_J, V14_K, V14_L, V14_M, V14_N, V14_O,
588 V15, V15_H, V15_J, V15_K, V15_L, V15_M, V15_N, V15_O,
589 );
590
591 alloc_class chunk2 (
592 P0,
593 P1,
594 P2,
595 P3,
596 P4,
597 P5,
598 P6,
599 P7,
600
601 P8,
602 P9,
603 P10,
604 P11,
605 P12,
606 P13,
607 P14,
608 P15,
609 );
610
611 alloc_class chunk3(RFLAGS);
612
613 //----------Architecture Description Register Classes--------------------------
614 // Several register classes are automatically defined based upon information in
615 // this architecture description.
616 // 1) reg_class inline_cache_reg ( /* as def'd in frame section */ )
617 // 2) reg_class stack_slots( /* one chunk of stack-based "registers" */ )
618 //
619
620 // Class for all 32 bit general purpose registers
621 reg_class all_reg32(
622 R0,
623 R1,
624 R2,
625 R3,
626 R4,
627 R5,
628 R6,
629 R7,
630 R10,
631 R11,
632 R12,
633 R13,
634 R14,
635 R15,
636 R16,
637 R17,
638 R18,
639 R19,
640 R20,
641 R21,
642 R22,
643 R23,
644 R24,
645 R25,
646 R26,
647 R27,
648 R28,
649 R29,
650 R30,
651 R31
652 );
653
654
655 // Class for all 32 bit integer registers (excluding SP which
656 // will never be used as an integer register)
657 reg_class any_reg32 %{
658 return _ANY_REG32_mask;
659 %}
660
661 // Singleton class for R0 int register
662 reg_class int_r0_reg(R0);
663
664 // Singleton class for R2 int register
665 reg_class int_r2_reg(R2);
666
667 // Singleton class for R3 int register
668 reg_class int_r3_reg(R3);
669
670 // Singleton class for R4 int register
671 reg_class int_r4_reg(R4);
672
673 // Singleton class for R31 int register
674 reg_class int_r31_reg(R31);
675
676 // Class for all 64 bit general purpose registers
677 reg_class all_reg(
678 R0, R0_H,
679 R1, R1_H,
680 R2, R2_H,
681 R3, R3_H,
682 R4, R4_H,
683 R5, R5_H,
684 R6, R6_H,
685 R7, R7_H,
686 R10, R10_H,
687 R11, R11_H,
688 R12, R12_H,
689 R13, R13_H,
690 R14, R14_H,
691 R15, R15_H,
692 R16, R16_H,
693 R17, R17_H,
694 R18, R18_H,
695 R19, R19_H,
696 R20, R20_H,
697 R21, R21_H,
698 R22, R22_H,
699 R23, R23_H,
700 R24, R24_H,
701 R25, R25_H,
702 R26, R26_H,
703 R27, R27_H,
704 R28, R28_H,
705 R29, R29_H,
706 R30, R30_H,
707 R31, R31_H
708 );
709
710 // Class for all long integer registers (including SP)
711 reg_class any_reg %{
712 return _ANY_REG_mask;
713 %}
714
715 // Class for non-allocatable 32 bit registers
716 reg_class non_allocatable_reg32(
717 #ifdef R18_RESERVED
718 // See comment in register_aarch64.hpp
719 R18, // tls on Windows
720 #endif
721 R28, // thread
722 R30, // lr
723 R31 // sp
724 );
725
726 // Class for non-allocatable 64 bit registers
727 reg_class non_allocatable_reg(
728 #ifdef R18_RESERVED
729 // See comment in register_aarch64.hpp
730 R18, R18_H, // tls on Windows, platform register on macOS
731 #endif
732 R28, R28_H, // thread
733 R30, R30_H, // lr
734 R31, R31_H // sp
735 );
736
737 // Class for all non-special integer registers
738 reg_class no_special_reg32 %{
739 return _NO_SPECIAL_REG32_mask;
740 %}
741
742 // Class for all non-special long integer registers
743 reg_class no_special_reg %{
744 return _NO_SPECIAL_REG_mask;
745 %}
746
747 // Class for 64 bit register r0
748 reg_class r0_reg(
749 R0, R0_H
750 );
751
752 // Class for 64 bit register r1
753 reg_class r1_reg(
754 R1, R1_H
755 );
756
757 // Class for 64 bit register r2
758 reg_class r2_reg(
759 R2, R2_H
760 );
761
762 // Class for 64 bit register r3
763 reg_class r3_reg(
764 R3, R3_H
765 );
766
767 // Class for 64 bit register r4
768 reg_class r4_reg(
769 R4, R4_H
770 );
771
772 // Class for 64 bit register r5
773 reg_class r5_reg(
774 R5, R5_H
775 );
776
777 // Class for 64 bit register r10
778 reg_class r10_reg(
779 R10, R10_H
780 );
781
782 // Class for 64 bit register r11
783 reg_class r11_reg(
784 R11, R11_H
785 );
786
787 // Class for method register
788 reg_class method_reg(
789 R12, R12_H
790 );
791
792 // Class for heapbase register
793 reg_class heapbase_reg(
794 R27, R27_H
795 );
796
797 // Class for thread register
798 reg_class thread_reg(
799 R28, R28_H
800 );
801
802 // Class for frame pointer register
803 reg_class fp_reg(
804 R29, R29_H
805 );
806
807 // Class for link register
808 reg_class lr_reg(
809 R30, R30_H
810 );
811
812 // Class for long sp register
813 reg_class sp_reg(
814 R31, R31_H
815 );
816
817 // Class for all pointer registers
818 reg_class ptr_reg %{
819 return _PTR_REG_mask;
820 %}
821
822 // Class for all non_special pointer registers
823 reg_class no_special_ptr_reg %{
824 return _NO_SPECIAL_PTR_REG_mask;
825 %}
826
827 // Class for all float registers
828 reg_class float_reg(
829 V0,
830 V1,
831 V2,
832 V3,
833 V4,
834 V5,
835 V6,
836 V7,
837 V8,
838 V9,
839 V10,
840 V11,
841 V12,
842 V13,
843 V14,
844 V15,
845 V16,
846 V17,
847 V18,
848 V19,
849 V20,
850 V21,
851 V22,
852 V23,
853 V24,
854 V25,
855 V26,
856 V27,
857 V28,
858 V29,
859 V30,
860 V31
861 );
862
863 // Double precision float registers have virtual `high halves' that
864 // are needed by the allocator.
865 // Class for all double registers
866 reg_class double_reg(
867 V0, V0_H,
868 V1, V1_H,
869 V2, V2_H,
870 V3, V3_H,
871 V4, V4_H,
872 V5, V5_H,
873 V6, V6_H,
874 V7, V7_H,
875 V8, V8_H,
876 V9, V9_H,
877 V10, V10_H,
878 V11, V11_H,
879 V12, V12_H,
880 V13, V13_H,
881 V14, V14_H,
882 V15, V15_H,
883 V16, V16_H,
884 V17, V17_H,
885 V18, V18_H,
886 V19, V19_H,
887 V20, V20_H,
888 V21, V21_H,
889 V22, V22_H,
890 V23, V23_H,
891 V24, V24_H,
892 V25, V25_H,
893 V26, V26_H,
894 V27, V27_H,
895 V28, V28_H,
896 V29, V29_H,
897 V30, V30_H,
898 V31, V31_H
899 );
900
901 // Class for all SVE vector registers.
902 reg_class vectora_reg (
903 V0, V0_H, V0_J, V0_K, V0_L, V0_M, V0_N, V0_O,
904 V1, V1_H, V1_J, V1_K, V1_L, V1_M, V1_N, V1_O,
905 V2, V2_H, V2_J, V2_K, V2_L, V2_M, V2_N, V2_O,
906 V3, V3_H, V3_J, V3_K, V3_L, V3_M, V3_N, V3_O,
907 V4, V4_H, V4_J, V4_K, V4_L, V4_M, V4_N, V4_O,
908 V5, V5_H, V5_J, V5_K, V5_L, V5_M, V5_N, V5_O,
909 V6, V6_H, V6_J, V6_K, V6_L, V6_M, V6_N, V6_O,
910 V7, V7_H, V7_J, V7_K, V7_L, V7_M, V7_N, V7_O,
911 V8, V8_H, V8_J, V8_K, V8_L, V8_M, V8_N, V8_O,
912 V9, V9_H, V9_J, V9_K, V9_L, V9_M, V9_N, V9_O,
913 V10, V10_H, V10_J, V10_K, V10_L, V10_M, V10_N, V10_O,
914 V11, V11_H, V11_J, V11_K, V11_L, V11_M, V11_N, V11_O,
915 V12, V12_H, V12_J, V12_K, V12_L, V12_M, V12_N, V12_O,
916 V13, V13_H, V13_J, V13_K, V13_L, V13_M, V13_N, V13_O,
917 V14, V14_H, V14_J, V14_K, V14_L, V14_M, V14_N, V14_O,
918 V15, V15_H, V15_J, V15_K, V15_L, V15_M, V15_N, V15_O,
919 V16, V16_H, V16_J, V16_K, V16_L, V16_M, V16_N, V16_O,
920 V17, V17_H, V17_J, V17_K, V17_L, V17_M, V17_N, V17_O,
921 V18, V18_H, V18_J, V18_K, V18_L, V18_M, V18_N, V18_O,
922 V19, V19_H, V19_J, V19_K, V19_L, V19_M, V19_N, V19_O,
923 V20, V20_H, V20_J, V20_K, V20_L, V20_M, V20_N, V20_O,
924 V21, V21_H, V21_J, V21_K, V21_L, V21_M, V21_N, V21_O,
925 V22, V22_H, V22_J, V22_K, V22_L, V22_M, V22_N, V22_O,
926 V23, V23_H, V23_J, V23_K, V23_L, V23_M, V23_N, V23_O,
927 V24, V24_H, V24_J, V24_K, V24_L, V24_M, V24_N, V24_O,
928 V25, V25_H, V25_J, V25_K, V25_L, V25_M, V25_N, V25_O,
929 V26, V26_H, V26_J, V26_K, V26_L, V26_M, V26_N, V26_O,
930 V27, V27_H, V27_J, V27_K, V27_L, V27_M, V27_N, V27_O,
931 V28, V28_H, V28_J, V28_K, V28_L, V28_M, V28_N, V28_O,
932 V29, V29_H, V29_J, V29_K, V29_L, V29_M, V29_N, V29_O,
933 V30, V30_H, V30_J, V30_K, V30_L, V30_M, V30_N, V30_O,
934 V31, V31_H, V31_J, V31_K, V31_L, V31_M, V31_N, V31_O,
935 );
936
937 // Class for all 64bit vector registers
938 reg_class vectord_reg(
939 V0, V0_H,
940 V1, V1_H,
941 V2, V2_H,
942 V3, V3_H,
943 V4, V4_H,
944 V5, V5_H,
945 V6, V6_H,
946 V7, V7_H,
947 V8, V8_H,
948 V9, V9_H,
949 V10, V10_H,
950 V11, V11_H,
951 V12, V12_H,
952 V13, V13_H,
953 V14, V14_H,
954 V15, V15_H,
955 V16, V16_H,
956 V17, V17_H,
957 V18, V18_H,
958 V19, V19_H,
959 V20, V20_H,
960 V21, V21_H,
961 V22, V22_H,
962 V23, V23_H,
963 V24, V24_H,
964 V25, V25_H,
965 V26, V26_H,
966 V27, V27_H,
967 V28, V28_H,
968 V29, V29_H,
969 V30, V30_H,
970 V31, V31_H
971 );
972
973 // Class for all 128bit vector registers
974 reg_class vectorx_reg(
975 V0, V0_H, V0_J, V0_K,
976 V1, V1_H, V1_J, V1_K,
977 V2, V2_H, V2_J, V2_K,
978 V3, V3_H, V3_J, V3_K,
979 V4, V4_H, V4_J, V4_K,
980 V5, V5_H, V5_J, V5_K,
981 V6, V6_H, V6_J, V6_K,
982 V7, V7_H, V7_J, V7_K,
983 V8, V8_H, V8_J, V8_K,
984 V9, V9_H, V9_J, V9_K,
985 V10, V10_H, V10_J, V10_K,
986 V11, V11_H, V11_J, V11_K,
987 V12, V12_H, V12_J, V12_K,
988 V13, V13_H, V13_J, V13_K,
989 V14, V14_H, V14_J, V14_K,
990 V15, V15_H, V15_J, V15_K,
991 V16, V16_H, V16_J, V16_K,
992 V17, V17_H, V17_J, V17_K,
993 V18, V18_H, V18_J, V18_K,
994 V19, V19_H, V19_J, V19_K,
995 V20, V20_H, V20_J, V20_K,
996 V21, V21_H, V21_J, V21_K,
997 V22, V22_H, V22_J, V22_K,
998 V23, V23_H, V23_J, V23_K,
999 V24, V24_H, V24_J, V24_K,
1000 V25, V25_H, V25_J, V25_K,
1001 V26, V26_H, V26_J, V26_K,
1002 V27, V27_H, V27_J, V27_K,
1003 V28, V28_H, V28_J, V28_K,
1004 V29, V29_H, V29_J, V29_K,
1005 V30, V30_H, V30_J, V30_K,
1006 V31, V31_H, V31_J, V31_K
1007 );
1008
1009 // Class for 128 bit register v0
1010 reg_class v0_reg(
1011 V0, V0_H
1012 );
1013
1014 // Class for 128 bit register v1
1015 reg_class v1_reg(
1016 V1, V1_H
1017 );
1018
1019 // Class for 128 bit register v2
1020 reg_class v2_reg(
1021 V2, V2_H
1022 );
1023
1024 // Class for 128 bit register v3
1025 reg_class v3_reg(
1026 V3, V3_H
1027 );
1028
1029 // Class for 128 bit register v4
1030 reg_class v4_reg(
1031 V4, V4_H
1032 );
1033
1034 // Class for 128 bit register v5
1035 reg_class v5_reg(
1036 V5, V5_H
1037 );
1038
1039 // Class for 128 bit register v6
1040 reg_class v6_reg(
1041 V6, V6_H
1042 );
1043
1044 // Class for 128 bit register v7
1045 reg_class v7_reg(
1046 V7, V7_H
1047 );
1048
1049 // Class for 128 bit register v8
1050 reg_class v8_reg(
1051 V8, V8_H
1052 );
1053
1054 // Class for 128 bit register v9
1055 reg_class v9_reg(
1056 V9, V9_H
1057 );
1058
1059 // Class for 128 bit register v10
1060 reg_class v10_reg(
1061 V10, V10_H
1062 );
1063
1064 // Class for 128 bit register v11
1065 reg_class v11_reg(
1066 V11, V11_H
1067 );
1068
1069 // Class for 128 bit register v12
1070 reg_class v12_reg(
1071 V12, V12_H
1072 );
1073
1074 // Class for 128 bit register v13
1075 reg_class v13_reg(
1076 V13, V13_H
1077 );
1078
1079 // Class for 128 bit register v14
1080 reg_class v14_reg(
1081 V14, V14_H
1082 );
1083
1084 // Class for 128 bit register v15
1085 reg_class v15_reg(
1086 V15, V15_H
1087 );
1088
1089 // Class for 128 bit register v16
1090 reg_class v16_reg(
1091 V16, V16_H
1092 );
1093
1094 // Class for 128 bit register v17
1095 reg_class v17_reg(
1096 V17, V17_H
1097 );
1098
1099 // Class for 128 bit register v18
1100 reg_class v18_reg(
1101 V18, V18_H
1102 );
1103
1104 // Class for 128 bit register v19
1105 reg_class v19_reg(
1106 V19, V19_H
1107 );
1108
1109 // Class for 128 bit register v20
1110 reg_class v20_reg(
1111 V20, V20_H
1112 );
1113
1114 // Class for 128 bit register v21
1115 reg_class v21_reg(
1116 V21, V21_H
1117 );
1118
1119 // Class for 128 bit register v22
1120 reg_class v22_reg(
1121 V22, V22_H
1122 );
1123
1124 // Class for 128 bit register v23
1125 reg_class v23_reg(
1126 V23, V23_H
1127 );
1128
1129 // Class for 128 bit register v24
1130 reg_class v24_reg(
1131 V24, V24_H
1132 );
1133
1134 // Class for 128 bit register v25
1135 reg_class v25_reg(
1136 V25, V25_H
1137 );
1138
1139 // Class for 128 bit register v26
1140 reg_class v26_reg(
1141 V26, V26_H
1142 );
1143
1144 // Class for 128 bit register v27
1145 reg_class v27_reg(
1146 V27, V27_H
1147 );
1148
1149 // Class for 128 bit register v28
1150 reg_class v28_reg(
1151 V28, V28_H
1152 );
1153
1154 // Class for 128 bit register v29
1155 reg_class v29_reg(
1156 V29, V29_H
1157 );
1158
1159 // Class for 128 bit register v30
1160 reg_class v30_reg(
1161 V30, V30_H
1162 );
1163
1164 // Class for 128 bit register v31
1165 reg_class v31_reg(
1166 V31, V31_H
1167 );
1168
1169 // Class for all SVE predicate registers.
1170 reg_class pr_reg (
1171 P0,
1172 P1,
1173 P2,
1174 P3,
1175 P4,
1176 P5,
1177 P6,
1178 // P7, non-allocatable, preserved with all elements preset to TRUE.
1179 P8,
1180 P9,
1181 P10,
1182 P11,
1183 P12,
1184 P13,
1185 P14,
1186 P15
1187 );
1188
1189 // Class for SVE governing predicate registers, which are used
1190 // to determine the active elements of a predicated instruction.
1191 reg_class gov_pr (
1192 P0,
1193 P1,
1194 P2,
1195 P3,
1196 P4,
1197 P5,
1198 P6,
1199 // P7, non-allocatable, preserved with all elements preset to TRUE.
1200 );
1201
1202 // Singleton class for condition codes
1203 reg_class int_flags(RFLAGS);
1204
1205 %}
1206
1207 //----------DEFINITION BLOCK---------------------------------------------------
1208 // Define name --> value mappings to inform the ADLC of an integer valued name
1209 // Current support includes integer values in the range [0, 0x7FFFFFFF]
1210 // Format:
1211 // int_def <name> ( <int_value>, <expression>);
1212 // Generated Code in ad_<arch>.hpp
1213 // #define <name> (<expression>)
1214 // // value == <int_value>
1215 // Generated code in ad_<arch>.cpp adlc_verification()
1216 // assert( <name> == <int_value>, "Expect (<expression>) to equal <int_value>");
1217 //
1218
1219 // we follow the ppc-aix port in using a simple cost model which ranks
1220 // register operations as cheap, memory ops as more expensive and
1221 // branches as most expensive. the first two have a low as well as a
1222 // normal cost. huge cost appears to be a way of saying don't do
1223 // something
1224
1225 definitions %{
1226 // The default cost (of a register move instruction).
1227 int_def INSN_COST ( 100, 100);
1228 int_def BRANCH_COST ( 200, 2 * INSN_COST);
1229 int_def CALL_COST ( 200, 2 * INSN_COST);
1230 int_def VOLATILE_REF_COST ( 1000, 10 * INSN_COST);
1231 %}
1232
1233
1234 //----------SOURCE BLOCK-------------------------------------------------------
1235 // This is a block of C++ code which provides values, functions, and
1236 // definitions necessary in the rest of the architecture description
1237
1238 source_hpp %{
1239
1240 #include "asm/macroAssembler.hpp"
1241 #include "gc/shared/barrierSetAssembler.hpp"
1242 #include "gc/shared/cardTable.hpp"
1243 #include "gc/shared/cardTableBarrierSet.hpp"
1244 #include "gc/shared/collectedHeap.hpp"
1245 #include "opto/addnode.hpp"
1246 #include "opto/convertnode.hpp"
1247 #include "runtime/objectMonitor.hpp"
1248
1249 extern RegMask _ANY_REG32_mask;
1250 extern RegMask _ANY_REG_mask;
1251 extern RegMask _PTR_REG_mask;
1252 extern RegMask _NO_SPECIAL_REG32_mask;
1253 extern RegMask _NO_SPECIAL_REG_mask;
1254 extern RegMask _NO_SPECIAL_PTR_REG_mask;
1255
1256 class CallStubImpl {
1257
1258 //--------------------------------------------------------------
1259 //---< Used for optimization in Compile::shorten_branches >---
1260 //--------------------------------------------------------------
1261
1262 public:
1263 // Size of call trampoline stub.
1264 static uint size_call_trampoline() {
1265 return 0; // no call trampolines on this platform
1266 }
1267
1268 // number of relocations needed by a call trampoline stub
1269 static uint reloc_call_trampoline() {
1270 return 0; // no call trampolines on this platform
1271 }
1272 };
1273
1274 class HandlerImpl {
1275
1276 public:
1277
1278 static int emit_exception_handler(CodeBuffer &cbuf);
1279 static int emit_deopt_handler(CodeBuffer& cbuf);
1280
1281 static uint size_exception_handler() {
1282 return MacroAssembler::far_codestub_branch_size();
1283 }
1284
1285 static uint size_deopt_handler() {
1286 // count one adr and one far branch instruction
1287 return NativeInstruction::instruction_size + MacroAssembler::far_codestub_branch_size();
1288 }
1289 };
1290
1291 class Node::PD {
1292 public:
1293 enum NodeFlags {
1294 _last_flag = Node::_last_flag
1295 };
1296 };
1297
1298 bool is_CAS(int opcode, bool maybe_volatile);
1299
1300 // predicates controlling emit of ldr<x>/ldar<x> and associated dmb
1301
1302 bool unnecessary_acquire(const Node *barrier);
1303 bool needs_acquiring_load(const Node *load);
1304
1305 // predicates controlling emit of str<x>/stlr<x> and associated dmbs
1306
1307 bool unnecessary_release(const Node *barrier);
1308 bool unnecessary_volatile(const Node *barrier);
1309 bool needs_releasing_store(const Node *store);
1310
1311 // predicate controlling translation of CompareAndSwapX
1312 bool needs_acquiring_load_exclusive(const Node *load);
1313
1314 // predicate controlling addressing modes
1315 bool size_fits_all_mem_uses(AddPNode* addp, int shift);
1316 %}
1317
1318 source %{
1319
1320 // Derived RegMask with conditionally allocatable registers
1321
1322 void PhaseOutput::pd_perform_mach_node_analysis() {
1323 }
1324
1325 int MachNode::pd_alignment_required() const {
1326 return 1;
1327 }
1328
1329 int MachNode::compute_padding(int current_offset) const {
1330 return 0;
1331 }
1332
1333 RegMask _ANY_REG32_mask;
1334 RegMask _ANY_REG_mask;
1335 RegMask _PTR_REG_mask;
1336 RegMask _NO_SPECIAL_REG32_mask;
1337 RegMask _NO_SPECIAL_REG_mask;
1338 RegMask _NO_SPECIAL_PTR_REG_mask;
1339
1340 void reg_mask_init() {
1341 // We derive below RegMask(s) from the ones which are auto-generated from
1342 // adlc register classes to make AArch64 rheapbase (r27) and rfp (r29)
1343 // registers conditionally reserved.
1344
1345 _ANY_REG32_mask = _ALL_REG32_mask;
1346 _ANY_REG32_mask.Remove(OptoReg::as_OptoReg(r31_sp->as_VMReg()));
1347
1348 _ANY_REG_mask = _ALL_REG_mask;
1349
1350 _PTR_REG_mask = _ALL_REG_mask;
1351
1352 _NO_SPECIAL_REG32_mask = _ALL_REG32_mask;
1353 _NO_SPECIAL_REG32_mask.SUBTRACT(_NON_ALLOCATABLE_REG32_mask);
1354
1355 _NO_SPECIAL_REG_mask = _ALL_REG_mask;
1356 _NO_SPECIAL_REG_mask.SUBTRACT(_NON_ALLOCATABLE_REG_mask);
1357
1358 _NO_SPECIAL_PTR_REG_mask = _ALL_REG_mask;
1359 _NO_SPECIAL_PTR_REG_mask.SUBTRACT(_NON_ALLOCATABLE_REG_mask);
1360
1361 // r27 is not allocatable when compressed oops is on and heapbase is not
1362 // zero, compressed klass pointers doesn't use r27 after JDK-8234794
1363 if (UseCompressedOops && (CompressedOops::ptrs_base() != NULL)) {
1364 _NO_SPECIAL_REG32_mask.Remove(OptoReg::as_OptoReg(r27->as_VMReg()));
1365 _NO_SPECIAL_REG_mask.SUBTRACT(_HEAPBASE_REG_mask);
1366 _NO_SPECIAL_PTR_REG_mask.SUBTRACT(_HEAPBASE_REG_mask);
1367 }
1368
1369 // r29 is not allocatable when PreserveFramePointer is on
1370 if (PreserveFramePointer) {
1371 _NO_SPECIAL_REG32_mask.Remove(OptoReg::as_OptoReg(r29->as_VMReg()));
1372 _NO_SPECIAL_REG_mask.SUBTRACT(_FP_REG_mask);
1373 _NO_SPECIAL_PTR_REG_mask.SUBTRACT(_FP_REG_mask);
1374 }
1375 }
1376
1377 // Optimizaton of volatile gets and puts
1378 // -------------------------------------
1379 //
1380 // AArch64 has ldar<x> and stlr<x> instructions which we can safely
1381 // use to implement volatile reads and writes. For a volatile read
1382 // we simply need
1383 //
1384 // ldar<x>
1385 //
1386 // and for a volatile write we need
1387 //
1388 // stlr<x>
1389 //
1390 // Alternatively, we can implement them by pairing a normal
1391 // load/store with a memory barrier. For a volatile read we need
1392 //
1393 // ldr<x>
1394 // dmb ishld
1395 //
1396 // for a volatile write
1397 //
1398 // dmb ish
1399 // str<x>
1400 // dmb ish
1401 //
1402 // We can also use ldaxr and stlxr to implement compare and swap CAS
1403 // sequences. These are normally translated to an instruction
1404 // sequence like the following
1405 //
1406 // dmb ish
1407 // retry:
1408 // ldxr<x> rval raddr
1409 // cmp rval rold
1410 // b.ne done
1411 // stlxr<x> rval, rnew, rold
1412 // cbnz rval retry
1413 // done:
1414 // cset r0, eq
1415 // dmb ishld
1416 //
1417 // Note that the exclusive store is already using an stlxr
1418 // instruction. That is required to ensure visibility to other
1419 // threads of the exclusive write (assuming it succeeds) before that
1420 // of any subsequent writes.
1421 //
1422 // The following instruction sequence is an improvement on the above
1423 //
1424 // retry:
1425 // ldaxr<x> rval raddr
1426 // cmp rval rold
1427 // b.ne done
1428 // stlxr<x> rval, rnew, rold
1429 // cbnz rval retry
1430 // done:
1431 // cset r0, eq
1432 //
1433 // We don't need the leading dmb ish since the stlxr guarantees
1434 // visibility of prior writes in the case that the swap is
1435 // successful. Crucially we don't have to worry about the case where
1436 // the swap is not successful since no valid program should be
1437 // relying on visibility of prior changes by the attempting thread
1438 // in the case where the CAS fails.
1439 //
1440 // Similarly, we don't need the trailing dmb ishld if we substitute
1441 // an ldaxr instruction since that will provide all the guarantees we
1442 // require regarding observation of changes made by other threads
1443 // before any change to the CAS address observed by the load.
1444 //
1445 // In order to generate the desired instruction sequence we need to
1446 // be able to identify specific 'signature' ideal graph node
1447 // sequences which i) occur as a translation of a volatile reads or
1448 // writes or CAS operations and ii) do not occur through any other
1449 // translation or graph transformation. We can then provide
1450 // alternative aldc matching rules which translate these node
1451 // sequences to the desired machine code sequences. Selection of the
1452 // alternative rules can be implemented by predicates which identify
1453 // the relevant node sequences.
1454 //
1455 // The ideal graph generator translates a volatile read to the node
1456 // sequence
1457 //
1458 // LoadX[mo_acquire]
1459 // MemBarAcquire
1460 //
1461 // As a special case when using the compressed oops optimization we
1462 // may also see this variant
1463 //
1464 // LoadN[mo_acquire]
1465 // DecodeN
1466 // MemBarAcquire
1467 //
1468 // A volatile write is translated to the node sequence
1469 //
1470 // MemBarRelease
1471 // StoreX[mo_release] {CardMark}-optional
1472 // MemBarVolatile
1473 //
1474 // n.b. the above node patterns are generated with a strict
1475 // 'signature' configuration of input and output dependencies (see
1476 // the predicates below for exact details). The card mark may be as
1477 // simple as a few extra nodes or, in a few GC configurations, may
1478 // include more complex control flow between the leading and
1479 // trailing memory barriers. However, whatever the card mark
1480 // configuration these signatures are unique to translated volatile
1481 // reads/stores -- they will not appear as a result of any other
1482 // bytecode translation or inlining nor as a consequence of
1483 // optimizing transforms.
1484 //
1485 // We also want to catch inlined unsafe volatile gets and puts and
1486 // be able to implement them using either ldar<x>/stlr<x> or some
1487 // combination of ldr<x>/stlr<x> and dmb instructions.
1488 //
1489 // Inlined unsafe volatiles puts manifest as a minor variant of the
1490 // normal volatile put node sequence containing an extra cpuorder
1491 // membar
1492 //
1493 // MemBarRelease
1494 // MemBarCPUOrder
1495 // StoreX[mo_release] {CardMark}-optional
1496 // MemBarCPUOrder
1497 // MemBarVolatile
1498 //
1499 // n.b. as an aside, a cpuorder membar is not itself subject to
1500 // matching and translation by adlc rules. However, the rule
1501 // predicates need to detect its presence in order to correctly
1502 // select the desired adlc rules.
1503 //
1504 // Inlined unsafe volatile gets manifest as a slightly different
1505 // node sequence to a normal volatile get because of the
1506 // introduction of some CPUOrder memory barriers to bracket the
1507 // Load. However, but the same basic skeleton of a LoadX feeding a
1508 // MemBarAcquire, possibly thorugh an optional DecodeN, is still
1509 // present
1510 //
1511 // MemBarCPUOrder
1512 // || \\
1513 // MemBarCPUOrder LoadX[mo_acquire]
1514 // || |
1515 // || {DecodeN} optional
1516 // || /
1517 // MemBarAcquire
1518 //
1519 // In this case the acquire membar does not directly depend on the
1520 // load. However, we can be sure that the load is generated from an
1521 // inlined unsafe volatile get if we see it dependent on this unique
1522 // sequence of membar nodes. Similarly, given an acquire membar we
1523 // can know that it was added because of an inlined unsafe volatile
1524 // get if it is fed and feeds a cpuorder membar and if its feed
1525 // membar also feeds an acquiring load.
1526 //
1527 // Finally an inlined (Unsafe) CAS operation is translated to the
1528 // following ideal graph
1529 //
1530 // MemBarRelease
1531 // MemBarCPUOrder
1532 // CompareAndSwapX {CardMark}-optional
1533 // MemBarCPUOrder
1534 // MemBarAcquire
1535 //
1536 // So, where we can identify these volatile read and write
1537 // signatures we can choose to plant either of the above two code
1538 // sequences. For a volatile read we can simply plant a normal
1539 // ldr<x> and translate the MemBarAcquire to a dmb. However, we can
1540 // also choose to inhibit translation of the MemBarAcquire and
1541 // inhibit planting of the ldr<x>, instead planting an ldar<x>.
1542 //
1543 // When we recognise a volatile store signature we can choose to
1544 // plant at a dmb ish as a translation for the MemBarRelease, a
1545 // normal str<x> and then a dmb ish for the MemBarVolatile.
1546 // Alternatively, we can inhibit translation of the MemBarRelease
1547 // and MemBarVolatile and instead plant a simple stlr<x>
1548 // instruction.
1549 //
1550 // when we recognise a CAS signature we can choose to plant a dmb
1551 // ish as a translation for the MemBarRelease, the conventional
1552 // macro-instruction sequence for the CompareAndSwap node (which
1553 // uses ldxr<x>) and then a dmb ishld for the MemBarAcquire.
1554 // Alternatively, we can elide generation of the dmb instructions
1555 // and plant the alternative CompareAndSwap macro-instruction
1556 // sequence (which uses ldaxr<x>).
1557 //
1558 // Of course, the above only applies when we see these signature
1559 // configurations. We still want to plant dmb instructions in any
1560 // other cases where we may see a MemBarAcquire, MemBarRelease or
1561 // MemBarVolatile. For example, at the end of a constructor which
1562 // writes final/volatile fields we will see a MemBarRelease
1563 // instruction and this needs a 'dmb ish' lest we risk the
1564 // constructed object being visible without making the
1565 // final/volatile field writes visible.
1566 //
1567 // n.b. the translation rules below which rely on detection of the
1568 // volatile signatures and insert ldar<x> or stlr<x> are failsafe.
1569 // If we see anything other than the signature configurations we
1570 // always just translate the loads and stores to ldr<x> and str<x>
1571 // and translate acquire, release and volatile membars to the
1572 // relevant dmb instructions.
1573 //
1574
1575 // is_CAS(int opcode, bool maybe_volatile)
1576 //
1577 // return true if opcode is one of the possible CompareAndSwapX
1578 // values otherwise false.
1579
1580 bool is_CAS(int opcode, bool maybe_volatile)
1581 {
1582 switch(opcode) {
1583 // We handle these
1584 case Op_CompareAndSwapI:
1585 case Op_CompareAndSwapL:
1586 case Op_CompareAndSwapP:
1587 case Op_CompareAndSwapN:
1588 case Op_ShenandoahCompareAndSwapP:
1589 case Op_ShenandoahCompareAndSwapN:
1590 case Op_CompareAndSwapB:
1591 case Op_CompareAndSwapS:
1592 case Op_GetAndSetI:
1593 case Op_GetAndSetL:
1594 case Op_GetAndSetP:
1595 case Op_GetAndSetN:
1596 case Op_GetAndAddI:
1597 case Op_GetAndAddL:
1598 return true;
1599 case Op_CompareAndExchangeI:
1600 case Op_CompareAndExchangeN:
1601 case Op_CompareAndExchangeB:
1602 case Op_CompareAndExchangeS:
1603 case Op_CompareAndExchangeL:
1604 case Op_CompareAndExchangeP:
1605 case Op_WeakCompareAndSwapB:
1606 case Op_WeakCompareAndSwapS:
1607 case Op_WeakCompareAndSwapI:
1608 case Op_WeakCompareAndSwapL:
1609 case Op_WeakCompareAndSwapP:
1610 case Op_WeakCompareAndSwapN:
1611 case Op_ShenandoahWeakCompareAndSwapP:
1612 case Op_ShenandoahWeakCompareAndSwapN:
1613 case Op_ShenandoahCompareAndExchangeP:
1614 case Op_ShenandoahCompareAndExchangeN:
1615 return maybe_volatile;
1616 default:
1617 return false;
1618 }
1619 }
1620
1621 // helper to determine the maximum number of Phi nodes we may need to
1622 // traverse when searching from a card mark membar for the merge mem
1623 // feeding a trailing membar or vice versa
1624
1625 // predicates controlling emit of ldr<x>/ldar<x>
1626
1627 bool unnecessary_acquire(const Node *barrier)
1628 {
1629 assert(barrier->is_MemBar(), "expecting a membar");
1630
1631 MemBarNode* mb = barrier->as_MemBar();
1632
1633 if (mb->trailing_load()) {
1634 return true;
1635 }
1636
1637 if (mb->trailing_load_store()) {
1638 Node* load_store = mb->in(MemBarNode::Precedent);
1639 assert(load_store->is_LoadStore(), "unexpected graph shape");
1640 return is_CAS(load_store->Opcode(), true);
1641 }
1642
1643 return false;
1644 }
1645
1646 bool needs_acquiring_load(const Node *n)
1647 {
1648 assert(n->is_Load(), "expecting a load");
1649 LoadNode *ld = n->as_Load();
1650 return ld->is_acquire();
1651 }
1652
1653 bool unnecessary_release(const Node *n)
1654 {
1655 assert((n->is_MemBar() &&
1656 n->Opcode() == Op_MemBarRelease),
1657 "expecting a release membar");
1658
1659 MemBarNode *barrier = n->as_MemBar();
1660 if (!barrier->leading()) {
1661 return false;
1662 } else {
1663 Node* trailing = barrier->trailing_membar();
1664 MemBarNode* trailing_mb = trailing->as_MemBar();
1665 assert(trailing_mb->trailing(), "Not a trailing membar?");
1666 assert(trailing_mb->leading_membar() == n, "inconsistent leading/trailing membars");
1667
1668 Node* mem = trailing_mb->in(MemBarNode::Precedent);
1669 if (mem->is_Store()) {
1670 assert(mem->as_Store()->is_release(), "");
1671 assert(trailing_mb->Opcode() == Op_MemBarVolatile, "");
1672 return true;
1673 } else {
1674 assert(mem->is_LoadStore(), "");
1675 assert(trailing_mb->Opcode() == Op_MemBarAcquire, "");
1676 return is_CAS(mem->Opcode(), true);
1677 }
1678 }
1679 return false;
1680 }
1681
1682 bool unnecessary_volatile(const Node *n)
1683 {
1684 // assert n->is_MemBar();
1685 MemBarNode *mbvol = n->as_MemBar();
1686
1687 bool release = mbvol->trailing_store();
1688 assert(!release || (mbvol->in(MemBarNode::Precedent)->is_Store() && mbvol->in(MemBarNode::Precedent)->as_Store()->is_release()), "");
1689 #ifdef ASSERT
1690 if (release) {
1691 Node* leading = mbvol->leading_membar();
1692 assert(leading->Opcode() == Op_MemBarRelease, "");
1693 assert(leading->as_MemBar()->leading_store(), "");
1694 assert(leading->as_MemBar()->trailing_membar() == mbvol, "");
1695 }
1696 #endif
1697
1698 return release;
1699 }
1700
1701 // predicates controlling emit of str<x>/stlr<x>
1702
1703 bool needs_releasing_store(const Node *n)
1704 {
1705 // assert n->is_Store();
1706 StoreNode *st = n->as_Store();
1707 return st->trailing_membar() != NULL;
1708 }
1709
1710 // predicate controlling translation of CAS
1711 //
1712 // returns true if CAS needs to use an acquiring load otherwise false
1713
1714 bool needs_acquiring_load_exclusive(const Node *n)
1715 {
1716 assert(is_CAS(n->Opcode(), true), "expecting a compare and swap");
1717 LoadStoreNode* ldst = n->as_LoadStore();
1718 if (is_CAS(n->Opcode(), false)) {
1719 assert(ldst->trailing_membar() != NULL, "expected trailing membar");
1720 } else {
1721 return ldst->trailing_membar() != NULL;
1722 }
1723
1724 // so we can just return true here
1725 return true;
1726 }
1727
1728 #define __ _masm.
1729
1730 // advance declarations for helper functions to convert register
1731 // indices to register objects
1732
1733 // the ad file has to provide implementations of certain methods
1734 // expected by the generic code
1735 //
1736 // REQUIRED FUNCTIONALITY
1737
1738 //=============================================================================
1739
1740 // !!!!! Special hack to get all types of calls to specify the byte offset
1741 // from the start of the call to the point where the return address
1742 // will point.
1743
1744 int MachCallStaticJavaNode::ret_addr_offset()
1745 {
1746 // call should be a simple bl
1747 int off = 4;
1748 return off;
1749 }
1750
1751 int MachCallDynamicJavaNode::ret_addr_offset()
1752 {
1753 return 16; // movz, movk, movk, bl
1754 }
1755
1756 int MachCallRuntimeNode::ret_addr_offset() {
1757 // for generated stubs the call will be
1758 // bl(addr)
1759 // or with far branches
1760 // bl(trampoline_stub)
1761 // for real runtime callouts it will be six instructions
1762 // see aarch64_enc_java_to_runtime
1763 // adr(rscratch2, retaddr)
1764 // lea(rscratch1, RuntimeAddress(addr)
1765 // stp(zr, rscratch2, Address(__ pre(sp, -2 * wordSize)))
1766 // blr(rscratch1)
1767 CodeBlob *cb = CodeCache::find_blob(_entry_point);
1768 if (cb) {
1769 return 1 * NativeInstruction::instruction_size;
1770 } else {
1771 return 6 * NativeInstruction::instruction_size;
1772 }
1773 }
1774
1775 int MachCallNativeNode::ret_addr_offset() {
1776 // This is implemented using aarch64_enc_java_to_runtime as above.
1777 CodeBlob *cb = CodeCache::find_blob(_entry_point);
1778 if (cb) {
1779 return 1 * NativeInstruction::instruction_size;
1780 } else {
1781 return 6 * NativeInstruction::instruction_size;
1782 }
1783 }
1784
1785 //=============================================================================
1786
1787 #ifndef PRODUCT
1788 void MachBreakpointNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
1789 st->print("BREAKPOINT");
1790 }
1791 #endif
1792
1793 void MachBreakpointNode::emit(CodeBuffer &cbuf, PhaseRegAlloc *ra_) const {
1794 C2_MacroAssembler _masm(&cbuf);
1795 __ brk(0);
1796 }
1797
1798 uint MachBreakpointNode::size(PhaseRegAlloc *ra_) const {
1799 return MachNode::size(ra_);
1800 }
1801
1802 //=============================================================================
1803
1804 #ifndef PRODUCT
1805 void MachNopNode::format(PhaseRegAlloc*, outputStream* st) const {
1806 st->print("nop \t# %d bytes pad for loops and calls", _count);
1807 }
1808 #endif
1809
1810 void MachNopNode::emit(CodeBuffer &cbuf, PhaseRegAlloc*) const {
1811 C2_MacroAssembler _masm(&cbuf);
1812 for (int i = 0; i < _count; i++) {
1813 __ nop();
1814 }
1815 }
1816
1817 uint MachNopNode::size(PhaseRegAlloc*) const {
1818 return _count * NativeInstruction::instruction_size;
1819 }
1820
1821 //=============================================================================
1822 const RegMask& MachConstantBaseNode::_out_RegMask = RegMask::Empty;
1823
1824 int ConstantTable::calculate_table_base_offset() const {
1825 return 0; // absolute addressing, no offset
1826 }
1827
1828 bool MachConstantBaseNode::requires_postalloc_expand() const { return false; }
1829 void MachConstantBaseNode::postalloc_expand(GrowableArray <Node *> *nodes, PhaseRegAlloc *ra_) {
1830 ShouldNotReachHere();
1831 }
1832
1833 void MachConstantBaseNode::emit(CodeBuffer& cbuf, PhaseRegAlloc* ra_) const {
1834 // Empty encoding
1835 }
1836
1837 uint MachConstantBaseNode::size(PhaseRegAlloc* ra_) const {
1838 return 0;
1839 }
1840
1841 #ifndef PRODUCT
1842 void MachConstantBaseNode::format(PhaseRegAlloc* ra_, outputStream* st) const {
1843 st->print("-- \t// MachConstantBaseNode (empty encoding)");
1844 }
1845 #endif
1846
1847 #ifndef PRODUCT
1848 void MachPrologNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
1849 Compile* C = ra_->C;
1850
1851 int framesize = C->output()->frame_slots() << LogBytesPerInt;
1852
1853 if (C->output()->need_stack_bang(framesize))
1854 st->print("# stack bang size=%d\n\t", framesize);
1855
1856 if (framesize < ((1 << 9) + 2 * wordSize)) {
1857 st->print("sub sp, sp, #%d\n\t", framesize);
1858 st->print("stp rfp, lr, [sp, #%d]", framesize - 2 * wordSize);
1859 if (PreserveFramePointer) st->print("\n\tadd rfp, sp, #%d", framesize - 2 * wordSize);
1860 } else {
1861 st->print("stp lr, rfp, [sp, #%d]!\n\t", -(2 * wordSize));
1862 if (PreserveFramePointer) st->print("mov rfp, sp\n\t");
1863 st->print("mov rscratch1, #%d\n\t", framesize - 2 * wordSize);
1864 st->print("sub sp, sp, rscratch1");
1865 }
1866 if (C->stub_function() == NULL && BarrierSet::barrier_set()->barrier_set_nmethod() != NULL) {
1867 st->print("\n\t");
1868 st->print("ldr rscratch1, [guard]\n\t");
1869 st->print("dmb ishld\n\t");
1870 st->print("ldr rscratch2, [rthread, #thread_disarmed_offset]\n\t");
1871 st->print("cmp rscratch1, rscratch2\n\t");
1872 st->print("b.eq skip");
1873 st->print("\n\t");
1874 st->print("blr #nmethod_entry_barrier_stub\n\t");
1875 st->print("b skip\n\t");
1876 st->print("guard: int\n\t");
1877 st->print("\n\t");
1878 st->print("skip:\n\t");
1879 }
1880 }
1881 #endif
1882
1883 void MachPrologNode::emit(CodeBuffer &cbuf, PhaseRegAlloc *ra_) const {
1884 Compile* C = ra_->C;
1885 C2_MacroAssembler _masm(&cbuf);
1886
1887 // n.b. frame size includes space for return pc and rfp
1888 const int framesize = C->output()->frame_size_in_bytes();
1889
1890 // insert a nop at the start of the prolog so we can patch in a
1891 // branch if we need to invalidate the method later
1892 __ nop();
1893
1894 if (C->clinit_barrier_on_entry()) {
1895 assert(!C->method()->holder()->is_not_initialized(), "initialization should have been started");
1896
1897 Label L_skip_barrier;
1898
1899 __ mov_metadata(rscratch2, C->method()->holder()->constant_encoding());
1900 __ clinit_barrier(rscratch2, rscratch1, &L_skip_barrier);
1901 __ far_jump(RuntimeAddress(SharedRuntime::get_handle_wrong_method_stub()));
1902 __ bind(L_skip_barrier);
1903 }
1904
1905 if (C->max_vector_size() >= 16) {
1906 __ reinitialize_ptrue();
1907 }
1908
1909 int bangsize = C->output()->bang_size_in_bytes();
1910 if (C->output()->need_stack_bang(bangsize))
1911 __ generate_stack_overflow_check(bangsize);
1912
1913 __ build_frame(framesize);
1914
1915 if (C->stub_function() == NULL) {
1916 BarrierSetAssembler* bs = BarrierSet::barrier_set()->barrier_set_assembler();
1917 bs->nmethod_entry_barrier(&_masm);
1918 }
1919
1920 if (VerifyStackAtCalls) {
1921 Unimplemented();
1922 }
1923
1924 C->output()->set_frame_complete(cbuf.insts_size());
1925
1926 if (C->has_mach_constant_base_node()) {
1927 // NOTE: We set the table base offset here because users might be
1928 // emitted before MachConstantBaseNode.
1929 ConstantTable& constant_table = C->output()->constant_table();
1930 constant_table.set_table_base_offset(constant_table.calculate_table_base_offset());
1931 }
1932 }
1933
1934 uint MachPrologNode::size(PhaseRegAlloc* ra_) const
1935 {
1936 return MachNode::size(ra_); // too many variables; just compute it
1937 // the hard way
1938 }
1939
1940 int MachPrologNode::reloc() const
1941 {
1942 return 0;
1943 }
1944
1945 //=============================================================================
1946
1947 #ifndef PRODUCT
1948 void MachEpilogNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
1949 Compile* C = ra_->C;
1950 int framesize = C->output()->frame_slots() << LogBytesPerInt;
1951
1952 st->print("# pop frame %d\n\t",framesize);
1953
1954 if (framesize == 0) {
1955 st->print("ldp lr, rfp, [sp],#%d\n\t", (2 * wordSize));
1956 } else if (framesize < ((1 << 9) + 2 * wordSize)) {
1957 st->print("ldp lr, rfp, [sp,#%d]\n\t", framesize - 2 * wordSize);
1958 st->print("add sp, sp, #%d\n\t", framesize);
1959 } else {
1960 st->print("mov rscratch1, #%d\n\t", framesize - 2 * wordSize);
1961 st->print("add sp, sp, rscratch1\n\t");
1962 st->print("ldp lr, rfp, [sp],#%d\n\t", (2 * wordSize));
1963 }
1964
1965 if (do_polling() && C->is_method_compilation()) {
1966 st->print("# test polling word\n\t");
1967 st->print("ldr rscratch1, [rthread],#%d\n\t", in_bytes(JavaThread::polling_word_offset()));
1968 st->print("cmp sp, rscratch1\n\t");
1969 st->print("bhi #slow_path");
1970 }
1971 }
1972 #endif
1973
1974 void MachEpilogNode::emit(CodeBuffer &cbuf, PhaseRegAlloc *ra_) const {
1975 Compile* C = ra_->C;
1976 C2_MacroAssembler _masm(&cbuf);
1977 int framesize = C->output()->frame_slots() << LogBytesPerInt;
1978
1979 __ remove_frame(framesize);
1980
1981 if (StackReservedPages > 0 && C->has_reserved_stack_access()) {
1982 __ reserved_stack_check();
1983 }
1984
1985 if (do_polling() && C->is_method_compilation()) {
1986 Label dummy_label;
1987 Label* code_stub = &dummy_label;
1988 if (!C->output()->in_scratch_emit_size()) {
1989 code_stub = &C->output()->safepoint_poll_table()->add_safepoint(__ offset());
1990 }
1991 __ relocate(relocInfo::poll_return_type);
1992 __ safepoint_poll(*code_stub, true /* at_return */, false /* acquire */, true /* in_nmethod */);
1993 }
1994 }
1995
1996 uint MachEpilogNode::size(PhaseRegAlloc *ra_) const {
1997 // Variable size. Determine dynamically.
1998 return MachNode::size(ra_);
1999 }
2000
2001 int MachEpilogNode::reloc() const {
2002 // Return number of relocatable values contained in this instruction.
2003 return 1; // 1 for polling page.
2004 }
2005
2006 const Pipeline * MachEpilogNode::pipeline() const {
2007 return MachNode::pipeline_class();
2008 }
2009
2010 //=============================================================================
2011
2012 // Figure out which register class each belongs in: rc_int, rc_float or
2013 // rc_stack.
2014 enum RC { rc_bad, rc_int, rc_float, rc_predicate, rc_stack };
2015
2016 static enum RC rc_class(OptoReg::Name reg) {
2017
2018 if (reg == OptoReg::Bad) {
2019 return rc_bad;
2020 }
2021
2022 // we have 32 int registers * 2 halves
2023 int slots_of_int_registers = RegisterImpl::max_slots_per_register * RegisterImpl::number_of_registers;
2024
2025 if (reg < slots_of_int_registers) {
2026 return rc_int;
2027 }
2028
2029 // we have 32 float register * 8 halves
2030 int slots_of_float_registers = FloatRegisterImpl::max_slots_per_register * FloatRegisterImpl::number_of_registers;
2031 if (reg < slots_of_int_registers + slots_of_float_registers) {
2032 return rc_float;
2033 }
2034
2035 int slots_of_predicate_registers = PRegisterImpl::max_slots_per_register * PRegisterImpl::number_of_registers;
2036 if (reg < slots_of_int_registers + slots_of_float_registers + slots_of_predicate_registers) {
2037 return rc_predicate;
2038 }
2039
2040 // Between predicate regs & stack is the flags.
2041 assert(OptoReg::is_stack(reg), "blow up if spilling flags");
2042
2043 return rc_stack;
2044 }
2045
2046 uint MachSpillCopyNode::implementation(CodeBuffer *cbuf, PhaseRegAlloc *ra_, bool do_size, outputStream *st) const {
2047 Compile* C = ra_->C;
2048
2049 // Get registers to move.
2050 OptoReg::Name src_hi = ra_->get_reg_second(in(1));
2051 OptoReg::Name src_lo = ra_->get_reg_first(in(1));
2052 OptoReg::Name dst_hi = ra_->get_reg_second(this);
2053 OptoReg::Name dst_lo = ra_->get_reg_first(this);
2054
2055 enum RC src_hi_rc = rc_class(src_hi);
2056 enum RC src_lo_rc = rc_class(src_lo);
2057 enum RC dst_hi_rc = rc_class(dst_hi);
2058 enum RC dst_lo_rc = rc_class(dst_lo);
2059
2060 assert(src_lo != OptoReg::Bad && dst_lo != OptoReg::Bad, "must move at least 1 register");
2061
2062 if (src_hi != OptoReg::Bad) {
2063 assert((src_lo&1)==0 && src_lo+1==src_hi &&
2064 (dst_lo&1)==0 && dst_lo+1==dst_hi,
2065 "expected aligned-adjacent pairs");
2066 }
2067
2068 if (src_lo == dst_lo && src_hi == dst_hi) {
2069 return 0; // Self copy, no move.
2070 }
2071
2072 bool is64 = (src_lo & 1) == 0 && src_lo + 1 == src_hi &&
2073 (dst_lo & 1) == 0 && dst_lo + 1 == dst_hi;
2074 int src_offset = ra_->reg2offset(src_lo);
2075 int dst_offset = ra_->reg2offset(dst_lo);
2076
2077 if (bottom_type()->isa_vect() != NULL) {
2078 uint ireg = ideal_reg();
2079 if (ireg == Op_VecA && cbuf) {
2080 C2_MacroAssembler _masm(cbuf);
2081 int sve_vector_reg_size_in_bytes = Matcher::scalable_vector_reg_size(T_BYTE);
2082 if (src_lo_rc == rc_stack && dst_lo_rc == rc_stack) {
2083 // stack->stack
2084 __ spill_copy_sve_vector_stack_to_stack(src_offset, dst_offset,
2085 sve_vector_reg_size_in_bytes);
2086 } else if (src_lo_rc == rc_float && dst_lo_rc == rc_stack) {
2087 __ spill_sve_vector(as_FloatRegister(Matcher::_regEncode[src_lo]), ra_->reg2offset(dst_lo),
2088 sve_vector_reg_size_in_bytes);
2089 } else if (src_lo_rc == rc_stack && dst_lo_rc == rc_float) {
2090 __ unspill_sve_vector(as_FloatRegister(Matcher::_regEncode[dst_lo]), ra_->reg2offset(src_lo),
2091 sve_vector_reg_size_in_bytes);
2092 } else if (src_lo_rc == rc_float && dst_lo_rc == rc_float) {
2093 __ sve_orr(as_FloatRegister(Matcher::_regEncode[dst_lo]),
2094 as_FloatRegister(Matcher::_regEncode[src_lo]),
2095 as_FloatRegister(Matcher::_regEncode[src_lo]));
2096 } else {
2097 ShouldNotReachHere();
2098 }
2099 } else if (cbuf) {
2100 assert(ireg == Op_VecD || ireg == Op_VecX, "must be 64 bit or 128 bit vector");
2101 C2_MacroAssembler _masm(cbuf);
2102 assert((src_lo_rc != rc_int && dst_lo_rc != rc_int), "sanity");
2103 if (src_lo_rc == rc_stack && dst_lo_rc == rc_stack) {
2104 // stack->stack
2105 assert((src_offset & 7) == 0 && (dst_offset & 7) == 0, "unaligned stack offset");
2106 if (ireg == Op_VecD) {
2107 __ unspill(rscratch1, true, src_offset);
2108 __ spill(rscratch1, true, dst_offset);
2109 } else {
2110 __ spill_copy128(src_offset, dst_offset);
2111 }
2112 } else if (src_lo_rc == rc_float && dst_lo_rc == rc_float) {
2113 __ mov(as_FloatRegister(Matcher::_regEncode[dst_lo]),
2114 ireg == Op_VecD ? __ T8B : __ T16B,
2115 as_FloatRegister(Matcher::_regEncode[src_lo]));
2116 } else if (src_lo_rc == rc_float && dst_lo_rc == rc_stack) {
2117 __ spill(as_FloatRegister(Matcher::_regEncode[src_lo]),
2118 ireg == Op_VecD ? __ D : __ Q,
2119 ra_->reg2offset(dst_lo));
2120 } else if (src_lo_rc == rc_stack && dst_lo_rc == rc_float) {
2121 __ unspill(as_FloatRegister(Matcher::_regEncode[dst_lo]),
2122 ireg == Op_VecD ? __ D : __ Q,
2123 ra_->reg2offset(src_lo));
2124 } else {
2125 ShouldNotReachHere();
2126 }
2127 }
2128 } else if (cbuf) {
2129 C2_MacroAssembler _masm(cbuf);
2130 switch (src_lo_rc) {
2131 case rc_int:
2132 if (dst_lo_rc == rc_int) { // gpr --> gpr copy
2133 if (is64) {
2134 __ mov(as_Register(Matcher::_regEncode[dst_lo]),
2135 as_Register(Matcher::_regEncode[src_lo]));
2136 } else {
2137 C2_MacroAssembler _masm(cbuf);
2138 __ movw(as_Register(Matcher::_regEncode[dst_lo]),
2139 as_Register(Matcher::_regEncode[src_lo]));
2140 }
2141 } else if (dst_lo_rc == rc_float) { // gpr --> fpr copy
2142 if (is64) {
2143 __ fmovd(as_FloatRegister(Matcher::_regEncode[dst_lo]),
2144 as_Register(Matcher::_regEncode[src_lo]));
2145 } else {
2146 __ fmovs(as_FloatRegister(Matcher::_regEncode[dst_lo]),
2147 as_Register(Matcher::_regEncode[src_lo]));
2148 }
2149 } else { // gpr --> stack spill
2150 assert(dst_lo_rc == rc_stack, "spill to bad register class");
2151 __ spill(as_Register(Matcher::_regEncode[src_lo]), is64, dst_offset);
2152 }
2153 break;
2154 case rc_float:
2155 if (dst_lo_rc == rc_int) { // fpr --> gpr copy
2156 if (is64) {
2157 __ fmovd(as_Register(Matcher::_regEncode[dst_lo]),
2158 as_FloatRegister(Matcher::_regEncode[src_lo]));
2159 } else {
2160 __ fmovs(as_Register(Matcher::_regEncode[dst_lo]),
2161 as_FloatRegister(Matcher::_regEncode[src_lo]));
2162 }
2163 } else if (dst_lo_rc == rc_float) { // fpr --> fpr copy
2164 if (cbuf) {
2165 __ fmovd(as_FloatRegister(Matcher::_regEncode[dst_lo]),
2166 as_FloatRegister(Matcher::_regEncode[src_lo]));
2167 } else {
2168 __ fmovs(as_FloatRegister(Matcher::_regEncode[dst_lo]),
2169 as_FloatRegister(Matcher::_regEncode[src_lo]));
2170 }
2171 } else { // fpr --> stack spill
2172 assert(dst_lo_rc == rc_stack, "spill to bad register class");
2173 __ spill(as_FloatRegister(Matcher::_regEncode[src_lo]),
2174 is64 ? __ D : __ S, dst_offset);
2175 }
2176 break;
2177 case rc_stack:
2178 if (dst_lo_rc == rc_int) { // stack --> gpr load
2179 __ unspill(as_Register(Matcher::_regEncode[dst_lo]), is64, src_offset);
2180 } else if (dst_lo_rc == rc_float) { // stack --> fpr load
2181 __ unspill(as_FloatRegister(Matcher::_regEncode[dst_lo]),
2182 is64 ? __ D : __ S, src_offset);
2183 } else { // stack --> stack copy
2184 assert(dst_lo_rc == rc_stack, "spill to bad register class");
2185 __ unspill(rscratch1, is64, src_offset);
2186 __ spill(rscratch1, is64, dst_offset);
2187 }
2188 break;
2189 default:
2190 assert(false, "bad rc_class for spill");
2191 ShouldNotReachHere();
2192 }
2193 }
2194
2195 if (st) {
2196 st->print("spill ");
2197 if (src_lo_rc == rc_stack) {
2198 st->print("[sp, #%d] -> ", ra_->reg2offset(src_lo));
2199 } else {
2200 st->print("%s -> ", Matcher::regName[src_lo]);
2201 }
2202 if (dst_lo_rc == rc_stack) {
2203 st->print("[sp, #%d]", ra_->reg2offset(dst_lo));
2204 } else {
2205 st->print("%s", Matcher::regName[dst_lo]);
2206 }
2207 if (bottom_type()->isa_vect() != NULL) {
2208 int vsize = 0;
2209 switch (ideal_reg()) {
2210 case Op_VecD:
2211 vsize = 64;
2212 break;
2213 case Op_VecX:
2214 vsize = 128;
2215 break;
2216 case Op_VecA:
2217 vsize = Matcher::scalable_vector_reg_size(T_BYTE) * 8;
2218 break;
2219 default:
2220 assert(false, "bad register type for spill");
2221 ShouldNotReachHere();
2222 }
2223 st->print("\t# vector spill size = %d", vsize);
2224 } else {
2225 st->print("\t# spill size = %d", is64 ? 64 : 32);
2226 }
2227 }
2228
2229 return 0;
2230
2231 }
2232
2233 #ifndef PRODUCT
2234 void MachSpillCopyNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
2235 if (!ra_)
2236 st->print("N%d = SpillCopy(N%d)", _idx, in(1)->_idx);
2237 else
2238 implementation(NULL, ra_, false, st);
2239 }
2240 #endif
2241
2242 void MachSpillCopyNode::emit(CodeBuffer &cbuf, PhaseRegAlloc *ra_) const {
2243 implementation(&cbuf, ra_, false, NULL);
2244 }
2245
2246 uint MachSpillCopyNode::size(PhaseRegAlloc *ra_) const {
2247 return MachNode::size(ra_);
2248 }
2249
2250 //=============================================================================
2251
2252 #ifndef PRODUCT
2253 void BoxLockNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
2254 int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
2255 int reg = ra_->get_reg_first(this);
2256 st->print("add %s, rsp, #%d]\t# box lock",
2257 Matcher::regName[reg], offset);
2258 }
2259 #endif
2260
2261 void BoxLockNode::emit(CodeBuffer &cbuf, PhaseRegAlloc *ra_) const {
2262 C2_MacroAssembler _masm(&cbuf);
2263
2264 int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
2265 int reg = ra_->get_encode(this);
2266
2267 // This add will handle any 24-bit signed offset. 24 bits allows an
2268 // 8 megabyte stack frame.
2269 __ add(as_Register(reg), sp, offset);
2270 }
2271
2272 uint BoxLockNode::size(PhaseRegAlloc *ra_) const {
2273 // BoxLockNode is not a MachNode, so we can't just call MachNode::size(ra_).
2274 int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
2275
2276 if (Assembler::operand_valid_for_add_sub_immediate(offset)) {
2277 return NativeInstruction::instruction_size;
2278 } else {
2279 return 2 * NativeInstruction::instruction_size;
2280 }
2281 }
2282
2283 //=============================================================================
2284
2285 #ifndef PRODUCT
2286 void MachUEPNode::format(PhaseRegAlloc* ra_, outputStream* st) const
2287 {
2288 st->print_cr("# MachUEPNode");
2289 if (UseCompressedClassPointers) {
2290 st->print_cr("\tldrw rscratch1, j_rarg0 + oopDesc::klass_offset_in_bytes()]\t# compressed klass");
2291 if (CompressedKlassPointers::shift() != 0) {
2292 st->print_cr("\tdecode_klass_not_null rscratch1, rscratch1");
2293 }
2294 } else {
2295 st->print_cr("\tldr rscratch1, j_rarg0 + oopDesc::klass_offset_in_bytes()]\t# compressed klass");
2296 }
2297 st->print_cr("\tcmp r0, rscratch1\t # Inline cache check");
2298 st->print_cr("\tbne, SharedRuntime::_ic_miss_stub");
2299 }
2300 #endif
2301
2302 void MachUEPNode::emit(CodeBuffer& cbuf, PhaseRegAlloc* ra_) const
2303 {
2304 // This is the unverified entry point.
2305 C2_MacroAssembler _masm(&cbuf);
2306
2307 __ cmp_klass(j_rarg0, rscratch2, rscratch1);
2308 Label skip;
2309 // TODO
2310 // can we avoid this skip and still use a reloc?
2311 __ br(Assembler::EQ, skip);
2312 __ far_jump(RuntimeAddress(SharedRuntime::get_ic_miss_stub()));
2313 __ bind(skip);
2314 }
2315
2316 uint MachUEPNode::size(PhaseRegAlloc* ra_) const
2317 {
2318 return MachNode::size(ra_);
2319 }
2320
2321 // REQUIRED EMIT CODE
2322
2323 //=============================================================================
2324
2325 // Emit exception handler code.
2326 int HandlerImpl::emit_exception_handler(CodeBuffer& cbuf)
2327 {
2328 // mov rscratch1 #exception_blob_entry_point
2329 // br rscratch1
2330 // Note that the code buffer's insts_mark is always relative to insts.
2331 // That's why we must use the macroassembler to generate a handler.
2332 C2_MacroAssembler _masm(&cbuf);
2333 address base = __ start_a_stub(size_exception_handler());
2334 if (base == NULL) {
2335 ciEnv::current()->record_failure("CodeCache is full");
2336 return 0; // CodeBuffer::expand failed
2337 }
2338 int offset = __ offset();
2339 __ far_jump(RuntimeAddress(OptoRuntime::exception_blob()->entry_point()));
2340 assert(__ offset() - offset <= (int) size_exception_handler(), "overflow");
2341 __ end_a_stub();
2342 return offset;
2343 }
2344
2345 // Emit deopt handler code.
2346 int HandlerImpl::emit_deopt_handler(CodeBuffer& cbuf)
2347 {
2348 // Note that the code buffer's insts_mark is always relative to insts.
2349 // That's why we must use the macroassembler to generate a handler.
2350 C2_MacroAssembler _masm(&cbuf);
2351 address base = __ start_a_stub(size_deopt_handler());
2352 if (base == NULL) {
2353 ciEnv::current()->record_failure("CodeCache is full");
2354 return 0; // CodeBuffer::expand failed
2355 }
2356 int offset = __ offset();
2357
2358 __ adr(lr, __ pc());
2359 __ far_jump(RuntimeAddress(SharedRuntime::deopt_blob()->unpack()));
2360
2361 assert(__ offset() - offset == (int) size_deopt_handler(), "overflow");
2362 __ end_a_stub();
2363 return offset;
2364 }
2365
2366 // REQUIRED MATCHER CODE
2367
2368 //=============================================================================
2369
2370 const bool Matcher::match_rule_supported(int opcode) {
2371 if (!has_match_rule(opcode))
2372 return false;
2373
2374 bool ret_value = true;
2375 switch (opcode) {
2376 case Op_OnSpinWait:
2377 return VM_Version::supports_on_spin_wait();
2378 case Op_CacheWB:
2379 case Op_CacheWBPreSync:
2380 case Op_CacheWBPostSync:
2381 if (!VM_Version::supports_data_cache_line_flush()) {
2382 ret_value = false;
2383 }
2384 break;
2385 }
2386
2387 return ret_value; // Per default match rules are supported.
2388 }
2389
2390 // Identify extra cases that we might want to provide match rules for vector nodes and
2391 // other intrinsics guarded with vector length (vlen) and element type (bt).
2392 const bool Matcher::match_rule_supported_vector(int opcode, int vlen, BasicType bt) {
2393 if (!match_rule_supported(opcode) || !vector_size_supported(bt, vlen)) {
2394 return false;
2395 }
2396 int bit_size = vlen * type2aelembytes(bt) * 8;
2397 if (UseSVE == 0 && bit_size > 128) {
2398 return false;
2399 }
2400 if (UseSVE > 0) {
2401 return op_sve_supported(opcode);
2402 } else { // NEON
2403 // Special cases
2404 switch (opcode) {
2405 case Op_VectorMaskCmp:
2406 // We don't have VectorReinterpret with bit_size less than 64 support for
2407 // now, even for byte type. To be refined with fully VectorCast support.
2408 case Op_VectorReinterpret:
2409 if (vlen < 2 || bit_size < 64) {
2410 return false;
2411 }
2412 break;
2413 case Op_MulAddVS2VI:
2414 if (bit_size < 128) {
2415 return false;
2416 }
2417 break;
2418 case Op_MulVL:
2419 return false;
2420 case Op_VectorLoadShuffle:
2421 case Op_VectorRearrange:
2422 if (vlen < 4) {
2423 return false;
2424 }
2425 break;
2426 // Some types of VectorCast are not implemented for now.
2427 case Op_VectorCastI2X:
2428 if (bt == T_BYTE) {
2429 return false;
2430 }
2431 break;
2432 case Op_VectorCastS2X:
2433 if (vlen < 4 || bit_size < 64) {
2434 return false;
2435 }
2436 break;
2437 case Op_VectorCastF2X:
2438 case Op_VectorCastD2X:
2439 if (bt == T_INT || bt == T_SHORT || bt == T_BYTE || bt == T_LONG) {
2440 return false;
2441 }
2442 break;
2443 default:
2444 break;
2445 }
2446 }
2447 return true; // Per default match rules are supported.
2448 }
2449
2450 const RegMask* Matcher::predicate_reg_mask(void) {
2451 return &_PR_REG_mask;
2452 }
2453
2454 const TypeVect* Matcher::predicate_reg_type(const Type* elemTy, int length) {
2455 return new TypeVectMask(elemTy, length);
2456 }
2457
2458 // Vector calling convention not yet implemented.
2459 const bool Matcher::supports_vector_calling_convention(void) {
2460 return false;
2461 }
2462
2463 OptoRegPair Matcher::vector_return_value(uint ideal_reg) {
2464 Unimplemented();
2465 return OptoRegPair(0, 0);
2466 }
2467
2468 const int Matcher::float_pressure(int default_pressure_threshold) {
2469 return default_pressure_threshold;
2470 }
2471
2472 // Is this branch offset short enough that a short branch can be used?
2473 //
2474 // NOTE: If the platform does not provide any short branch variants, then
2475 // this method should return false for offset 0.
2476 bool Matcher::is_short_branch_offset(int rule, int br_size, int offset) {
2477 // The passed offset is relative to address of the branch.
2478
2479 return (-32768 <= offset && offset < 32768);
2480 }
2481
2482 // Vector width in bytes.
2483 const int Matcher::vector_width_in_bytes(BasicType bt) {
2484 // The MaxVectorSize should have been set by detecting SVE max vector register size.
2485 int size = MIN2((UseSVE > 0) ? 256 : 16, (int)MaxVectorSize);
2486 // Minimum 2 values in vector
2487 if (size < 2*type2aelembytes(bt)) size = 0;
2488 // But never < 4
2489 if (size < 4) size = 0;
2490 return size;
2491 }
2492
2493 // Limits on vector size (number of elements) loaded into vector.
2494 const int Matcher::max_vector_size(const BasicType bt) {
2495 return vector_width_in_bytes(bt)/type2aelembytes(bt);
2496 }
2497 const int Matcher::min_vector_size(const BasicType bt) {
2498 int max_size = max_vector_size(bt);
2499 if ((UseSVE > 0) && (MaxVectorSize >= 16)) {
2500 // Currently vector length less than SVE vector register size is not supported.
2501 return max_size;
2502 } else { // NEON
2503 // Limit the vector size to 8 bytes
2504 int size = 8 / type2aelembytes(bt);
2505 if (bt == T_BYTE) {
2506 // To support vector api shuffle/rearrange.
2507 size = 4;
2508 } else if (bt == T_BOOLEAN) {
2509 // To support vector api load/store mask.
2510 size = 2;
2511 }
2512 if (size < 2) size = 2;
2513 return MIN2(size,max_size);
2514 }
2515 }
2516
2517 // Actual max scalable vector register length.
2518 const int Matcher::scalable_vector_reg_size(const BasicType bt) {
2519 return Matcher::max_vector_size(bt);
2520 }
2521
2522 // Vector ideal reg.
2523 const uint Matcher::vector_ideal_reg(int len) {
2524 if (UseSVE > 0 && 16 <= len && len <= 256) {
2525 return Op_VecA;
2526 }
2527 switch(len) {
2528 // For 16-bit/32-bit mask vector, reuse VecD.
2529 case 2:
2530 case 4:
2531 case 8: return Op_VecD;
2532 case 16: return Op_VecX;
2533 }
2534 ShouldNotReachHere();
2535 return 0;
2536 }
2537
2538 MachOper* Matcher::pd_specialize_generic_vector_operand(MachOper* original_opnd, uint ideal_reg, bool is_temp) {
2539 ShouldNotReachHere(); // generic vector operands not supported
2540 return NULL;
2541 }
2542
2543 bool Matcher::is_generic_reg2reg_move(MachNode* m) {
2544 ShouldNotReachHere(); // generic vector operands not supported
2545 return false;
2546 }
2547
2548 bool Matcher::is_generic_vector(MachOper* opnd) {
2549 ShouldNotReachHere(); // generic vector operands not supported
2550 return false;
2551 }
2552
2553 // Return whether or not this register is ever used as an argument.
2554 // This function is used on startup to build the trampoline stubs in
2555 // generateOptoStub. Registers not mentioned will be killed by the VM
2556 // call in the trampoline, and arguments in those registers not be
2557 // available to the callee.
2558 bool Matcher::can_be_java_arg(int reg)
2559 {
2560 return
2561 reg == R0_num || reg == R0_H_num ||
2562 reg == R1_num || reg == R1_H_num ||
2563 reg == R2_num || reg == R2_H_num ||
2564 reg == R3_num || reg == R3_H_num ||
2565 reg == R4_num || reg == R4_H_num ||
2566 reg == R5_num || reg == R5_H_num ||
2567 reg == R6_num || reg == R6_H_num ||
2568 reg == R7_num || reg == R7_H_num ||
2569 reg == V0_num || reg == V0_H_num ||
2570 reg == V1_num || reg == V1_H_num ||
2571 reg == V2_num || reg == V2_H_num ||
2572 reg == V3_num || reg == V3_H_num ||
2573 reg == V4_num || reg == V4_H_num ||
2574 reg == V5_num || reg == V5_H_num ||
2575 reg == V6_num || reg == V6_H_num ||
2576 reg == V7_num || reg == V7_H_num;
2577 }
2578
2579 bool Matcher::is_spillable_arg(int reg)
2580 {
2581 return can_be_java_arg(reg);
2582 }
2583
2584 bool Matcher::use_asm_for_ldiv_by_con(jlong divisor) {
2585 return false;
2586 }
2587
2588 RegMask Matcher::divI_proj_mask() {
2589 ShouldNotReachHere();
2590 return RegMask();
2591 }
2592
2593 // Register for MODI projection of divmodI.
2594 RegMask Matcher::modI_proj_mask() {
2595 ShouldNotReachHere();
2596 return RegMask();
2597 }
2598
2599 // Register for DIVL projection of divmodL.
2600 RegMask Matcher::divL_proj_mask() {
2601 ShouldNotReachHere();
2602 return RegMask();
2603 }
2604
2605 // Register for MODL projection of divmodL.
2606 RegMask Matcher::modL_proj_mask() {
2607 ShouldNotReachHere();
2608 return RegMask();
2609 }
2610
2611 const RegMask Matcher::method_handle_invoke_SP_save_mask() {
2612 return FP_REG_mask();
2613 }
2614
2615 bool size_fits_all_mem_uses(AddPNode* addp, int shift) {
2616 for (DUIterator_Fast imax, i = addp->fast_outs(imax); i < imax; i++) {
2617 Node* u = addp->fast_out(i);
2618 if (u->is_LoadStore()) {
2619 // On AArch64, LoadStoreNodes (i.e. compare and swap
2620 // instructions) only take register indirect as an operand, so
2621 // any attempt to use an AddPNode as an input to a LoadStoreNode
2622 // must fail.
2623 return false;
2624 }
2625 if (u->is_Mem()) {
2626 int opsize = u->as_Mem()->memory_size();
2627 assert(opsize > 0, "unexpected memory operand size");
2628 if (u->as_Mem()->memory_size() != (1<<shift)) {
2629 return false;
2630 }
2631 }
2632 }
2633 return true;
2634 }
2635
2636 // Should the matcher clone input 'm' of node 'n'?
2637 bool Matcher::pd_clone_node(Node* n, Node* m, Matcher::MStack& mstack) {
2638 if (is_vshift_con_pattern(n, m)) { // ShiftV src (ShiftCntV con)
2639 mstack.push(m, Visit); // m = ShiftCntV
2640 return true;
2641 }
2642 return false;
2643 }
2644
2645 // Should the Matcher clone shifts on addressing modes, expecting them
2646 // to be subsumed into complex addressing expressions or compute them
2647 // into registers?
2648 bool Matcher::pd_clone_address_expressions(AddPNode* m, Matcher::MStack& mstack, VectorSet& address_visited) {
2649 if (clone_base_plus_offset_address(m, mstack, address_visited)) {
2650 return true;
2651 }
2652
2653 Node *off = m->in(AddPNode::Offset);
2654 if (off->Opcode() == Op_LShiftL && off->in(2)->is_Con() &&
2655 size_fits_all_mem_uses(m, off->in(2)->get_int()) &&
2656 // Are there other uses besides address expressions?
2657 !is_visited(off)) {
2658 address_visited.set(off->_idx); // Flag as address_visited
2659 mstack.push(off->in(2), Visit);
2660 Node *conv = off->in(1);
2661 if (conv->Opcode() == Op_ConvI2L &&
2662 // Are there other uses besides address expressions?
2663 !is_visited(conv)) {
2664 address_visited.set(conv->_idx); // Flag as address_visited
2665 mstack.push(conv->in(1), Pre_Visit);
2666 } else {
2667 mstack.push(conv, Pre_Visit);
2668 }
2669 address_visited.test_set(m->_idx); // Flag as address_visited
2670 mstack.push(m->in(AddPNode::Address), Pre_Visit);
2671 mstack.push(m->in(AddPNode::Base), Pre_Visit);
2672 return true;
2673 } else if (off->Opcode() == Op_ConvI2L &&
2674 // Are there other uses besides address expressions?
2675 !is_visited(off)) {
2676 address_visited.test_set(m->_idx); // Flag as address_visited
2677 address_visited.set(off->_idx); // Flag as address_visited
2678 mstack.push(off->in(1), Pre_Visit);
2679 mstack.push(m->in(AddPNode::Address), Pre_Visit);
2680 mstack.push(m->in(AddPNode::Base), Pre_Visit);
2681 return true;
2682 }
2683 return false;
2684 }
2685
2686 #define MOV_VOLATILE(REG, BASE, INDEX, SCALE, DISP, SCRATCH, INSN) \
2687 C2_MacroAssembler _masm(&cbuf); \
2688 { \
2689 guarantee(INDEX == -1, "mode not permitted for volatile"); \
2690 guarantee(DISP == 0, "mode not permitted for volatile"); \
2691 guarantee(SCALE == 0, "mode not permitted for volatile"); \
2692 __ INSN(REG, as_Register(BASE)); \
2693 }
2694
2695
2696 static Address mem2address(int opcode, Register base, int index, int size, int disp)
2697 {
2698 Address::extend scale;
2699
2700 // Hooboy, this is fugly. We need a way to communicate to the
2701 // encoder that the index needs to be sign extended, so we have to
2702 // enumerate all the cases.
2703 switch (opcode) {
2704 case INDINDEXSCALEDI2L:
2705 case INDINDEXSCALEDI2LN:
2706 case INDINDEXI2L:
2707 case INDINDEXI2LN:
2708 scale = Address::sxtw(size);
2709 break;
2710 default:
2711 scale = Address::lsl(size);
2712 }
2713
2714 if (index == -1) {
2715 return Address(base, disp);
2716 } else {
2717 assert(disp == 0, "unsupported address mode: disp = %d", disp);
2718 return Address(base, as_Register(index), scale);
2719 }
2720 }
2721
2722
2723 typedef void (MacroAssembler::* mem_insn)(Register Rt, const Address &adr);
2724 typedef void (MacroAssembler::* mem_insn2)(Register Rt, Register adr);
2725 typedef void (MacroAssembler::* mem_float_insn)(FloatRegister Rt, const Address &adr);
2726 typedef void (MacroAssembler::* mem_vector_insn)(FloatRegister Rt,
2727 MacroAssembler::SIMD_RegVariant T, const Address &adr);
2728
2729 // Used for all non-volatile memory accesses. The use of
2730 // $mem->opcode() to discover whether this pattern uses sign-extended
2731 // offsets is something of a kludge.
2732 static void loadStore(C2_MacroAssembler masm, mem_insn insn,
2733 Register reg, int opcode,
2734 Register base, int index, int scale, int disp,
2735 int size_in_memory)
2736 {
2737 Address addr = mem2address(opcode, base, index, scale, disp);
2738 if (addr.getMode() == Address::base_plus_offset) {
2739 /* If we get an out-of-range offset it is a bug in the compiler,
2740 so we assert here. */
2741 assert(Address::offset_ok_for_immed(addr.offset(), exact_log2(size_in_memory)),
2742 "c2 compiler bug");
2743 /* Fix up any out-of-range offsets. */
2744 assert_different_registers(rscratch1, base);
2745 assert_different_registers(rscratch1, reg);
2746 addr = masm.legitimize_address(addr, size_in_memory, rscratch1);
2747 }
2748 (masm.*insn)(reg, addr);
2749 }
2750
2751 static void loadStore(C2_MacroAssembler masm, mem_float_insn insn,
2752 FloatRegister reg, int opcode,
2753 Register base, int index, int size, int disp,
2754 int size_in_memory)
2755 {
2756 Address::extend scale;
2757
2758 switch (opcode) {
2759 case INDINDEXSCALEDI2L:
2760 case INDINDEXSCALEDI2LN:
2761 scale = Address::sxtw(size);
2762 break;
2763 default:
2764 scale = Address::lsl(size);
2765 }
2766
2767 if (index == -1) {
2768 /* If we get an out-of-range offset it is a bug in the compiler,
2769 so we assert here. */
2770 assert(Address::offset_ok_for_immed(disp, exact_log2(size_in_memory)), "c2 compiler bug");
2771 /* Fix up any out-of-range offsets. */
2772 assert_different_registers(rscratch1, base);
2773 Address addr = Address(base, disp);
2774 addr = masm.legitimize_address(addr, size_in_memory, rscratch1);
2775 (masm.*insn)(reg, addr);
2776 } else {
2777 assert(disp == 0, "unsupported address mode: disp = %d", disp);
2778 (masm.*insn)(reg, Address(base, as_Register(index), scale));
2779 }
2780 }
2781
2782 static void loadStore(C2_MacroAssembler masm, mem_vector_insn insn,
2783 FloatRegister reg, MacroAssembler::SIMD_RegVariant T,
2784 int opcode, Register base, int index, int size, int disp)
2785 {
2786 if (index == -1) {
2787 (masm.*insn)(reg, T, Address(base, disp));
2788 } else {
2789 assert(disp == 0, "unsupported address mode");
2790 (masm.*insn)(reg, T, Address(base, as_Register(index), Address::lsl(size)));
2791 }
2792 }
2793
2794 %}
2795
2796
2797
2798 //----------ENCODING BLOCK-----------------------------------------------------
2799 // This block specifies the encoding classes used by the compiler to
2800 // output byte streams. Encoding classes are parameterized macros
2801 // used by Machine Instruction Nodes in order to generate the bit
2802 // encoding of the instruction. Operands specify their base encoding
2803 // interface with the interface keyword. There are currently
2804 // supported four interfaces, REG_INTER, CONST_INTER, MEMORY_INTER, &
2805 // COND_INTER. REG_INTER causes an operand to generate a function
2806 // which returns its register number when queried. CONST_INTER causes
2807 // an operand to generate a function which returns the value of the
2808 // constant when queried. MEMORY_INTER causes an operand to generate
2809 // four functions which return the Base Register, the Index Register,
2810 // the Scale Value, and the Offset Value of the operand when queried.
2811 // COND_INTER causes an operand to generate six functions which return
2812 // the encoding code (ie - encoding bits for the instruction)
2813 // associated with each basic boolean condition for a conditional
2814 // instruction.
2815 //
2816 // Instructions specify two basic values for encoding. Again, a
2817 // function is available to check if the constant displacement is an
2818 // oop. They use the ins_encode keyword to specify their encoding
2819 // classes (which must be a sequence of enc_class names, and their
2820 // parameters, specified in the encoding block), and they use the
2821 // opcode keyword to specify, in order, their primary, secondary, and
2822 // tertiary opcode. Only the opcode sections which a particular
2823 // instruction needs for encoding need to be specified.
2824 encode %{
2825 // Build emit functions for each basic byte or larger field in the
2826 // intel encoding scheme (opcode, rm, sib, immediate), and call them
2827 // from C++ code in the enc_class source block. Emit functions will
2828 // live in the main source block for now. In future, we can
2829 // generalize this by adding a syntax that specifies the sizes of
2830 // fields in an order, so that the adlc can build the emit functions
2831 // automagically
2832
2833 // catch all for unimplemented encodings
2834 enc_class enc_unimplemented %{
2835 C2_MacroAssembler _masm(&cbuf);
2836 __ unimplemented("C2 catch all");
2837 %}
2838
2839 // BEGIN Non-volatile memory access
2840
2841 // This encoding class is generated automatically from ad_encode.m4.
2842 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2843 enc_class aarch64_enc_ldrsbw(iRegI dst, memory1 mem) %{
2844 Register dst_reg = as_Register($dst$$reg);
2845 loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrsbw, dst_reg, $mem->opcode(),
2846 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
2847 %}
2848
2849 // This encoding class is generated automatically from ad_encode.m4.
2850 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2851 enc_class aarch64_enc_ldrsb(iRegI dst, memory1 mem) %{
2852 Register dst_reg = as_Register($dst$$reg);
2853 loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrsb, dst_reg, $mem->opcode(),
2854 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
2855 %}
2856
2857 // This encoding class is generated automatically from ad_encode.m4.
2858 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2859 enc_class aarch64_enc_ldrb(iRegI dst, memory1 mem) %{
2860 Register dst_reg = as_Register($dst$$reg);
2861 loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrb, dst_reg, $mem->opcode(),
2862 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
2863 %}
2864
2865 // This encoding class is generated automatically from ad_encode.m4.
2866 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2867 enc_class aarch64_enc_ldrb(iRegL dst, memory1 mem) %{
2868 Register dst_reg = as_Register($dst$$reg);
2869 loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrb, dst_reg, $mem->opcode(),
2870 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
2871 %}
2872
2873 // This encoding class is generated automatically from ad_encode.m4.
2874 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2875 enc_class aarch64_enc_ldrshw(iRegI dst, memory2 mem) %{
2876 Register dst_reg = as_Register($dst$$reg);
2877 loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrshw, dst_reg, $mem->opcode(),
2878 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
2879 %}
2880
2881 // This encoding class is generated automatically from ad_encode.m4.
2882 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2883 enc_class aarch64_enc_ldrsh(iRegI dst, memory2 mem) %{
2884 Register dst_reg = as_Register($dst$$reg);
2885 loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrsh, dst_reg, $mem->opcode(),
2886 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
2887 %}
2888
2889 // This encoding class is generated automatically from ad_encode.m4.
2890 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2891 enc_class aarch64_enc_ldrh(iRegI dst, memory2 mem) %{
2892 Register dst_reg = as_Register($dst$$reg);
2893 loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrh, dst_reg, $mem->opcode(),
2894 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
2895 %}
2896
2897 // This encoding class is generated automatically from ad_encode.m4.
2898 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2899 enc_class aarch64_enc_ldrh(iRegL dst, memory2 mem) %{
2900 Register dst_reg = as_Register($dst$$reg);
2901 loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrh, dst_reg, $mem->opcode(),
2902 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
2903 %}
2904
2905 // This encoding class is generated automatically from ad_encode.m4.
2906 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2907 enc_class aarch64_enc_ldrw(iRegI dst, memory4 mem) %{
2908 Register dst_reg = as_Register($dst$$reg);
2909 loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrw, dst_reg, $mem->opcode(),
2910 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
2911 %}
2912
2913 // This encoding class is generated automatically from ad_encode.m4.
2914 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2915 enc_class aarch64_enc_ldrw(iRegL dst, memory4 mem) %{
2916 Register dst_reg = as_Register($dst$$reg);
2917 loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrw, dst_reg, $mem->opcode(),
2918 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
2919 %}
2920
2921 // This encoding class is generated automatically from ad_encode.m4.
2922 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2923 enc_class aarch64_enc_ldrsw(iRegL dst, memory4 mem) %{
2924 Register dst_reg = as_Register($dst$$reg);
2925 loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrsw, dst_reg, $mem->opcode(),
2926 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
2927 %}
2928
2929 // This encoding class is generated automatically from ad_encode.m4.
2930 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2931 enc_class aarch64_enc_ldr(iRegL dst, memory8 mem) %{
2932 Register dst_reg = as_Register($dst$$reg);
2933 loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldr, dst_reg, $mem->opcode(),
2934 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
2935 %}
2936
2937 // This encoding class is generated automatically from ad_encode.m4.
2938 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2939 enc_class aarch64_enc_ldrs(vRegF dst, memory4 mem) %{
2940 FloatRegister dst_reg = as_FloatRegister($dst$$reg);
2941 loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrs, dst_reg, $mem->opcode(),
2942 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
2943 %}
2944
2945 // This encoding class is generated automatically from ad_encode.m4.
2946 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2947 enc_class aarch64_enc_ldrd(vRegD dst, memory8 mem) %{
2948 FloatRegister dst_reg = as_FloatRegister($dst$$reg);
2949 loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrd, dst_reg, $mem->opcode(),
2950 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
2951 %}
2952
2953 // This encoding class is generated automatically from ad_encode.m4.
2954 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2955 enc_class aarch64_enc_strb(iRegI src, memory1 mem) %{
2956 Register src_reg = as_Register($src$$reg);
2957 loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::strb, src_reg, $mem->opcode(),
2958 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
2959 %}
2960
2961 // This encoding class is generated automatically from ad_encode.m4.
2962 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2963 enc_class aarch64_enc_strb0(memory1 mem) %{
2964 C2_MacroAssembler _masm(&cbuf);
2965 loadStore(_masm, &MacroAssembler::strb, zr, $mem->opcode(),
2966 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
2967 %}
2968
2969 // This encoding class is generated automatically from ad_encode.m4.
2970 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2971 enc_class aarch64_enc_strh(iRegI src, memory2 mem) %{
2972 Register src_reg = as_Register($src$$reg);
2973 loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::strh, src_reg, $mem->opcode(),
2974 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
2975 %}
2976
2977 // This encoding class is generated automatically from ad_encode.m4.
2978 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2979 enc_class aarch64_enc_strh0(memory2 mem) %{
2980 C2_MacroAssembler _masm(&cbuf);
2981 loadStore(_masm, &MacroAssembler::strh, zr, $mem->opcode(),
2982 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
2983 %}
2984
2985 // This encoding class is generated automatically from ad_encode.m4.
2986 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2987 enc_class aarch64_enc_strw(iRegI src, memory4 mem) %{
2988 Register src_reg = as_Register($src$$reg);
2989 loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::strw, src_reg, $mem->opcode(),
2990 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
2991 %}
2992
2993 // This encoding class is generated automatically from ad_encode.m4.
2994 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2995 enc_class aarch64_enc_strw0(memory4 mem) %{
2996 C2_MacroAssembler _masm(&cbuf);
2997 loadStore(_masm, &MacroAssembler::strw, zr, $mem->opcode(),
2998 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
2999 %}
3000
3001 // This encoding class is generated automatically from ad_encode.m4.
3002 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
3003 enc_class aarch64_enc_str(iRegL src, memory8 mem) %{
3004 Register src_reg = as_Register($src$$reg);
3005 // we sometimes get asked to store the stack pointer into the
3006 // current thread -- we cannot do that directly on AArch64
3007 if (src_reg == r31_sp) {
3008 C2_MacroAssembler _masm(&cbuf);
3009 assert(as_Register($mem$$base) == rthread, "unexpected store for sp");
3010 __ mov(rscratch2, sp);
3011 src_reg = rscratch2;
3012 }
3013 loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::str, src_reg, $mem->opcode(),
3014 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
3015 %}
3016
3017 // This encoding class is generated automatically from ad_encode.m4.
3018 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
3019 enc_class aarch64_enc_str0(memory8 mem) %{
3020 C2_MacroAssembler _masm(&cbuf);
3021 loadStore(_masm, &MacroAssembler::str, zr, $mem->opcode(),
3022 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
3023 %}
3024
3025 // This encoding class is generated automatically from ad_encode.m4.
3026 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
3027 enc_class aarch64_enc_strs(vRegF src, memory4 mem) %{
3028 FloatRegister src_reg = as_FloatRegister($src$$reg);
3029 loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::strs, src_reg, $mem->opcode(),
3030 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
3031 %}
3032
3033 // This encoding class is generated automatically from ad_encode.m4.
3034 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
3035 enc_class aarch64_enc_strd(vRegD src, memory8 mem) %{
3036 FloatRegister src_reg = as_FloatRegister($src$$reg);
3037 loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::strd, src_reg, $mem->opcode(),
3038 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
3039 %}
3040
3041 // This encoding class is generated automatically from ad_encode.m4.
3042 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
3043 enc_class aarch64_enc_strb0_ordered(memory4 mem) %{
3044 C2_MacroAssembler _masm(&cbuf);
3045 __ membar(Assembler::StoreStore);
3046 loadStore(_masm, &MacroAssembler::strb, zr, $mem->opcode(),
3047 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
3048 %}
3049
3050 // END Non-volatile memory access
3051
3052 // Vector loads and stores
3053 enc_class aarch64_enc_ldrvH(vecD dst, memory mem) %{
3054 FloatRegister dst_reg = as_FloatRegister($dst$$reg);
3055 loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldr, dst_reg, MacroAssembler::H,
3056 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3057 %}
3058
3059 enc_class aarch64_enc_ldrvS(vecD dst, memory mem) %{
3060 FloatRegister dst_reg = as_FloatRegister($dst$$reg);
3061 loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldr, dst_reg, MacroAssembler::S,
3062 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3063 %}
3064
3065 enc_class aarch64_enc_ldrvD(vecD dst, memory mem) %{
3066 FloatRegister dst_reg = as_FloatRegister($dst$$reg);
3067 loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldr, dst_reg, MacroAssembler::D,
3068 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3069 %}
3070
3071 enc_class aarch64_enc_ldrvQ(vecX dst, memory mem) %{
3072 FloatRegister dst_reg = as_FloatRegister($dst$$reg);
3073 loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldr, dst_reg, MacroAssembler::Q,
3074 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3075 %}
3076
3077 enc_class aarch64_enc_strvH(vecD src, memory mem) %{
3078 FloatRegister src_reg = as_FloatRegister($src$$reg);
3079 loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::str, src_reg, MacroAssembler::H,
3080 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3081 %}
3082
3083 enc_class aarch64_enc_strvS(vecD src, memory mem) %{
3084 FloatRegister src_reg = as_FloatRegister($src$$reg);
3085 loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::str, src_reg, MacroAssembler::S,
3086 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3087 %}
3088
3089 enc_class aarch64_enc_strvD(vecD src, memory mem) %{
3090 FloatRegister src_reg = as_FloatRegister($src$$reg);
3091 loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::str, src_reg, MacroAssembler::D,
3092 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3093 %}
3094
3095 enc_class aarch64_enc_strvQ(vecX src, memory mem) %{
3096 FloatRegister src_reg = as_FloatRegister($src$$reg);
3097 loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::str, src_reg, MacroAssembler::Q,
3098 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3099 %}
3100
3101 // volatile loads and stores
3102
3103 enc_class aarch64_enc_stlrb(iRegI src, memory mem) %{
3104 MOV_VOLATILE(as_Register($src$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3105 rscratch1, stlrb);
3106 %}
3107
3108 enc_class aarch64_enc_stlrb0(memory mem) %{
3109 MOV_VOLATILE(zr, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3110 rscratch1, stlrb);
3111 %}
3112
3113 enc_class aarch64_enc_stlrh(iRegI src, memory mem) %{
3114 MOV_VOLATILE(as_Register($src$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3115 rscratch1, stlrh);
3116 %}
3117
3118 enc_class aarch64_enc_stlrh0(memory mem) %{
3119 MOV_VOLATILE(zr, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3120 rscratch1, stlrh);
3121 %}
3122
3123 enc_class aarch64_enc_stlrw(iRegI src, memory mem) %{
3124 MOV_VOLATILE(as_Register($src$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3125 rscratch1, stlrw);
3126 %}
3127
3128 enc_class aarch64_enc_stlrw0(memory mem) %{
3129 MOV_VOLATILE(zr, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3130 rscratch1, stlrw);
3131 %}
3132
3133 enc_class aarch64_enc_ldarsbw(iRegI dst, memory mem) %{
3134 Register dst_reg = as_Register($dst$$reg);
3135 MOV_VOLATILE(dst_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3136 rscratch1, ldarb);
3137 __ sxtbw(dst_reg, dst_reg);
3138 %}
3139
3140 enc_class aarch64_enc_ldarsb(iRegL dst, memory mem) %{
3141 Register dst_reg = as_Register($dst$$reg);
3142 MOV_VOLATILE(dst_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3143 rscratch1, ldarb);
3144 __ sxtb(dst_reg, dst_reg);
3145 %}
3146
3147 enc_class aarch64_enc_ldarbw(iRegI dst, memory mem) %{
3148 MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3149 rscratch1, ldarb);
3150 %}
3151
3152 enc_class aarch64_enc_ldarb(iRegL dst, memory mem) %{
3153 MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3154 rscratch1, ldarb);
3155 %}
3156
3157 enc_class aarch64_enc_ldarshw(iRegI dst, memory mem) %{
3158 Register dst_reg = as_Register($dst$$reg);
3159 MOV_VOLATILE(dst_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3160 rscratch1, ldarh);
3161 __ sxthw(dst_reg, dst_reg);
3162 %}
3163
3164 enc_class aarch64_enc_ldarsh(iRegL dst, memory mem) %{
3165 Register dst_reg = as_Register($dst$$reg);
3166 MOV_VOLATILE(dst_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3167 rscratch1, ldarh);
3168 __ sxth(dst_reg, dst_reg);
3169 %}
3170
3171 enc_class aarch64_enc_ldarhw(iRegI dst, memory mem) %{
3172 MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3173 rscratch1, ldarh);
3174 %}
3175
3176 enc_class aarch64_enc_ldarh(iRegL dst, memory mem) %{
3177 MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3178 rscratch1, ldarh);
3179 %}
3180
3181 enc_class aarch64_enc_ldarw(iRegI dst, memory mem) %{
3182 MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3183 rscratch1, ldarw);
3184 %}
3185
3186 enc_class aarch64_enc_ldarw(iRegL dst, memory mem) %{
3187 MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3188 rscratch1, ldarw);
3189 %}
3190
3191 enc_class aarch64_enc_ldar(iRegL dst, memory mem) %{
3192 MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3193 rscratch1, ldar);
3194 %}
3195
3196 enc_class aarch64_enc_fldars(vRegF dst, memory mem) %{
3197 MOV_VOLATILE(rscratch1, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3198 rscratch1, ldarw);
3199 __ fmovs(as_FloatRegister($dst$$reg), rscratch1);
3200 %}
3201
3202 enc_class aarch64_enc_fldard(vRegD dst, memory mem) %{
3203 MOV_VOLATILE(rscratch1, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3204 rscratch1, ldar);
3205 __ fmovd(as_FloatRegister($dst$$reg), rscratch1);
3206 %}
3207
3208 enc_class aarch64_enc_stlr(iRegL src, memory mem) %{
3209 Register src_reg = as_Register($src$$reg);
3210 // we sometimes get asked to store the stack pointer into the
3211 // current thread -- we cannot do that directly on AArch64
3212 if (src_reg == r31_sp) {
3213 C2_MacroAssembler _masm(&cbuf);
3214 assert(as_Register($mem$$base) == rthread, "unexpected store for sp");
3215 __ mov(rscratch2, sp);
3216 src_reg = rscratch2;
3217 }
3218 MOV_VOLATILE(src_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3219 rscratch1, stlr);
3220 %}
3221
3222 enc_class aarch64_enc_stlr0(memory mem) %{
3223 MOV_VOLATILE(zr, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3224 rscratch1, stlr);
3225 %}
3226
3227 enc_class aarch64_enc_fstlrs(vRegF src, memory mem) %{
3228 {
3229 C2_MacroAssembler _masm(&cbuf);
3230 FloatRegister src_reg = as_FloatRegister($src$$reg);
3231 __ fmovs(rscratch2, src_reg);
3232 }
3233 MOV_VOLATILE(rscratch2, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3234 rscratch1, stlrw);
3235 %}
3236
3237 enc_class aarch64_enc_fstlrd(vRegD src, memory mem) %{
3238 {
3239 C2_MacroAssembler _masm(&cbuf);
3240 FloatRegister src_reg = as_FloatRegister($src$$reg);
3241 __ fmovd(rscratch2, src_reg);
3242 }
3243 MOV_VOLATILE(rscratch2, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3244 rscratch1, stlr);
3245 %}
3246
3247 // synchronized read/update encodings
3248
3249 enc_class aarch64_enc_ldaxr(iRegL dst, memory8 mem) %{
3250 C2_MacroAssembler _masm(&cbuf);
3251 Register dst_reg = as_Register($dst$$reg);
3252 Register base = as_Register($mem$$base);
3253 int index = $mem$$index;
3254 int scale = $mem$$scale;
3255 int disp = $mem$$disp;
3256 if (index == -1) {
3257 if (disp != 0) {
3258 __ lea(rscratch1, Address(base, disp));
3259 __ ldaxr(dst_reg, rscratch1);
3260 } else {
3261 // TODO
3262 // should we ever get anything other than this case?
3263 __ ldaxr(dst_reg, base);
3264 }
3265 } else {
3266 Register index_reg = as_Register(index);
3267 if (disp == 0) {
3268 __ lea(rscratch1, Address(base, index_reg, Address::lsl(scale)));
3269 __ ldaxr(dst_reg, rscratch1);
3270 } else {
3271 __ lea(rscratch1, Address(base, disp));
3272 __ lea(rscratch1, Address(rscratch1, index_reg, Address::lsl(scale)));
3273 __ ldaxr(dst_reg, rscratch1);
3274 }
3275 }
3276 %}
3277
3278 enc_class aarch64_enc_stlxr(iRegLNoSp src, memory8 mem) %{
3279 C2_MacroAssembler _masm(&cbuf);
3280 Register src_reg = as_Register($src$$reg);
3281 Register base = as_Register($mem$$base);
3282 int index = $mem$$index;
3283 int scale = $mem$$scale;
3284 int disp = $mem$$disp;
3285 if (index == -1) {
3286 if (disp != 0) {
3287 __ lea(rscratch2, Address(base, disp));
3288 __ stlxr(rscratch1, src_reg, rscratch2);
3289 } else {
3290 // TODO
3291 // should we ever get anything other than this case?
3292 __ stlxr(rscratch1, src_reg, base);
3293 }
3294 } else {
3295 Register index_reg = as_Register(index);
3296 if (disp == 0) {
3297 __ lea(rscratch2, Address(base, index_reg, Address::lsl(scale)));
3298 __ stlxr(rscratch1, src_reg, rscratch2);
3299 } else {
3300 __ lea(rscratch2, Address(base, disp));
3301 __ lea(rscratch2, Address(rscratch2, index_reg, Address::lsl(scale)));
3302 __ stlxr(rscratch1, src_reg, rscratch2);
3303 }
3304 }
3305 __ cmpw(rscratch1, zr);
3306 %}
3307
3308 enc_class aarch64_enc_cmpxchg(memory mem, iRegLNoSp oldval, iRegLNoSp newval) %{
3309 C2_MacroAssembler _masm(&cbuf);
3310 guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
3311 __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
3312 Assembler::xword, /*acquire*/ false, /*release*/ true,
3313 /*weak*/ false, noreg);
3314 %}
3315
3316 enc_class aarch64_enc_cmpxchgw(memory mem, iRegINoSp oldval, iRegINoSp newval) %{
3317 C2_MacroAssembler _masm(&cbuf);
3318 guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
3319 __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
3320 Assembler::word, /*acquire*/ false, /*release*/ true,
3321 /*weak*/ false, noreg);
3322 %}
3323
3324 enc_class aarch64_enc_cmpxchgs(memory mem, iRegINoSp oldval, iRegINoSp newval) %{
3325 C2_MacroAssembler _masm(&cbuf);
3326 guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
3327 __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
3328 Assembler::halfword, /*acquire*/ false, /*release*/ true,
3329 /*weak*/ false, noreg);
3330 %}
3331
3332 enc_class aarch64_enc_cmpxchgb(memory mem, iRegINoSp oldval, iRegINoSp newval) %{
3333 C2_MacroAssembler _masm(&cbuf);
3334 guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
3335 __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
3336 Assembler::byte, /*acquire*/ false, /*release*/ true,
3337 /*weak*/ false, noreg);
3338 %}
3339
3340
3341 // The only difference between aarch64_enc_cmpxchg and
3342 // aarch64_enc_cmpxchg_acq is that we use load-acquire in the
3343 // CompareAndSwap sequence to serve as a barrier on acquiring a
3344 // lock.
3345 enc_class aarch64_enc_cmpxchg_acq(memory mem, iRegLNoSp oldval, iRegLNoSp newval) %{
3346 C2_MacroAssembler _masm(&cbuf);
3347 guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
3348 __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
3349 Assembler::xword, /*acquire*/ true, /*release*/ true,
3350 /*weak*/ false, noreg);
3351 %}
3352
3353 enc_class aarch64_enc_cmpxchgw_acq(memory mem, iRegINoSp oldval, iRegINoSp newval) %{
3354 C2_MacroAssembler _masm(&cbuf);
3355 guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
3356 __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
3357 Assembler::word, /*acquire*/ true, /*release*/ true,
3358 /*weak*/ false, noreg);
3359 %}
3360
3361 enc_class aarch64_enc_cmpxchgs_acq(memory mem, iRegINoSp oldval, iRegINoSp newval) %{
3362 C2_MacroAssembler _masm(&cbuf);
3363 guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
3364 __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
3365 Assembler::halfword, /*acquire*/ true, /*release*/ true,
3366 /*weak*/ false, noreg);
3367 %}
3368
3369 enc_class aarch64_enc_cmpxchgb_acq(memory mem, iRegINoSp oldval, iRegINoSp newval) %{
3370 C2_MacroAssembler _masm(&cbuf);
3371 guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
3372 __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
3373 Assembler::byte, /*acquire*/ true, /*release*/ true,
3374 /*weak*/ false, noreg);
3375 %}
3376
3377 // auxiliary used for CompareAndSwapX to set result register
3378 enc_class aarch64_enc_cset_eq(iRegINoSp res) %{
3379 C2_MacroAssembler _masm(&cbuf);
3380 Register res_reg = as_Register($res$$reg);
3381 __ cset(res_reg, Assembler::EQ);
3382 %}
3383
3384 // prefetch encodings
3385
3386 enc_class aarch64_enc_prefetchw(memory mem) %{
3387 C2_MacroAssembler _masm(&cbuf);
3388 Register base = as_Register($mem$$base);
3389 int index = $mem$$index;
3390 int scale = $mem$$scale;
3391 int disp = $mem$$disp;
3392 if (index == -1) {
3393 __ prfm(Address(base, disp), PSTL1KEEP);
3394 } else {
3395 Register index_reg = as_Register(index);
3396 if (disp == 0) {
3397 __ prfm(Address(base, index_reg, Address::lsl(scale)), PSTL1KEEP);
3398 } else {
3399 __ lea(rscratch1, Address(base, disp));
3400 __ prfm(Address(rscratch1, index_reg, Address::lsl(scale)), PSTL1KEEP);
3401 }
3402 }
3403 %}
3404
3405 /// mov envcodings
3406
3407 enc_class aarch64_enc_movw_imm(iRegI dst, immI src) %{
3408 C2_MacroAssembler _masm(&cbuf);
3409 uint32_t con = (uint32_t)$src$$constant;
3410 Register dst_reg = as_Register($dst$$reg);
3411 if (con == 0) {
3412 __ movw(dst_reg, zr);
3413 } else {
3414 __ movw(dst_reg, con);
3415 }
3416 %}
3417
3418 enc_class aarch64_enc_mov_imm(iRegL dst, immL src) %{
3419 C2_MacroAssembler _masm(&cbuf);
3420 Register dst_reg = as_Register($dst$$reg);
3421 uint64_t con = (uint64_t)$src$$constant;
3422 if (con == 0) {
3423 __ mov(dst_reg, zr);
3424 } else {
3425 __ mov(dst_reg, con);
3426 }
3427 %}
3428
3429 enc_class aarch64_enc_mov_p(iRegP dst, immP src) %{
3430 C2_MacroAssembler _masm(&cbuf);
3431 Register dst_reg = as_Register($dst$$reg);
3432 address con = (address)$src$$constant;
3433 if (con == NULL || con == (address)1) {
3434 ShouldNotReachHere();
3435 } else {
3436 relocInfo::relocType rtype = $src->constant_reloc();
3437 if (rtype == relocInfo::oop_type) {
3438 __ movoop(dst_reg, (jobject)con, /*immediate*/true);
3439 } else if (rtype == relocInfo::metadata_type) {
3440 __ mov_metadata(dst_reg, (Metadata*)con);
3441 } else {
3442 assert(rtype == relocInfo::none, "unexpected reloc type");
3443 if (! __ is_valid_AArch64_address(con) ||
3444 con < (address)(uintptr_t)os::vm_page_size()) {
3445 __ mov(dst_reg, con);
3446 } else {
3447 uint64_t offset;
3448 __ adrp(dst_reg, con, offset);
3449 __ add(dst_reg, dst_reg, offset);
3450 }
3451 }
3452 }
3453 %}
3454
3455 enc_class aarch64_enc_mov_p0(iRegP dst, immP0 src) %{
3456 C2_MacroAssembler _masm(&cbuf);
3457 Register dst_reg = as_Register($dst$$reg);
3458 __ mov(dst_reg, zr);
3459 %}
3460
3461 enc_class aarch64_enc_mov_p1(iRegP dst, immP_1 src) %{
3462 C2_MacroAssembler _masm(&cbuf);
3463 Register dst_reg = as_Register($dst$$reg);
3464 __ mov(dst_reg, (uint64_t)1);
3465 %}
3466
3467 enc_class aarch64_enc_mov_byte_map_base(iRegP dst, immByteMapBase src) %{
3468 C2_MacroAssembler _masm(&cbuf);
3469 __ load_byte_map_base($dst$$Register);
3470 %}
3471
3472 enc_class aarch64_enc_mov_n(iRegN dst, immN src) %{
3473 C2_MacroAssembler _masm(&cbuf);
3474 Register dst_reg = as_Register($dst$$reg);
3475 address con = (address)$src$$constant;
3476 if (con == NULL) {
3477 ShouldNotReachHere();
3478 } else {
3479 relocInfo::relocType rtype = $src->constant_reloc();
3480 assert(rtype == relocInfo::oop_type, "unexpected reloc type");
3481 __ set_narrow_oop(dst_reg, (jobject)con);
3482 }
3483 %}
3484
3485 enc_class aarch64_enc_mov_n0(iRegN dst, immN0 src) %{
3486 C2_MacroAssembler _masm(&cbuf);
3487 Register dst_reg = as_Register($dst$$reg);
3488 __ mov(dst_reg, zr);
3489 %}
3490
3491 enc_class aarch64_enc_mov_nk(iRegN dst, immNKlass src) %{
3492 C2_MacroAssembler _masm(&cbuf);
3493 Register dst_reg = as_Register($dst$$reg);
3494 address con = (address)$src$$constant;
3495 if (con == NULL) {
3496 ShouldNotReachHere();
3497 } else {
3498 relocInfo::relocType rtype = $src->constant_reloc();
3499 assert(rtype == relocInfo::metadata_type, "unexpected reloc type");
3500 __ set_narrow_klass(dst_reg, (Klass *)con);
3501 }
3502 %}
3503
3504 // arithmetic encodings
3505
3506 enc_class aarch64_enc_addsubw_imm(iRegI dst, iRegI src1, immIAddSub src2) %{
3507 C2_MacroAssembler _masm(&cbuf);
3508 Register dst_reg = as_Register($dst$$reg);
3509 Register src_reg = as_Register($src1$$reg);
3510 int32_t con = (int32_t)$src2$$constant;
3511 // add has primary == 0, subtract has primary == 1
3512 if ($primary) { con = -con; }
3513 if (con < 0) {
3514 __ subw(dst_reg, src_reg, -con);
3515 } else {
3516 __ addw(dst_reg, src_reg, con);
3517 }
3518 %}
3519
3520 enc_class aarch64_enc_addsub_imm(iRegL dst, iRegL src1, immLAddSub src2) %{
3521 C2_MacroAssembler _masm(&cbuf);
3522 Register dst_reg = as_Register($dst$$reg);
3523 Register src_reg = as_Register($src1$$reg);
3524 int32_t con = (int32_t)$src2$$constant;
3525 // add has primary == 0, subtract has primary == 1
3526 if ($primary) { con = -con; }
3527 if (con < 0) {
3528 __ sub(dst_reg, src_reg, -con);
3529 } else {
3530 __ add(dst_reg, src_reg, con);
3531 }
3532 %}
3533
3534 enc_class aarch64_enc_divw(iRegI dst, iRegI src1, iRegI src2) %{
3535 C2_MacroAssembler _masm(&cbuf);
3536 Register dst_reg = as_Register($dst$$reg);
3537 Register src1_reg = as_Register($src1$$reg);
3538 Register src2_reg = as_Register($src2$$reg);
3539 __ corrected_idivl(dst_reg, src1_reg, src2_reg, false, rscratch1);
3540 %}
3541
3542 enc_class aarch64_enc_div(iRegI dst, iRegI src1, iRegI src2) %{
3543 C2_MacroAssembler _masm(&cbuf);
3544 Register dst_reg = as_Register($dst$$reg);
3545 Register src1_reg = as_Register($src1$$reg);
3546 Register src2_reg = as_Register($src2$$reg);
3547 __ corrected_idivq(dst_reg, src1_reg, src2_reg, false, rscratch1);
3548 %}
3549
3550 enc_class aarch64_enc_modw(iRegI dst, iRegI src1, iRegI src2) %{
3551 C2_MacroAssembler _masm(&cbuf);
3552 Register dst_reg = as_Register($dst$$reg);
3553 Register src1_reg = as_Register($src1$$reg);
3554 Register src2_reg = as_Register($src2$$reg);
3555 __ corrected_idivl(dst_reg, src1_reg, src2_reg, true, rscratch1);
3556 %}
3557
3558 enc_class aarch64_enc_mod(iRegI dst, iRegI src1, iRegI src2) %{
3559 C2_MacroAssembler _masm(&cbuf);
3560 Register dst_reg = as_Register($dst$$reg);
3561 Register src1_reg = as_Register($src1$$reg);
3562 Register src2_reg = as_Register($src2$$reg);
3563 __ corrected_idivq(dst_reg, src1_reg, src2_reg, true, rscratch1);
3564 %}
3565
3566 // compare instruction encodings
3567
3568 enc_class aarch64_enc_cmpw(iRegI src1, iRegI src2) %{
3569 C2_MacroAssembler _masm(&cbuf);
3570 Register reg1 = as_Register($src1$$reg);
3571 Register reg2 = as_Register($src2$$reg);
3572 __ cmpw(reg1, reg2);
3573 %}
3574
3575 enc_class aarch64_enc_cmpw_imm_addsub(iRegI src1, immIAddSub src2) %{
3576 C2_MacroAssembler _masm(&cbuf);
3577 Register reg = as_Register($src1$$reg);
3578 int32_t val = $src2$$constant;
3579 if (val >= 0) {
3580 __ subsw(zr, reg, val);
3581 } else {
3582 __ addsw(zr, reg, -val);
3583 }
3584 %}
3585
3586 enc_class aarch64_enc_cmpw_imm(iRegI src1, immI src2) %{
3587 C2_MacroAssembler _masm(&cbuf);
3588 Register reg1 = as_Register($src1$$reg);
3589 uint32_t val = (uint32_t)$src2$$constant;
3590 __ movw(rscratch1, val);
3591 __ cmpw(reg1, rscratch1);
3592 %}
3593
3594 enc_class aarch64_enc_cmp(iRegL src1, iRegL src2) %{
3595 C2_MacroAssembler _masm(&cbuf);
3596 Register reg1 = as_Register($src1$$reg);
3597 Register reg2 = as_Register($src2$$reg);
3598 __ cmp(reg1, reg2);
3599 %}
3600
3601 enc_class aarch64_enc_cmp_imm_addsub(iRegL src1, immL12 src2) %{
3602 C2_MacroAssembler _masm(&cbuf);
3603 Register reg = as_Register($src1$$reg);
3604 int64_t val = $src2$$constant;
3605 if (val >= 0) {
3606 __ subs(zr, reg, val);
3607 } else if (val != -val) {
3608 __ adds(zr, reg, -val);
3609 } else {
3610 // aargh, Long.MIN_VALUE is a special case
3611 __ orr(rscratch1, zr, (uint64_t)val);
3612 __ subs(zr, reg, rscratch1);
3613 }
3614 %}
3615
3616 enc_class aarch64_enc_cmp_imm(iRegL src1, immL src2) %{
3617 C2_MacroAssembler _masm(&cbuf);
3618 Register reg1 = as_Register($src1$$reg);
3619 uint64_t val = (uint64_t)$src2$$constant;
3620 __ mov(rscratch1, val);
3621 __ cmp(reg1, rscratch1);
3622 %}
3623
3624 enc_class aarch64_enc_cmpp(iRegP src1, iRegP src2) %{
3625 C2_MacroAssembler _masm(&cbuf);
3626 Register reg1 = as_Register($src1$$reg);
3627 Register reg2 = as_Register($src2$$reg);
3628 __ cmp(reg1, reg2);
3629 %}
3630
3631 enc_class aarch64_enc_cmpn(iRegN src1, iRegN src2) %{
3632 C2_MacroAssembler _masm(&cbuf);
3633 Register reg1 = as_Register($src1$$reg);
3634 Register reg2 = as_Register($src2$$reg);
3635 __ cmpw(reg1, reg2);
3636 %}
3637
3638 enc_class aarch64_enc_testp(iRegP src) %{
3639 C2_MacroAssembler _masm(&cbuf);
3640 Register reg = as_Register($src$$reg);
3641 __ cmp(reg, zr);
3642 %}
3643
3644 enc_class aarch64_enc_testn(iRegN src) %{
3645 C2_MacroAssembler _masm(&cbuf);
3646 Register reg = as_Register($src$$reg);
3647 __ cmpw(reg, zr);
3648 %}
3649
3650 enc_class aarch64_enc_b(label lbl) %{
3651 C2_MacroAssembler _masm(&cbuf);
3652 Label *L = $lbl$$label;
3653 __ b(*L);
3654 %}
3655
3656 enc_class aarch64_enc_br_con(cmpOp cmp, label lbl) %{
3657 C2_MacroAssembler _masm(&cbuf);
3658 Label *L = $lbl$$label;
3659 __ br ((Assembler::Condition)$cmp$$cmpcode, *L);
3660 %}
3661
3662 enc_class aarch64_enc_br_conU(cmpOpU cmp, label lbl) %{
3663 C2_MacroAssembler _masm(&cbuf);
3664 Label *L = $lbl$$label;
3665 __ br ((Assembler::Condition)$cmp$$cmpcode, *L);
3666 %}
3667
3668 enc_class aarch64_enc_partial_subtype_check(iRegP sub, iRegP super, iRegP temp, iRegP result)
3669 %{
3670 Register sub_reg = as_Register($sub$$reg);
3671 Register super_reg = as_Register($super$$reg);
3672 Register temp_reg = as_Register($temp$$reg);
3673 Register result_reg = as_Register($result$$reg);
3674
3675 Label miss;
3676 C2_MacroAssembler _masm(&cbuf);
3677 __ check_klass_subtype_slow_path(sub_reg, super_reg, temp_reg, result_reg,
3678 NULL, &miss,
3679 /*set_cond_codes:*/ true);
3680 if ($primary) {
3681 __ mov(result_reg, zr);
3682 }
3683 __ bind(miss);
3684 %}
3685
3686 enc_class aarch64_enc_java_static_call(method meth) %{
3687 C2_MacroAssembler _masm(&cbuf);
3688
3689 address addr = (address)$meth$$method;
3690 address call;
3691 if (!_method) {
3692 // A call to a runtime wrapper, e.g. new, new_typeArray_Java, uncommon_trap.
3693 call = __ trampoline_call(Address(addr, relocInfo::runtime_call_type), &cbuf);
3694 if (call == NULL) {
3695 ciEnv::current()->record_failure("CodeCache is full");
3696 return;
3697 }
3698 } else {
3699 int method_index = resolved_method_index(cbuf);
3700 RelocationHolder rspec = _optimized_virtual ? opt_virtual_call_Relocation::spec(method_index)
3701 : static_call_Relocation::spec(method_index);
3702 call = __ trampoline_call(Address(addr, rspec), &cbuf);
3703 if (call == NULL) {
3704 ciEnv::current()->record_failure("CodeCache is full");
3705 return;
3706 }
3707 // Emit stub for static call
3708 address stub = CompiledStaticCall::emit_to_interp_stub(cbuf);
3709 if (stub == NULL) {
3710 ciEnv::current()->record_failure("CodeCache is full");
3711 return;
3712 }
3713 }
3714
3715 // Only non uncommon_trap calls need to reinitialize ptrue.
3716 if (Compile::current()->max_vector_size() >= 16 && uncommon_trap_request() == 0) {
3717 __ reinitialize_ptrue();
3718 }
3719 %}
3720
3721 enc_class aarch64_enc_java_dynamic_call(method meth) %{
3722 C2_MacroAssembler _masm(&cbuf);
3723 int method_index = resolved_method_index(cbuf);
3724 address call = __ ic_call((address)$meth$$method, method_index);
3725 if (call == NULL) {
3726 ciEnv::current()->record_failure("CodeCache is full");
3727 return;
3728 } else if (Compile::current()->max_vector_size() >= 16) {
3729 __ reinitialize_ptrue();
3730 }
3731 %}
3732
3733 enc_class aarch64_enc_call_epilog() %{
3734 C2_MacroAssembler _masm(&cbuf);
3735 if (VerifyStackAtCalls) {
3736 // Check that stack depth is unchanged: find majik cookie on stack
3737 __ call_Unimplemented();
3738 }
3739 %}
3740
3741 enc_class aarch64_enc_java_to_runtime(method meth) %{
3742 C2_MacroAssembler _masm(&cbuf);
3743
3744 // some calls to generated routines (arraycopy code) are scheduled
3745 // by C2 as runtime calls. if so we can call them using a br (they
3746 // will be in a reachable segment) otherwise we have to use a blr
3747 // which loads the absolute address into a register.
3748 address entry = (address)$meth$$method;
3749 CodeBlob *cb = CodeCache::find_blob(entry);
3750 if (cb) {
3751 address call = __ trampoline_call(Address(entry, relocInfo::runtime_call_type));
3752 if (call == NULL) {
3753 ciEnv::current()->record_failure("CodeCache is full");
3754 return;
3755 }
3756 } else {
3757 Label retaddr;
3758 __ adr(rscratch2, retaddr);
3759 __ lea(rscratch1, RuntimeAddress(entry));
3760 // Leave a breadcrumb for JavaFrameAnchor::capture_last_Java_pc()
3761 __ stp(zr, rscratch2, Address(__ pre(sp, -2 * wordSize)));
3762 __ blr(rscratch1);
3763 __ bind(retaddr);
3764 __ add(sp, sp, 2 * wordSize);
3765 }
3766 if (Compile::current()->max_vector_size() >= 16) {
3767 __ reinitialize_ptrue();
3768 }
3769 %}
3770
3771 enc_class aarch64_enc_rethrow() %{
3772 C2_MacroAssembler _masm(&cbuf);
3773 __ far_jump(RuntimeAddress(OptoRuntime::rethrow_stub()));
3774 %}
3775
3776 enc_class aarch64_enc_ret() %{
3777 C2_MacroAssembler _masm(&cbuf);
3778 #ifdef ASSERT
3779 if (Compile::current()->max_vector_size() >= 16) {
3780 __ verify_ptrue();
3781 }
3782 #endif
3783 __ ret(lr);
3784 %}
3785
3786 enc_class aarch64_enc_tail_call(iRegP jump_target) %{
3787 C2_MacroAssembler _masm(&cbuf);
3788 Register target_reg = as_Register($jump_target$$reg);
3789 __ br(target_reg);
3790 %}
3791
3792 enc_class aarch64_enc_tail_jmp(iRegP jump_target) %{
3793 C2_MacroAssembler _masm(&cbuf);
3794 Register target_reg = as_Register($jump_target$$reg);
3795 // exception oop should be in r0
3796 // ret addr has been popped into lr
3797 // callee expects it in r3
3798 __ mov(r3, lr);
3799 __ br(target_reg);
3800 %}
3801
3802 enc_class aarch64_enc_fast_lock(iRegP object, iRegP box, iRegP tmp, iRegP tmp2) %{
3803 C2_MacroAssembler _masm(&cbuf);
3804 Register oop = as_Register($object$$reg);
3805 Register box = as_Register($box$$reg);
3806 Register disp_hdr = as_Register($tmp$$reg);
3807 Register tmp = as_Register($tmp2$$reg);
3808 Label cont;
3809 Label object_has_monitor;
3810 Label cas_failed;
3811
3812 assert_different_registers(oop, box, tmp, disp_hdr);
3813
3814 // Load markWord from object into displaced_header.
3815 __ ldr(disp_hdr, Address(oop, oopDesc::mark_offset_in_bytes()));
3816
3817 if (DiagnoseSyncOnValueBasedClasses != 0) {
3818 __ load_klass(tmp, oop);
3819 __ ldrw(tmp, Address(tmp, Klass::access_flags_offset()));
3820 __ tstw(tmp, JVM_ACC_IS_VALUE_BASED_CLASS);
3821 __ br(Assembler::NE, cont);
3822 }
3823
3824 if (UseBiasedLocking && !UseOptoBiasInlining) {
3825 __ biased_locking_enter(box, oop, disp_hdr, tmp, true, cont);
3826 }
3827
3828 // Check for existing monitor
3829 __ tbnz(disp_hdr, exact_log2(markWord::monitor_value), object_has_monitor);
3830
3831 // Set tmp to be (markWord of object | UNLOCK_VALUE).
3832 __ orr(tmp, disp_hdr, markWord::unlocked_value);
3833
3834 // Initialize the box. (Must happen before we update the object mark!)
3835 __ str(tmp, Address(box, BasicLock::displaced_header_offset_in_bytes()));
3836
3837 // Compare object markWord with an unlocked value (tmp) and if
3838 // equal exchange the stack address of our box with object markWord.
3839 // On failure disp_hdr contains the possibly locked markWord.
3840 __ cmpxchg(oop, tmp, box, Assembler::xword, /*acquire*/ true,
3841 /*release*/ true, /*weak*/ false, disp_hdr);
3842 __ br(Assembler::EQ, cont);
3843
3844 assert(oopDesc::mark_offset_in_bytes() == 0, "offset of _mark is not 0");
3845
3846 // If the compare-and-exchange succeeded, then we found an unlocked
3847 // object, will have now locked it will continue at label cont
3848
3849 __ bind(cas_failed);
3850 // We did not see an unlocked object so try the fast recursive case.
3851
3852 // Check if the owner is self by comparing the value in the
3853 // markWord of object (disp_hdr) with the stack pointer.
3854 __ mov(rscratch1, sp);
3855 __ sub(disp_hdr, disp_hdr, rscratch1);
3856 __ mov(tmp, (address) (~(os::vm_page_size()-1) | markWord::lock_mask_in_place));
3857 // If condition is true we are cont and hence we can store 0 as the
3858 // displaced header in the box, which indicates that it is a recursive lock.
3859 __ ands(tmp/*==0?*/, disp_hdr, tmp); // Sets flags for result
3860 __ str(tmp/*==0, perhaps*/, Address(box, BasicLock::displaced_header_offset_in_bytes()));
3861
3862 __ b(cont);
3863
3864 // Handle existing monitor.
3865 __ bind(object_has_monitor);
3866
3867 // The object's monitor m is unlocked iff m->owner == NULL,
3868 // otherwise m->owner may contain a thread or a stack address.
3869 //
3870 // Try to CAS m->owner from NULL to current thread.
3871 __ add(tmp, disp_hdr, (ObjectMonitor::owner_offset_in_bytes()-markWord::monitor_value));
3872 __ cmpxchg(tmp, zr, rthread, Assembler::xword, /*acquire*/ true,
3873 /*release*/ true, /*weak*/ false, rscratch1); // Sets flags for result
3874
3875 // Store a non-null value into the box to avoid looking like a re-entrant
3876 // lock. The fast-path monitor unlock code checks for
3877 // markWord::monitor_value so use markWord::unused_mark which has the
3878 // relevant bit set, and also matches ObjectSynchronizer::enter.
3879 __ mov(tmp, (address)markWord::unused_mark().value());
3880 __ str(tmp, Address(box, BasicLock::displaced_header_offset_in_bytes()));
3881
3882 __ br(Assembler::EQ, cont); // CAS success means locking succeeded
3883
3884 __ cmp(rscratch1, rthread);
3885 __ br(Assembler::NE, cont); // Check for recursive locking
3886
3887 // Recursive lock case
3888 __ increment(Address(disp_hdr, ObjectMonitor::recursions_offset_in_bytes() - markWord::monitor_value), 1);
3889 // flag == EQ still from the cmp above, checking if this is a reentrant lock
3890
3891 __ bind(cont);
3892 // flag == EQ indicates success
3893 // flag == NE indicates failure
3894 %}
3895
3896 enc_class aarch64_enc_fast_unlock(iRegP object, iRegP box, iRegP tmp, iRegP tmp2) %{
3897 C2_MacroAssembler _masm(&cbuf);
3898 Register oop = as_Register($object$$reg);
3899 Register box = as_Register($box$$reg);
3900 Register disp_hdr = as_Register($tmp$$reg);
3901 Register tmp = as_Register($tmp2$$reg);
3902 Label cont;
3903 Label object_has_monitor;
3904
3905 assert_different_registers(oop, box, tmp, disp_hdr);
3906
3907 if (UseBiasedLocking && !UseOptoBiasInlining) {
3908 __ biased_locking_exit(oop, tmp, cont);
3909 }
3910
3911 // Find the lock address and load the displaced header from the stack.
3912 __ ldr(disp_hdr, Address(box, BasicLock::displaced_header_offset_in_bytes()));
3913
3914 // If the displaced header is 0, we have a recursive unlock.
3915 __ cmp(disp_hdr, zr);
3916 __ br(Assembler::EQ, cont);
3917
3918 // Handle existing monitor.
3919 __ ldr(tmp, Address(oop, oopDesc::mark_offset_in_bytes()));
3920 __ tbnz(tmp, exact_log2(markWord::monitor_value), object_has_monitor);
3921
3922 // Check if it is still a light weight lock, this is is true if we
3923 // see the stack address of the basicLock in the markWord of the
3924 // object.
3925
3926 __ cmpxchg(oop, box, disp_hdr, Assembler::xword, /*acquire*/ false,
3927 /*release*/ true, /*weak*/ false, tmp);
3928 __ b(cont);
3929
3930 assert(oopDesc::mark_offset_in_bytes() == 0, "offset of _mark is not 0");
3931
3932 // Handle existing monitor.
3933 __ bind(object_has_monitor);
3934 STATIC_ASSERT(markWord::monitor_value <= INT_MAX);
3935 __ add(tmp, tmp, -(int)markWord::monitor_value); // monitor
3936 __ ldr(disp_hdr, Address(tmp, ObjectMonitor::recursions_offset_in_bytes()));
3937
3938 Label notRecursive;
3939 __ cbz(disp_hdr, notRecursive);
3940
3941 // Recursive lock
3942 __ sub(disp_hdr, disp_hdr, 1u);
3943 __ str(disp_hdr, Address(tmp, ObjectMonitor::recursions_offset_in_bytes()));
3944 __ cmp(disp_hdr, disp_hdr); // Sets flags for result
3945 __ b(cont);
3946
3947 __ bind(notRecursive);
3948 __ ldr(rscratch1, Address(tmp, ObjectMonitor::EntryList_offset_in_bytes()));
3949 __ ldr(disp_hdr, Address(tmp, ObjectMonitor::cxq_offset_in_bytes()));
3950 __ orr(rscratch1, rscratch1, disp_hdr); // Will be 0 if both are 0.
3951 __ cmp(rscratch1, zr); // Sets flags for result
3952 __ cbnz(rscratch1, cont);
3953 // need a release store here
3954 __ lea(tmp, Address(tmp, ObjectMonitor::owner_offset_in_bytes()));
3955 __ stlr(zr, tmp); // set unowned
3956
3957 __ bind(cont);
3958 // flag == EQ indicates success
3959 // flag == NE indicates failure
3960 %}
3961
3962 %}
3963
3964 //----------FRAME--------------------------------------------------------------
3965 // Definition of frame structure and management information.
3966 //
3967 // S T A C K L A Y O U T Allocators stack-slot number
3968 // | (to get allocators register number
3969 // G Owned by | | v add OptoReg::stack0())
3970 // r CALLER | |
3971 // o | +--------+ pad to even-align allocators stack-slot
3972 // w V | pad0 | numbers; owned by CALLER
3973 // t -----------+--------+----> Matcher::_in_arg_limit, unaligned
3974 // h ^ | in | 5
3975 // | | args | 4 Holes in incoming args owned by SELF
3976 // | | | | 3
3977 // | | +--------+
3978 // V | | old out| Empty on Intel, window on Sparc
3979 // | old |preserve| Must be even aligned.
3980 // | SP-+--------+----> Matcher::_old_SP, even aligned
3981 // | | in | 3 area for Intel ret address
3982 // Owned by |preserve| Empty on Sparc.
3983 // SELF +--------+
3984 // | | pad2 | 2 pad to align old SP
3985 // | +--------+ 1
3986 // | | locks | 0
3987 // | +--------+----> OptoReg::stack0(), even aligned
3988 // | | pad1 | 11 pad to align new SP
3989 // | +--------+
3990 // | | | 10
3991 // | | spills | 9 spills
3992 // V | | 8 (pad0 slot for callee)
3993 // -----------+--------+----> Matcher::_out_arg_limit, unaligned
3994 // ^ | out | 7
3995 // | | args | 6 Holes in outgoing args owned by CALLEE
3996 // Owned by +--------+
3997 // CALLEE | new out| 6 Empty on Intel, window on Sparc
3998 // | new |preserve| Must be even-aligned.
3999 // | SP-+--------+----> Matcher::_new_SP, even aligned
4000 // | | |
4001 //
4002 // Note 1: Only region 8-11 is determined by the allocator. Region 0-5 is
4003 // known from SELF's arguments and the Java calling convention.
4004 // Region 6-7 is determined per call site.
4005 // Note 2: If the calling convention leaves holes in the incoming argument
4006 // area, those holes are owned by SELF. Holes in the outgoing area
4007 // are owned by the CALLEE. Holes should not be nessecary in the
4008 // incoming area, as the Java calling convention is completely under
4009 // the control of the AD file. Doubles can be sorted and packed to
4010 // avoid holes. Holes in the outgoing arguments may be nessecary for
4011 // varargs C calling conventions.
4012 // Note 3: Region 0-3 is even aligned, with pad2 as needed. Region 3-5 is
4013 // even aligned with pad0 as needed.
4014 // Region 6 is even aligned. Region 6-7 is NOT even aligned;
4015 // (the latter is true on Intel but is it false on AArch64?)
4016 // region 6-11 is even aligned; it may be padded out more so that
4017 // the region from SP to FP meets the minimum stack alignment.
4018 // Note 4: For I2C adapters, the incoming FP may not meet the minimum stack
4019 // alignment. Region 11, pad1, may be dynamically extended so that
4020 // SP meets the minimum alignment.
4021
4022 frame %{
4023 // These three registers define part of the calling convention
4024 // between compiled code and the interpreter.
4025
4026 // Inline Cache Register or Method for I2C.
4027 inline_cache_reg(R12);
4028
4029 // Number of stack slots consumed by locking an object
4030 sync_stack_slots(2);
4031
4032 // Compiled code's Frame Pointer
4033 frame_pointer(R31);
4034
4035 // Interpreter stores its frame pointer in a register which is
4036 // stored to the stack by I2CAdaptors.
4037 // I2CAdaptors convert from interpreted java to compiled java.
4038 interpreter_frame_pointer(R29);
4039
4040 // Stack alignment requirement
4041 stack_alignment(StackAlignmentInBytes); // Alignment size in bytes (128-bit -> 16 bytes)
4042
4043 // Number of outgoing stack slots killed above the out_preserve_stack_slots
4044 // for calls to C. Supports the var-args backing area for register parms.
4045 varargs_C_out_slots_killed(frame::arg_reg_save_area_bytes/BytesPerInt);
4046
4047 // The after-PROLOG location of the return address. Location of
4048 // return address specifies a type (REG or STACK) and a number
4049 // representing the register number (i.e. - use a register name) or
4050 // stack slot.
4051 // Ret Addr is on stack in slot 0 if no locks or verification or alignment.
4052 // Otherwise, it is above the locks and verification slot and alignment word
4053 // TODO this may well be correct but need to check why that - 2 is there
4054 // ppc port uses 0 but we definitely need to allow for fixed_slots
4055 // which folds in the space used for monitors
4056 return_addr(STACK - 2 +
4057 align_up((Compile::current()->in_preserve_stack_slots() +
4058 Compile::current()->fixed_slots()),
4059 stack_alignment_in_slots()));
4060
4061 // Location of compiled Java return values. Same as C for now.
4062 return_value
4063 %{
4064 // TODO do we allow ideal_reg == Op_RegN???
4065 assert(ideal_reg >= Op_RegI && ideal_reg <= Op_RegL,
4066 "only return normal values");
4067
4068 static const int lo[Op_RegL + 1] = { // enum name
4069 0, // Op_Node
4070 0, // Op_Set
4071 R0_num, // Op_RegN
4072 R0_num, // Op_RegI
4073 R0_num, // Op_RegP
4074 V0_num, // Op_RegF
4075 V0_num, // Op_RegD
4076 R0_num // Op_RegL
4077 };
4078
4079 static const int hi[Op_RegL + 1] = { // enum name
4080 0, // Op_Node
4081 0, // Op_Set
4082 OptoReg::Bad, // Op_RegN
4083 OptoReg::Bad, // Op_RegI
4084 R0_H_num, // Op_RegP
4085 OptoReg::Bad, // Op_RegF
4086 V0_H_num, // Op_RegD
4087 R0_H_num // Op_RegL
4088 };
4089
4090 return OptoRegPair(hi[ideal_reg], lo[ideal_reg]);
4091 %}
4092 %}
4093
4094 //----------ATTRIBUTES---------------------------------------------------------
4095 //----------Operand Attributes-------------------------------------------------
4096 op_attrib op_cost(1); // Required cost attribute
4097
4098 //----------Instruction Attributes---------------------------------------------
4099 ins_attrib ins_cost(INSN_COST); // Required cost attribute
4100 ins_attrib ins_size(32); // Required size attribute (in bits)
4101 ins_attrib ins_short_branch(0); // Required flag: is this instruction
4102 // a non-matching short branch variant
4103 // of some long branch?
4104 ins_attrib ins_alignment(4); // Required alignment attribute (must
4105 // be a power of 2) specifies the
4106 // alignment that some part of the
4107 // instruction (not necessarily the
4108 // start) requires. If > 1, a
4109 // compute_padding() function must be
4110 // provided for the instruction
4111
4112 //----------OPERANDS-----------------------------------------------------------
4113 // Operand definitions must precede instruction definitions for correct parsing
4114 // in the ADLC because operands constitute user defined types which are used in
4115 // instruction definitions.
4116
4117 //----------Simple Operands----------------------------------------------------
4118
4119 // Integer operands 32 bit
4120 // 32 bit immediate
4121 operand immI()
4122 %{
4123 match(ConI);
4124
4125 op_cost(0);
4126 format %{ %}
4127 interface(CONST_INTER);
4128 %}
4129
4130 // 32 bit zero
4131 operand immI0()
4132 %{
4133 predicate(n->get_int() == 0);
4134 match(ConI);
4135
4136 op_cost(0);
4137 format %{ %}
4138 interface(CONST_INTER);
4139 %}
4140
4141 // 32 bit unit increment
4142 operand immI_1()
4143 %{
4144 predicate(n->get_int() == 1);
4145 match(ConI);
4146
4147 op_cost(0);
4148 format %{ %}
4149 interface(CONST_INTER);
4150 %}
4151
4152 // 32 bit unit decrement
4153 operand immI_M1()
4154 %{
4155 predicate(n->get_int() == -1);
4156 match(ConI);
4157
4158 op_cost(0);
4159 format %{ %}
4160 interface(CONST_INTER);
4161 %}
4162
4163 // Shift values for add/sub extension shift
4164 operand immIExt()
4165 %{
4166 predicate(0 <= n->get_int() && (n->get_int() <= 4));
4167 match(ConI);
4168
4169 op_cost(0);
4170 format %{ %}
4171 interface(CONST_INTER);
4172 %}
4173
4174 operand immI_le_4()
4175 %{
4176 predicate(n->get_int() <= 4);
4177 match(ConI);
4178
4179 op_cost(0);
4180 format %{ %}
4181 interface(CONST_INTER);
4182 %}
4183
4184 operand immI_31()
4185 %{
4186 predicate(n->get_int() == 31);
4187 match(ConI);
4188
4189 op_cost(0);
4190 format %{ %}
4191 interface(CONST_INTER);
4192 %}
4193
4194 operand immI_2()
4195 %{
4196 predicate(n->get_int() == 2);
4197 match(ConI);
4198
4199 op_cost(0);
4200 format %{ %}
4201 interface(CONST_INTER);
4202 %}
4203
4204 operand immI_4()
4205 %{
4206 predicate(n->get_int() == 4);
4207 match(ConI);
4208
4209 op_cost(0);
4210 format %{ %}
4211 interface(CONST_INTER);
4212 %}
4213
4214 operand immI_8()
4215 %{
4216 predicate(n->get_int() == 8);
4217 match(ConI);
4218
4219 op_cost(0);
4220 format %{ %}
4221 interface(CONST_INTER);
4222 %}
4223
4224 operand immI_16()
4225 %{
4226 predicate(n->get_int() == 16);
4227 match(ConI);
4228
4229 op_cost(0);
4230 format %{ %}
4231 interface(CONST_INTER);
4232 %}
4233
4234 operand immI_24()
4235 %{
4236 predicate(n->get_int() == 24);
4237 match(ConI);
4238
4239 op_cost(0);
4240 format %{ %}
4241 interface(CONST_INTER);
4242 %}
4243
4244 operand immI_32()
4245 %{
4246 predicate(n->get_int() == 32);
4247 match(ConI);
4248
4249 op_cost(0);
4250 format %{ %}
4251 interface(CONST_INTER);
4252 %}
4253
4254 operand immI_48()
4255 %{
4256 predicate(n->get_int() == 48);
4257 match(ConI);
4258
4259 op_cost(0);
4260 format %{ %}
4261 interface(CONST_INTER);
4262 %}
4263
4264 operand immI_56()
4265 %{
4266 predicate(n->get_int() == 56);
4267 match(ConI);
4268
4269 op_cost(0);
4270 format %{ %}
4271 interface(CONST_INTER);
4272 %}
4273
4274 operand immI_63()
4275 %{
4276 predicate(n->get_int() == 63);
4277 match(ConI);
4278
4279 op_cost(0);
4280 format %{ %}
4281 interface(CONST_INTER);
4282 %}
4283
4284 operand immI_64()
4285 %{
4286 predicate(n->get_int() == 64);
4287 match(ConI);
4288
4289 op_cost(0);
4290 format %{ %}
4291 interface(CONST_INTER);
4292 %}
4293
4294 operand immI_255()
4295 %{
4296 predicate(n->get_int() == 255);
4297 match(ConI);
4298
4299 op_cost(0);
4300 format %{ %}
4301 interface(CONST_INTER);
4302 %}
4303
4304 operand immI_65535()
4305 %{
4306 predicate(n->get_int() == 65535);
4307 match(ConI);
4308
4309 op_cost(0);
4310 format %{ %}
4311 interface(CONST_INTER);
4312 %}
4313
4314 operand immI_positive()
4315 %{
4316 predicate(n->get_int() > 0);
4317 match(ConI);
4318
4319 op_cost(0);
4320 format %{ %}
4321 interface(CONST_INTER);
4322 %}
4323
4324 operand immL_255()
4325 %{
4326 predicate(n->get_long() == 255L);
4327 match(ConL);
4328
4329 op_cost(0);
4330 format %{ %}
4331 interface(CONST_INTER);
4332 %}
4333
4334 operand immL_65535()
4335 %{
4336 predicate(n->get_long() == 65535L);
4337 match(ConL);
4338
4339 op_cost(0);
4340 format %{ %}
4341 interface(CONST_INTER);
4342 %}
4343
4344 operand immL_4294967295()
4345 %{
4346 predicate(n->get_long() == 4294967295L);
4347 match(ConL);
4348
4349 op_cost(0);
4350 format %{ %}
4351 interface(CONST_INTER);
4352 %}
4353
4354 operand immL_bitmask()
4355 %{
4356 predicate((n->get_long() != 0)
4357 && ((n->get_long() & 0xc000000000000000l) == 0)
4358 && is_power_of_2(n->get_long() + 1));
4359 match(ConL);
4360
4361 op_cost(0);
4362 format %{ %}
4363 interface(CONST_INTER);
4364 %}
4365
4366 operand immI_bitmask()
4367 %{
4368 predicate((n->get_int() != 0)
4369 && ((n->get_int() & 0xc0000000) == 0)
4370 && is_power_of_2(n->get_int() + 1));
4371 match(ConI);
4372
4373 op_cost(0);
4374 format %{ %}
4375 interface(CONST_INTER);
4376 %}
4377
4378 operand immL_positive_bitmaskI()
4379 %{
4380 predicate((n->get_long() != 0)
4381 && ((julong)n->get_long() < 0x80000000ULL)
4382 && is_power_of_2(n->get_long() + 1));
4383 match(ConL);
4384
4385 op_cost(0);
4386 format %{ %}
4387 interface(CONST_INTER);
4388 %}
4389
4390 // Scale values for scaled offset addressing modes (up to long but not quad)
4391 operand immIScale()
4392 %{
4393 predicate(0 <= n->get_int() && (n->get_int() <= 3));
4394 match(ConI);
4395
4396 op_cost(0);
4397 format %{ %}
4398 interface(CONST_INTER);
4399 %}
4400
4401 // 26 bit signed offset -- for pc-relative branches
4402 operand immI26()
4403 %{
4404 predicate(((-(1 << 25)) <= n->get_int()) && (n->get_int() < (1 << 25)));
4405 match(ConI);
4406
4407 op_cost(0);
4408 format %{ %}
4409 interface(CONST_INTER);
4410 %}
4411
4412 // 19 bit signed offset -- for pc-relative loads
4413 operand immI19()
4414 %{
4415 predicate(((-(1 << 18)) <= n->get_int()) && (n->get_int() < (1 << 18)));
4416 match(ConI);
4417
4418 op_cost(0);
4419 format %{ %}
4420 interface(CONST_INTER);
4421 %}
4422
4423 // 12 bit unsigned offset -- for base plus immediate loads
4424 operand immIU12()
4425 %{
4426 predicate((0 <= n->get_int()) && (n->get_int() < (1 << 12)));
4427 match(ConI);
4428
4429 op_cost(0);
4430 format %{ %}
4431 interface(CONST_INTER);
4432 %}
4433
4434 operand immLU12()
4435 %{
4436 predicate((0 <= n->get_long()) && (n->get_long() < (1 << 12)));
4437 match(ConL);
4438
4439 op_cost(0);
4440 format %{ %}
4441 interface(CONST_INTER);
4442 %}
4443
4444 // Offset for scaled or unscaled immediate loads and stores
4445 operand immIOffset()
4446 %{
4447 predicate(Address::offset_ok_for_immed(n->get_int(), 0));
4448 match(ConI);
4449
4450 op_cost(0);
4451 format %{ %}
4452 interface(CONST_INTER);
4453 %}
4454
4455 operand immIOffset1()
4456 %{
4457 predicate(Address::offset_ok_for_immed(n->get_int(), 0));
4458 match(ConI);
4459
4460 op_cost(0);
4461 format %{ %}
4462 interface(CONST_INTER);
4463 %}
4464
4465 operand immIOffset2()
4466 %{
4467 predicate(Address::offset_ok_for_immed(n->get_int(), 1));
4468 match(ConI);
4469
4470 op_cost(0);
4471 format %{ %}
4472 interface(CONST_INTER);
4473 %}
4474
4475 operand immIOffset4()
4476 %{
4477 predicate(Address::offset_ok_for_immed(n->get_int(), 2));
4478 match(ConI);
4479
4480 op_cost(0);
4481 format %{ %}
4482 interface(CONST_INTER);
4483 %}
4484
4485 operand immIOffset8()
4486 %{
4487 predicate(Address::offset_ok_for_immed(n->get_int(), 3));
4488 match(ConI);
4489
4490 op_cost(0);
4491 format %{ %}
4492 interface(CONST_INTER);
4493 %}
4494
4495 operand immIOffset16()
4496 %{
4497 predicate(Address::offset_ok_for_immed(n->get_int(), 4));
4498 match(ConI);
4499
4500 op_cost(0);
4501 format %{ %}
4502 interface(CONST_INTER);
4503 %}
4504
4505 operand immLoffset()
4506 %{
4507 predicate(Address::offset_ok_for_immed(n->get_long(), 0));
4508 match(ConL);
4509
4510 op_cost(0);
4511 format %{ %}
4512 interface(CONST_INTER);
4513 %}
4514
4515 operand immLoffset1()
4516 %{
4517 predicate(Address::offset_ok_for_immed(n->get_long(), 0));
4518 match(ConL);
4519
4520 op_cost(0);
4521 format %{ %}
4522 interface(CONST_INTER);
4523 %}
4524
4525 operand immLoffset2()
4526 %{
4527 predicate(Address::offset_ok_for_immed(n->get_long(), 1));
4528 match(ConL);
4529
4530 op_cost(0);
4531 format %{ %}
4532 interface(CONST_INTER);
4533 %}
4534
4535 operand immLoffset4()
4536 %{
4537 predicate(Address::offset_ok_for_immed(n->get_long(), 2));
4538 match(ConL);
4539
4540 op_cost(0);
4541 format %{ %}
4542 interface(CONST_INTER);
4543 %}
4544
4545 operand immLoffset8()
4546 %{
4547 predicate(Address::offset_ok_for_immed(n->get_long(), 3));
4548 match(ConL);
4549
4550 op_cost(0);
4551 format %{ %}
4552 interface(CONST_INTER);
4553 %}
4554
4555 operand immLoffset16()
4556 %{
4557 predicate(Address::offset_ok_for_immed(n->get_long(), 4));
4558 match(ConL);
4559
4560 op_cost(0);
4561 format %{ %}
4562 interface(CONST_INTER);
4563 %}
4564
4565 // 8 bit signed value.
4566 operand immI8()
4567 %{
4568 predicate(n->get_int() <= 127 && n->get_int() >= -128);
4569 match(ConI);
4570
4571 op_cost(0);
4572 format %{ %}
4573 interface(CONST_INTER);
4574 %}
4575
4576 // 8 bit signed value (simm8), or #simm8 LSL 8.
4577 operand immI8_shift8()
4578 %{
4579 predicate((n->get_int() <= 127 && n->get_int() >= -128) ||
4580 (n->get_int() <= 32512 && n->get_int() >= -32768 && (n->get_int() & 0xff) == 0));
4581 match(ConI);
4582
4583 op_cost(0);
4584 format %{ %}
4585 interface(CONST_INTER);
4586 %}
4587
4588 // 8 bit signed value (simm8), or #simm8 LSL 8.
4589 operand immL8_shift8()
4590 %{
4591 predicate((n->get_long() <= 127 && n->get_long() >= -128) ||
4592 (n->get_long() <= 32512 && n->get_long() >= -32768 && (n->get_long() & 0xff) == 0));
4593 match(ConL);
4594
4595 op_cost(0);
4596 format %{ %}
4597 interface(CONST_INTER);
4598 %}
4599
4600 // 32 bit integer valid for add sub immediate
4601 operand immIAddSub()
4602 %{
4603 predicate(Assembler::operand_valid_for_add_sub_immediate((int64_t)n->get_int()));
4604 match(ConI);
4605 op_cost(0);
4606 format %{ %}
4607 interface(CONST_INTER);
4608 %}
4609
4610 // 32 bit unsigned integer valid for logical immediate
4611 // TODO -- check this is right when e.g the mask is 0x80000000
4612 operand immILog()
4613 %{
4614 predicate(Assembler::operand_valid_for_logical_immediate(/*is32*/true, (uint64_t)n->get_int()));
4615 match(ConI);
4616
4617 op_cost(0);
4618 format %{ %}
4619 interface(CONST_INTER);
4620 %}
4621
4622 // Integer operands 64 bit
4623 // 64 bit immediate
4624 operand immL()
4625 %{
4626 match(ConL);
4627
4628 op_cost(0);
4629 format %{ %}
4630 interface(CONST_INTER);
4631 %}
4632
4633 // 64 bit zero
4634 operand immL0()
4635 %{
4636 predicate(n->get_long() == 0);
4637 match(ConL);
4638
4639 op_cost(0);
4640 format %{ %}
4641 interface(CONST_INTER);
4642 %}
4643
4644 // 64 bit unit increment
4645 operand immL_1()
4646 %{
4647 predicate(n->get_long() == 1);
4648 match(ConL);
4649
4650 op_cost(0);
4651 format %{ %}
4652 interface(CONST_INTER);
4653 %}
4654
4655 // 64 bit unit decrement
4656 operand immL_M1()
4657 %{
4658 predicate(n->get_long() == -1);
4659 match(ConL);
4660
4661 op_cost(0);
4662 format %{ %}
4663 interface(CONST_INTER);
4664 %}
4665
4666 // 32 bit offset of pc in thread anchor
4667
4668 operand immL_pc_off()
4669 %{
4670 predicate(n->get_long() == in_bytes(JavaThread::frame_anchor_offset()) +
4671 in_bytes(JavaFrameAnchor::last_Java_pc_offset()));
4672 match(ConL);
4673
4674 op_cost(0);
4675 format %{ %}
4676 interface(CONST_INTER);
4677 %}
4678
4679 // 64 bit integer valid for add sub immediate
4680 operand immLAddSub()
4681 %{
4682 predicate(Assembler::operand_valid_for_add_sub_immediate(n->get_long()));
4683 match(ConL);
4684 op_cost(0);
4685 format %{ %}
4686 interface(CONST_INTER);
4687 %}
4688
4689 // 64 bit integer valid for logical immediate
4690 operand immLLog()
4691 %{
4692 predicate(Assembler::operand_valid_for_logical_immediate(/*is32*/false, (uint64_t)n->get_long()));
4693 match(ConL);
4694 op_cost(0);
4695 format %{ %}
4696 interface(CONST_INTER);
4697 %}
4698
4699 // Long Immediate: low 32-bit mask
4700 operand immL_32bits()
4701 %{
4702 predicate(n->get_long() == 0xFFFFFFFFL);
4703 match(ConL);
4704 op_cost(0);
4705 format %{ %}
4706 interface(CONST_INTER);
4707 %}
4708
4709 // Pointer operands
4710 // Pointer Immediate
4711 operand immP()
4712 %{
4713 match(ConP);
4714
4715 op_cost(0);
4716 format %{ %}
4717 interface(CONST_INTER);
4718 %}
4719
4720 // NULL Pointer Immediate
4721 operand immP0()
4722 %{
4723 predicate(n->get_ptr() == 0);
4724 match(ConP);
4725
4726 op_cost(0);
4727 format %{ %}
4728 interface(CONST_INTER);
4729 %}
4730
4731 // Pointer Immediate One
4732 // this is used in object initialization (initial object header)
4733 operand immP_1()
4734 %{
4735 predicate(n->get_ptr() == 1);
4736 match(ConP);
4737
4738 op_cost(0);
4739 format %{ %}
4740 interface(CONST_INTER);
4741 %}
4742
4743 // Card Table Byte Map Base
4744 operand immByteMapBase()
4745 %{
4746 // Get base of card map
4747 predicate(BarrierSet::barrier_set()->is_a(BarrierSet::CardTableBarrierSet) &&
4748 (CardTable::CardValue*)n->get_ptr() == ((CardTableBarrierSet*)(BarrierSet::barrier_set()))->card_table()->byte_map_base());
4749 match(ConP);
4750
4751 op_cost(0);
4752 format %{ %}
4753 interface(CONST_INTER);
4754 %}
4755
4756 // Pointer Immediate Minus One
4757 // this is used when we want to write the current PC to the thread anchor
4758 operand immP_M1()
4759 %{
4760 predicate(n->get_ptr() == -1);
4761 match(ConP);
4762
4763 op_cost(0);
4764 format %{ %}
4765 interface(CONST_INTER);
4766 %}
4767
4768 // Pointer Immediate Minus Two
4769 // this is used when we want to write the current PC to the thread anchor
4770 operand immP_M2()
4771 %{
4772 predicate(n->get_ptr() == -2);
4773 match(ConP);
4774
4775 op_cost(0);
4776 format %{ %}
4777 interface(CONST_INTER);
4778 %}
4779
4780 // Float and Double operands
4781 // Double Immediate
4782 operand immD()
4783 %{
4784 match(ConD);
4785 op_cost(0);
4786 format %{ %}
4787 interface(CONST_INTER);
4788 %}
4789
4790 // Double Immediate: +0.0d
4791 operand immD0()
4792 %{
4793 predicate(jlong_cast(n->getd()) == 0);
4794 match(ConD);
4795
4796 op_cost(0);
4797 format %{ %}
4798 interface(CONST_INTER);
4799 %}
4800
4801 // constant 'double +0.0'.
4802 operand immDPacked()
4803 %{
4804 predicate(Assembler::operand_valid_for_float_immediate(n->getd()));
4805 match(ConD);
4806 op_cost(0);
4807 format %{ %}
4808 interface(CONST_INTER);
4809 %}
4810
4811 // Float Immediate
4812 operand immF()
4813 %{
4814 match(ConF);
4815 op_cost(0);
4816 format %{ %}
4817 interface(CONST_INTER);
4818 %}
4819
4820 // Float Immediate: +0.0f.
4821 operand immF0()
4822 %{
4823 predicate(jint_cast(n->getf()) == 0);
4824 match(ConF);
4825
4826 op_cost(0);
4827 format %{ %}
4828 interface(CONST_INTER);
4829 %}
4830
4831 //
4832 operand immFPacked()
4833 %{
4834 predicate(Assembler::operand_valid_for_float_immediate((double)n->getf()));
4835 match(ConF);
4836 op_cost(0);
4837 format %{ %}
4838 interface(CONST_INTER);
4839 %}
4840
4841 // Narrow pointer operands
4842 // Narrow Pointer Immediate
4843 operand immN()
4844 %{
4845 match(ConN);
4846
4847 op_cost(0);
4848 format %{ %}
4849 interface(CONST_INTER);
4850 %}
4851
4852 // Narrow NULL Pointer Immediate
4853 operand immN0()
4854 %{
4855 predicate(n->get_narrowcon() == 0);
4856 match(ConN);
4857
4858 op_cost(0);
4859 format %{ %}
4860 interface(CONST_INTER);
4861 %}
4862
4863 operand immNKlass()
4864 %{
4865 match(ConNKlass);
4866
4867 op_cost(0);
4868 format %{ %}
4869 interface(CONST_INTER);
4870 %}
4871
4872 // Integer 32 bit Register Operands
4873 // Integer 32 bitRegister (excludes SP)
4874 operand iRegI()
4875 %{
4876 constraint(ALLOC_IN_RC(any_reg32));
4877 match(RegI);
4878 match(iRegINoSp);
4879 op_cost(0);
4880 format %{ %}
4881 interface(REG_INTER);
4882 %}
4883
4884 // Integer 32 bit Register not Special
4885 operand iRegINoSp()
4886 %{
4887 constraint(ALLOC_IN_RC(no_special_reg32));
4888 match(RegI);
4889 op_cost(0);
4890 format %{ %}
4891 interface(REG_INTER);
4892 %}
4893
4894 // Integer 64 bit Register Operands
4895 // Integer 64 bit Register (includes SP)
4896 operand iRegL()
4897 %{
4898 constraint(ALLOC_IN_RC(any_reg));
4899 match(RegL);
4900 match(iRegLNoSp);
4901 op_cost(0);
4902 format %{ %}
4903 interface(REG_INTER);
4904 %}
4905
4906 // Integer 64 bit Register not Special
4907 operand iRegLNoSp()
4908 %{
4909 constraint(ALLOC_IN_RC(no_special_reg));
4910 match(RegL);
4911 match(iRegL_R0);
4912 format %{ %}
4913 interface(REG_INTER);
4914 %}
4915
4916 // Pointer Register Operands
4917 // Pointer Register
4918 operand iRegP()
4919 %{
4920 constraint(ALLOC_IN_RC(ptr_reg));
4921 match(RegP);
4922 match(iRegPNoSp);
4923 match(iRegP_R0);
4924 //match(iRegP_R2);
4925 //match(iRegP_R4);
4926 match(iRegP_R5);
4927 match(thread_RegP);
4928 op_cost(0);
4929 format %{ %}
4930 interface(REG_INTER);
4931 %}
4932
4933 // Pointer 64 bit Register not Special
4934 operand iRegPNoSp()
4935 %{
4936 constraint(ALLOC_IN_RC(no_special_ptr_reg));
4937 match(RegP);
4938 // match(iRegP);
4939 // match(iRegP_R0);
4940 // match(iRegP_R2);
4941 // match(iRegP_R4);
4942 // match(iRegP_R5);
4943 // match(thread_RegP);
4944 op_cost(0);
4945 format %{ %}
4946 interface(REG_INTER);
4947 %}
4948
4949 // Pointer 64 bit Register R0 only
4950 operand iRegP_R0()
4951 %{
4952 constraint(ALLOC_IN_RC(r0_reg));
4953 match(RegP);
4954 // match(iRegP);
4955 match(iRegPNoSp);
4956 op_cost(0);
4957 format %{ %}
4958 interface(REG_INTER);
4959 %}
4960
4961 // Pointer 64 bit Register R1 only
4962 operand iRegP_R1()
4963 %{
4964 constraint(ALLOC_IN_RC(r1_reg));
4965 match(RegP);
4966 // match(iRegP);
4967 match(iRegPNoSp);
4968 op_cost(0);
4969 format %{ %}
4970 interface(REG_INTER);
4971 %}
4972
4973 // Pointer 64 bit Register R2 only
4974 operand iRegP_R2()
4975 %{
4976 constraint(ALLOC_IN_RC(r2_reg));
4977 match(RegP);
4978 // match(iRegP);
4979 match(iRegPNoSp);
4980 op_cost(0);
4981 format %{ %}
4982 interface(REG_INTER);
4983 %}
4984
4985 // Pointer 64 bit Register R3 only
4986 operand iRegP_R3()
4987 %{
4988 constraint(ALLOC_IN_RC(r3_reg));
4989 match(RegP);
4990 // match(iRegP);
4991 match(iRegPNoSp);
4992 op_cost(0);
4993 format %{ %}
4994 interface(REG_INTER);
4995 %}
4996
4997 // Pointer 64 bit Register R4 only
4998 operand iRegP_R4()
4999 %{
5000 constraint(ALLOC_IN_RC(r4_reg));
5001 match(RegP);
5002 // match(iRegP);
5003 match(iRegPNoSp);
5004 op_cost(0);
5005 format %{ %}
5006 interface(REG_INTER);
5007 %}
5008
5009 // Pointer 64 bit Register R5 only
5010 operand iRegP_R5()
5011 %{
5012 constraint(ALLOC_IN_RC(r5_reg));
5013 match(RegP);
5014 // match(iRegP);
5015 match(iRegPNoSp);
5016 op_cost(0);
5017 format %{ %}
5018 interface(REG_INTER);
5019 %}
5020
5021 // Pointer 64 bit Register R10 only
5022 operand iRegP_R10()
5023 %{
5024 constraint(ALLOC_IN_RC(r10_reg));
5025 match(RegP);
5026 // match(iRegP);
5027 match(iRegPNoSp);
5028 op_cost(0);
5029 format %{ %}
5030 interface(REG_INTER);
5031 %}
5032
5033 // Long 64 bit Register R0 only
5034 operand iRegL_R0()
5035 %{
5036 constraint(ALLOC_IN_RC(r0_reg));
5037 match(RegL);
5038 match(iRegLNoSp);
5039 op_cost(0);
5040 format %{ %}
5041 interface(REG_INTER);
5042 %}
5043
5044 // Long 64 bit Register R2 only
5045 operand iRegL_R2()
5046 %{
5047 constraint(ALLOC_IN_RC(r2_reg));
5048 match(RegL);
5049 match(iRegLNoSp);
5050 op_cost(0);
5051 format %{ %}
5052 interface(REG_INTER);
5053 %}
5054
5055 // Long 64 bit Register R3 only
5056 operand iRegL_R3()
5057 %{
5058 constraint(ALLOC_IN_RC(r3_reg));
5059 match(RegL);
5060 match(iRegLNoSp);
5061 op_cost(0);
5062 format %{ %}
5063 interface(REG_INTER);
5064 %}
5065
5066 // Long 64 bit Register R11 only
5067 operand iRegL_R11()
5068 %{
5069 constraint(ALLOC_IN_RC(r11_reg));
5070 match(RegL);
5071 match(iRegLNoSp);
5072 op_cost(0);
5073 format %{ %}
5074 interface(REG_INTER);
5075 %}
5076
5077 // Pointer 64 bit Register FP only
5078 operand iRegP_FP()
5079 %{
5080 constraint(ALLOC_IN_RC(fp_reg));
5081 match(RegP);
5082 // match(iRegP);
5083 op_cost(0);
5084 format %{ %}
5085 interface(REG_INTER);
5086 %}
5087
5088 // Register R0 only
5089 operand iRegI_R0()
5090 %{
5091 constraint(ALLOC_IN_RC(int_r0_reg));
5092 match(RegI);
5093 match(iRegINoSp);
5094 op_cost(0);
5095 format %{ %}
5096 interface(REG_INTER);
5097 %}
5098
5099 // Register R2 only
5100 operand iRegI_R2()
5101 %{
5102 constraint(ALLOC_IN_RC(int_r2_reg));
5103 match(RegI);
5104 match(iRegINoSp);
5105 op_cost(0);
5106 format %{ %}
5107 interface(REG_INTER);
5108 %}
5109
5110 // Register R3 only
5111 operand iRegI_R3()
5112 %{
5113 constraint(ALLOC_IN_RC(int_r3_reg));
5114 match(RegI);
5115 match(iRegINoSp);
5116 op_cost(0);
5117 format %{ %}
5118 interface(REG_INTER);
5119 %}
5120
5121
5122 // Register R4 only
5123 operand iRegI_R4()
5124 %{
5125 constraint(ALLOC_IN_RC(int_r4_reg));
5126 match(RegI);
5127 match(iRegINoSp);
5128 op_cost(0);
5129 format %{ %}
5130 interface(REG_INTER);
5131 %}
5132
5133
5134 // Pointer Register Operands
5135 // Narrow Pointer Register
5136 operand iRegN()
5137 %{
5138 constraint(ALLOC_IN_RC(any_reg32));
5139 match(RegN);
5140 match(iRegNNoSp);
5141 op_cost(0);
5142 format %{ %}
5143 interface(REG_INTER);
5144 %}
5145
5146 operand iRegN_R0()
5147 %{
5148 constraint(ALLOC_IN_RC(r0_reg));
5149 match(iRegN);
5150 op_cost(0);
5151 format %{ %}
5152 interface(REG_INTER);
5153 %}
5154
5155 operand iRegN_R2()
5156 %{
5157 constraint(ALLOC_IN_RC(r2_reg));
5158 match(iRegN);
5159 op_cost(0);
5160 format %{ %}
5161 interface(REG_INTER);
5162 %}
5163
5164 operand iRegN_R3()
5165 %{
5166 constraint(ALLOC_IN_RC(r3_reg));
5167 match(iRegN);
5168 op_cost(0);
5169 format %{ %}
5170 interface(REG_INTER);
5171 %}
5172
5173 // Integer 64 bit Register not Special
5174 operand iRegNNoSp()
5175 %{
5176 constraint(ALLOC_IN_RC(no_special_reg32));
5177 match(RegN);
5178 op_cost(0);
5179 format %{ %}
5180 interface(REG_INTER);
5181 %}
5182
5183 // heap base register -- used for encoding immN0
5184
5185 operand iRegIHeapbase()
5186 %{
5187 constraint(ALLOC_IN_RC(heapbase_reg));
5188 match(RegI);
5189 op_cost(0);
5190 format %{ %}
5191 interface(REG_INTER);
5192 %}
5193
5194 // Float Register
5195 // Float register operands
5196 operand vRegF()
5197 %{
5198 constraint(ALLOC_IN_RC(float_reg));
5199 match(RegF);
5200
5201 op_cost(0);
5202 format %{ %}
5203 interface(REG_INTER);
5204 %}
5205
5206 // Double Register
5207 // Double register operands
5208 operand vRegD()
5209 %{
5210 constraint(ALLOC_IN_RC(double_reg));
5211 match(RegD);
5212
5213 op_cost(0);
5214 format %{ %}
5215 interface(REG_INTER);
5216 %}
5217
5218 // Generic vector class. This will be used for
5219 // all vector operands, including NEON and SVE,
5220 // but currently only used for SVE VecA.
5221 operand vReg()
5222 %{
5223 constraint(ALLOC_IN_RC(vectora_reg));
5224 match(VecA);
5225 op_cost(0);
5226 format %{ %}
5227 interface(REG_INTER);
5228 %}
5229
5230 operand vecD()
5231 %{
5232 constraint(ALLOC_IN_RC(vectord_reg));
5233 match(VecD);
5234
5235 op_cost(0);
5236 format %{ %}
5237 interface(REG_INTER);
5238 %}
5239
5240 operand vecX()
5241 %{
5242 constraint(ALLOC_IN_RC(vectorx_reg));
5243 match(VecX);
5244
5245 op_cost(0);
5246 format %{ %}
5247 interface(REG_INTER);
5248 %}
5249
5250 operand vRegD_V0()
5251 %{
5252 constraint(ALLOC_IN_RC(v0_reg));
5253 match(RegD);
5254 op_cost(0);
5255 format %{ %}
5256 interface(REG_INTER);
5257 %}
5258
5259 operand vRegD_V1()
5260 %{
5261 constraint(ALLOC_IN_RC(v1_reg));
5262 match(RegD);
5263 op_cost(0);
5264 format %{ %}
5265 interface(REG_INTER);
5266 %}
5267
5268 operand vRegD_V2()
5269 %{
5270 constraint(ALLOC_IN_RC(v2_reg));
5271 match(RegD);
5272 op_cost(0);
5273 format %{ %}
5274 interface(REG_INTER);
5275 %}
5276
5277 operand vRegD_V3()
5278 %{
5279 constraint(ALLOC_IN_RC(v3_reg));
5280 match(RegD);
5281 op_cost(0);
5282 format %{ %}
5283 interface(REG_INTER);
5284 %}
5285
5286 operand vRegD_V4()
5287 %{
5288 constraint(ALLOC_IN_RC(v4_reg));
5289 match(RegD);
5290 op_cost(0);
5291 format %{ %}
5292 interface(REG_INTER);
5293 %}
5294
5295 operand vRegD_V5()
5296 %{
5297 constraint(ALLOC_IN_RC(v5_reg));
5298 match(RegD);
5299 op_cost(0);
5300 format %{ %}
5301 interface(REG_INTER);
5302 %}
5303
5304 operand vRegD_V6()
5305 %{
5306 constraint(ALLOC_IN_RC(v6_reg));
5307 match(RegD);
5308 op_cost(0);
5309 format %{ %}
5310 interface(REG_INTER);
5311 %}
5312
5313 operand vRegD_V7()
5314 %{
5315 constraint(ALLOC_IN_RC(v7_reg));
5316 match(RegD);
5317 op_cost(0);
5318 format %{ %}
5319 interface(REG_INTER);
5320 %}
5321
5322 operand vRegD_V8()
5323 %{
5324 constraint(ALLOC_IN_RC(v8_reg));
5325 match(RegD);
5326 op_cost(0);
5327 format %{ %}
5328 interface(REG_INTER);
5329 %}
5330
5331 operand vRegD_V9()
5332 %{
5333 constraint(ALLOC_IN_RC(v9_reg));
5334 match(RegD);
5335 op_cost(0);
5336 format %{ %}
5337 interface(REG_INTER);
5338 %}
5339
5340 operand vRegD_V10()
5341 %{
5342 constraint(ALLOC_IN_RC(v10_reg));
5343 match(RegD);
5344 op_cost(0);
5345 format %{ %}
5346 interface(REG_INTER);
5347 %}
5348
5349 operand vRegD_V11()
5350 %{
5351 constraint(ALLOC_IN_RC(v11_reg));
5352 match(RegD);
5353 op_cost(0);
5354 format %{ %}
5355 interface(REG_INTER);
5356 %}
5357
5358 operand vRegD_V12()
5359 %{
5360 constraint(ALLOC_IN_RC(v12_reg));
5361 match(RegD);
5362 op_cost(0);
5363 format %{ %}
5364 interface(REG_INTER);
5365 %}
5366
5367 operand vRegD_V13()
5368 %{
5369 constraint(ALLOC_IN_RC(v13_reg));
5370 match(RegD);
5371 op_cost(0);
5372 format %{ %}
5373 interface(REG_INTER);
5374 %}
5375
5376 operand vRegD_V14()
5377 %{
5378 constraint(ALLOC_IN_RC(v14_reg));
5379 match(RegD);
5380 op_cost(0);
5381 format %{ %}
5382 interface(REG_INTER);
5383 %}
5384
5385 operand vRegD_V15()
5386 %{
5387 constraint(ALLOC_IN_RC(v15_reg));
5388 match(RegD);
5389 op_cost(0);
5390 format %{ %}
5391 interface(REG_INTER);
5392 %}
5393
5394 operand vRegD_V16()
5395 %{
5396 constraint(ALLOC_IN_RC(v16_reg));
5397 match(RegD);
5398 op_cost(0);
5399 format %{ %}
5400 interface(REG_INTER);
5401 %}
5402
5403 operand vRegD_V17()
5404 %{
5405 constraint(ALLOC_IN_RC(v17_reg));
5406 match(RegD);
5407 op_cost(0);
5408 format %{ %}
5409 interface(REG_INTER);
5410 %}
5411
5412 operand vRegD_V18()
5413 %{
5414 constraint(ALLOC_IN_RC(v18_reg));
5415 match(RegD);
5416 op_cost(0);
5417 format %{ %}
5418 interface(REG_INTER);
5419 %}
5420
5421 operand vRegD_V19()
5422 %{
5423 constraint(ALLOC_IN_RC(v19_reg));
5424 match(RegD);
5425 op_cost(0);
5426 format %{ %}
5427 interface(REG_INTER);
5428 %}
5429
5430 operand vRegD_V20()
5431 %{
5432 constraint(ALLOC_IN_RC(v20_reg));
5433 match(RegD);
5434 op_cost(0);
5435 format %{ %}
5436 interface(REG_INTER);
5437 %}
5438
5439 operand vRegD_V21()
5440 %{
5441 constraint(ALLOC_IN_RC(v21_reg));
5442 match(RegD);
5443 op_cost(0);
5444 format %{ %}
5445 interface(REG_INTER);
5446 %}
5447
5448 operand vRegD_V22()
5449 %{
5450 constraint(ALLOC_IN_RC(v22_reg));
5451 match(RegD);
5452 op_cost(0);
5453 format %{ %}
5454 interface(REG_INTER);
5455 %}
5456
5457 operand vRegD_V23()
5458 %{
5459 constraint(ALLOC_IN_RC(v23_reg));
5460 match(RegD);
5461 op_cost(0);
5462 format %{ %}
5463 interface(REG_INTER);
5464 %}
5465
5466 operand vRegD_V24()
5467 %{
5468 constraint(ALLOC_IN_RC(v24_reg));
5469 match(RegD);
5470 op_cost(0);
5471 format %{ %}
5472 interface(REG_INTER);
5473 %}
5474
5475 operand vRegD_V25()
5476 %{
5477 constraint(ALLOC_IN_RC(v25_reg));
5478 match(RegD);
5479 op_cost(0);
5480 format %{ %}
5481 interface(REG_INTER);
5482 %}
5483
5484 operand vRegD_V26()
5485 %{
5486 constraint(ALLOC_IN_RC(v26_reg));
5487 match(RegD);
5488 op_cost(0);
5489 format %{ %}
5490 interface(REG_INTER);
5491 %}
5492
5493 operand vRegD_V27()
5494 %{
5495 constraint(ALLOC_IN_RC(v27_reg));
5496 match(RegD);
5497 op_cost(0);
5498 format %{ %}
5499 interface(REG_INTER);
5500 %}
5501
5502 operand vRegD_V28()
5503 %{
5504 constraint(ALLOC_IN_RC(v28_reg));
5505 match(RegD);
5506 op_cost(0);
5507 format %{ %}
5508 interface(REG_INTER);
5509 %}
5510
5511 operand vRegD_V29()
5512 %{
5513 constraint(ALLOC_IN_RC(v29_reg));
5514 match(RegD);
5515 op_cost(0);
5516 format %{ %}
5517 interface(REG_INTER);
5518 %}
5519
5520 operand vRegD_V30()
5521 %{
5522 constraint(ALLOC_IN_RC(v30_reg));
5523 match(RegD);
5524 op_cost(0);
5525 format %{ %}
5526 interface(REG_INTER);
5527 %}
5528
5529 operand vRegD_V31()
5530 %{
5531 constraint(ALLOC_IN_RC(v31_reg));
5532 match(RegD);
5533 op_cost(0);
5534 format %{ %}
5535 interface(REG_INTER);
5536 %}
5537
5538 operand pRegGov()
5539 %{
5540 constraint(ALLOC_IN_RC(gov_pr));
5541 match(RegVectMask);
5542 op_cost(0);
5543 format %{ %}
5544 interface(REG_INTER);
5545 %}
5546
5547 // Flags register, used as output of signed compare instructions
5548
5549 // note that on AArch64 we also use this register as the output for
5550 // for floating point compare instructions (CmpF CmpD). this ensures
5551 // that ordered inequality tests use GT, GE, LT or LE none of which
5552 // pass through cases where the result is unordered i.e. one or both
5553 // inputs to the compare is a NaN. this means that the ideal code can
5554 // replace e.g. a GT with an LE and not end up capturing the NaN case
5555 // (where the comparison should always fail). EQ and NE tests are
5556 // always generated in ideal code so that unordered folds into the NE
5557 // case, matching the behaviour of AArch64 NE.
5558 //
5559 // This differs from x86 where the outputs of FP compares use a
5560 // special FP flags registers and where compares based on this
5561 // register are distinguished into ordered inequalities (cmpOpUCF) and
5562 // EQ/NEQ tests (cmpOpUCF2). x86 has to special case the latter tests
5563 // to explicitly handle the unordered case in branches. x86 also has
5564 // to include extra CMoveX rules to accept a cmpOpUCF input.
5565
5566 operand rFlagsReg()
5567 %{
5568 constraint(ALLOC_IN_RC(int_flags));
5569 match(RegFlags);
5570
5571 op_cost(0);
5572 format %{ "RFLAGS" %}
5573 interface(REG_INTER);
5574 %}
5575
5576 // Flags register, used as output of unsigned compare instructions
5577 operand rFlagsRegU()
5578 %{
5579 constraint(ALLOC_IN_RC(int_flags));
5580 match(RegFlags);
5581
5582 op_cost(0);
5583 format %{ "RFLAGSU" %}
5584 interface(REG_INTER);
5585 %}
5586
5587 // Special Registers
5588
5589 // Method Register
5590 operand inline_cache_RegP(iRegP reg)
5591 %{
5592 constraint(ALLOC_IN_RC(method_reg)); // inline_cache_reg
5593 match(reg);
5594 match(iRegPNoSp);
5595 op_cost(0);
5596 format %{ %}
5597 interface(REG_INTER);
5598 %}
5599
5600 // Thread Register
5601 operand thread_RegP(iRegP reg)
5602 %{
5603 constraint(ALLOC_IN_RC(thread_reg)); // link_reg
5604 match(reg);
5605 op_cost(0);
5606 format %{ %}
5607 interface(REG_INTER);
5608 %}
5609
5610 operand lr_RegP(iRegP reg)
5611 %{
5612 constraint(ALLOC_IN_RC(lr_reg)); // link_reg
5613 match(reg);
5614 op_cost(0);
5615 format %{ %}
5616 interface(REG_INTER);
5617 %}
5618
5619 //----------Memory Operands----------------------------------------------------
5620
5621 operand indirect(iRegP reg)
5622 %{
5623 constraint(ALLOC_IN_RC(ptr_reg));
5624 match(reg);
5625 op_cost(0);
5626 format %{ "[$reg]" %}
5627 interface(MEMORY_INTER) %{
5628 base($reg);
5629 index(0xffffffff);
5630 scale(0x0);
5631 disp(0x0);
5632 %}
5633 %}
5634
5635 operand indIndexScaledI2L(iRegP reg, iRegI ireg, immIScale scale)
5636 %{
5637 constraint(ALLOC_IN_RC(ptr_reg));
5638 predicate(size_fits_all_mem_uses(n->as_AddP(), n->in(AddPNode::Offset)->in(2)->get_int()));
5639 match(AddP reg (LShiftL (ConvI2L ireg) scale));
5640 op_cost(0);
5641 format %{ "$reg, $ireg sxtw($scale), 0, I2L" %}
5642 interface(MEMORY_INTER) %{
5643 base($reg);
5644 index($ireg);
5645 scale($scale);
5646 disp(0x0);
5647 %}
5648 %}
5649
5650 operand indIndexScaled(iRegP reg, iRegL lreg, immIScale scale)
5651 %{
5652 constraint(ALLOC_IN_RC(ptr_reg));
5653 predicate(size_fits_all_mem_uses(n->as_AddP(), n->in(AddPNode::Offset)->in(2)->get_int()));
5654 match(AddP reg (LShiftL lreg scale));
5655 op_cost(0);
5656 format %{ "$reg, $lreg lsl($scale)" %}
5657 interface(MEMORY_INTER) %{
5658 base($reg);
5659 index($lreg);
5660 scale($scale);
5661 disp(0x0);
5662 %}
5663 %}
5664
5665 operand indIndexI2L(iRegP reg, iRegI ireg)
5666 %{
5667 constraint(ALLOC_IN_RC(ptr_reg));
5668 match(AddP reg (ConvI2L ireg));
5669 op_cost(0);
5670 format %{ "$reg, $ireg, 0, I2L" %}
5671 interface(MEMORY_INTER) %{
5672 base($reg);
5673 index($ireg);
5674 scale(0x0);
5675 disp(0x0);
5676 %}
5677 %}
5678
5679 operand indIndex(iRegP reg, iRegL lreg)
5680 %{
5681 constraint(ALLOC_IN_RC(ptr_reg));
5682 match(AddP reg lreg);
5683 op_cost(0);
5684 format %{ "$reg, $lreg" %}
5685 interface(MEMORY_INTER) %{
5686 base($reg);
5687 index($lreg);
5688 scale(0x0);
5689 disp(0x0);
5690 %}
5691 %}
5692
5693 operand indOffI(iRegP reg, immIOffset off)
5694 %{
5695 constraint(ALLOC_IN_RC(ptr_reg));
5696 match(AddP reg off);
5697 op_cost(0);
5698 format %{ "[$reg, $off]" %}
5699 interface(MEMORY_INTER) %{
5700 base($reg);
5701 index(0xffffffff);
5702 scale(0x0);
5703 disp($off);
5704 %}
5705 %}
5706
5707 operand indOffI1(iRegP reg, immIOffset1 off)
5708 %{
5709 constraint(ALLOC_IN_RC(ptr_reg));
5710 match(AddP reg off);
5711 op_cost(0);
5712 format %{ "[$reg, $off]" %}
5713 interface(MEMORY_INTER) %{
5714 base($reg);
5715 index(0xffffffff);
5716 scale(0x0);
5717 disp($off);
5718 %}
5719 %}
5720
5721 operand indOffI2(iRegP reg, immIOffset2 off)
5722 %{
5723 constraint(ALLOC_IN_RC(ptr_reg));
5724 match(AddP reg off);
5725 op_cost(0);
5726 format %{ "[$reg, $off]" %}
5727 interface(MEMORY_INTER) %{
5728 base($reg);
5729 index(0xffffffff);
5730 scale(0x0);
5731 disp($off);
5732 %}
5733 %}
5734
5735 operand indOffI4(iRegP reg, immIOffset4 off)
5736 %{
5737 constraint(ALLOC_IN_RC(ptr_reg));
5738 match(AddP reg off);
5739 op_cost(0);
5740 format %{ "[$reg, $off]" %}
5741 interface(MEMORY_INTER) %{
5742 base($reg);
5743 index(0xffffffff);
5744 scale(0x0);
5745 disp($off);
5746 %}
5747 %}
5748
5749 operand indOffI8(iRegP reg, immIOffset8 off)
5750 %{
5751 constraint(ALLOC_IN_RC(ptr_reg));
5752 match(AddP reg off);
5753 op_cost(0);
5754 format %{ "[$reg, $off]" %}
5755 interface(MEMORY_INTER) %{
5756 base($reg);
5757 index(0xffffffff);
5758 scale(0x0);
5759 disp($off);
5760 %}
5761 %}
5762
5763 operand indOffI16(iRegP reg, immIOffset16 off)
5764 %{
5765 constraint(ALLOC_IN_RC(ptr_reg));
5766 match(AddP reg off);
5767 op_cost(0);
5768 format %{ "[$reg, $off]" %}
5769 interface(MEMORY_INTER) %{
5770 base($reg);
5771 index(0xffffffff);
5772 scale(0x0);
5773 disp($off);
5774 %}
5775 %}
5776
5777 operand indOffL(iRegP reg, immLoffset off)
5778 %{
5779 constraint(ALLOC_IN_RC(ptr_reg));
5780 match(AddP reg off);
5781 op_cost(0);
5782 format %{ "[$reg, $off]" %}
5783 interface(MEMORY_INTER) %{
5784 base($reg);
5785 index(0xffffffff);
5786 scale(0x0);
5787 disp($off);
5788 %}
5789 %}
5790
5791 operand indOffL1(iRegP reg, immLoffset1 off)
5792 %{
5793 constraint(ALLOC_IN_RC(ptr_reg));
5794 match(AddP reg off);
5795 op_cost(0);
5796 format %{ "[$reg, $off]" %}
5797 interface(MEMORY_INTER) %{
5798 base($reg);
5799 index(0xffffffff);
5800 scale(0x0);
5801 disp($off);
5802 %}
5803 %}
5804
5805 operand indOffL2(iRegP reg, immLoffset2 off)
5806 %{
5807 constraint(ALLOC_IN_RC(ptr_reg));
5808 match(AddP reg off);
5809 op_cost(0);
5810 format %{ "[$reg, $off]" %}
5811 interface(MEMORY_INTER) %{
5812 base($reg);
5813 index(0xffffffff);
5814 scale(0x0);
5815 disp($off);
5816 %}
5817 %}
5818
5819 operand indOffL4(iRegP reg, immLoffset4 off)
5820 %{
5821 constraint(ALLOC_IN_RC(ptr_reg));
5822 match(AddP reg off);
5823 op_cost(0);
5824 format %{ "[$reg, $off]" %}
5825 interface(MEMORY_INTER) %{
5826 base($reg);
5827 index(0xffffffff);
5828 scale(0x0);
5829 disp($off);
5830 %}
5831 %}
5832
5833 operand indOffL8(iRegP reg, immLoffset8 off)
5834 %{
5835 constraint(ALLOC_IN_RC(ptr_reg));
5836 match(AddP reg off);
5837 op_cost(0);
5838 format %{ "[$reg, $off]" %}
5839 interface(MEMORY_INTER) %{
5840 base($reg);
5841 index(0xffffffff);
5842 scale(0x0);
5843 disp($off);
5844 %}
5845 %}
5846
5847 operand indOffL16(iRegP reg, immLoffset16 off)
5848 %{
5849 constraint(ALLOC_IN_RC(ptr_reg));
5850 match(AddP reg off);
5851 op_cost(0);
5852 format %{ "[$reg, $off]" %}
5853 interface(MEMORY_INTER) %{
5854 base($reg);
5855 index(0xffffffff);
5856 scale(0x0);
5857 disp($off);
5858 %}
5859 %}
5860
5861 operand indirectN(iRegN reg)
5862 %{
5863 predicate(CompressedOops::shift() == 0);
5864 constraint(ALLOC_IN_RC(ptr_reg));
5865 match(DecodeN reg);
5866 op_cost(0);
5867 format %{ "[$reg]\t# narrow" %}
5868 interface(MEMORY_INTER) %{
5869 base($reg);
5870 index(0xffffffff);
5871 scale(0x0);
5872 disp(0x0);
5873 %}
5874 %}
5875
5876 operand indIndexScaledI2LN(iRegN reg, iRegI ireg, immIScale scale)
5877 %{
5878 predicate(CompressedOops::shift() == 0 && size_fits_all_mem_uses(n->as_AddP(), n->in(AddPNode::Offset)->in(2)->get_int()));
5879 constraint(ALLOC_IN_RC(ptr_reg));
5880 match(AddP (DecodeN reg) (LShiftL (ConvI2L ireg) scale));
5881 op_cost(0);
5882 format %{ "$reg, $ireg sxtw($scale), 0, I2L\t# narrow" %}
5883 interface(MEMORY_INTER) %{
5884 base($reg);
5885 index($ireg);
5886 scale($scale);
5887 disp(0x0);
5888 %}
5889 %}
5890
5891 operand indIndexScaledN(iRegN reg, iRegL lreg, immIScale scale)
5892 %{
5893 predicate(CompressedOops::shift() == 0 && size_fits_all_mem_uses(n->as_AddP(), n->in(AddPNode::Offset)->in(2)->get_int()));
5894 constraint(ALLOC_IN_RC(ptr_reg));
5895 match(AddP (DecodeN reg) (LShiftL lreg scale));
5896 op_cost(0);
5897 format %{ "$reg, $lreg lsl($scale)\t# narrow" %}
5898 interface(MEMORY_INTER) %{
5899 base($reg);
5900 index($lreg);
5901 scale($scale);
5902 disp(0x0);
5903 %}
5904 %}
5905
5906 operand indIndexI2LN(iRegN reg, iRegI ireg)
5907 %{
5908 predicate(CompressedOops::shift() == 0);
5909 constraint(ALLOC_IN_RC(ptr_reg));
5910 match(AddP (DecodeN reg) (ConvI2L ireg));
5911 op_cost(0);
5912 format %{ "$reg, $ireg, 0, I2L\t# narrow" %}
5913 interface(MEMORY_INTER) %{
5914 base($reg);
5915 index($ireg);
5916 scale(0x0);
5917 disp(0x0);
5918 %}
5919 %}
5920
5921 operand indIndexN(iRegN reg, iRegL lreg)
5922 %{
5923 predicate(CompressedOops::shift() == 0);
5924 constraint(ALLOC_IN_RC(ptr_reg));
5925 match(AddP (DecodeN reg) lreg);
5926 op_cost(0);
5927 format %{ "$reg, $lreg\t# narrow" %}
5928 interface(MEMORY_INTER) %{
5929 base($reg);
5930 index($lreg);
5931 scale(0x0);
5932 disp(0x0);
5933 %}
5934 %}
5935
5936 operand indOffIN(iRegN reg, immIOffset off)
5937 %{
5938 predicate(CompressedOops::shift() == 0);
5939 constraint(ALLOC_IN_RC(ptr_reg));
5940 match(AddP (DecodeN reg) off);
5941 op_cost(0);
5942 format %{ "[$reg, $off]\t# narrow" %}
5943 interface(MEMORY_INTER) %{
5944 base($reg);
5945 index(0xffffffff);
5946 scale(0x0);
5947 disp($off);
5948 %}
5949 %}
5950
5951 operand indOffLN(iRegN reg, immLoffset off)
5952 %{
5953 predicate(CompressedOops::shift() == 0);
5954 constraint(ALLOC_IN_RC(ptr_reg));
5955 match(AddP (DecodeN reg) off);
5956 op_cost(0);
5957 format %{ "[$reg, $off]\t# narrow" %}
5958 interface(MEMORY_INTER) %{
5959 base($reg);
5960 index(0xffffffff);
5961 scale(0x0);
5962 disp($off);
5963 %}
5964 %}
5965
5966
5967
5968 // AArch64 opto stubs need to write to the pc slot in the thread anchor
5969 operand thread_anchor_pc(thread_RegP reg, immL_pc_off off)
5970 %{
5971 constraint(ALLOC_IN_RC(ptr_reg));
5972 match(AddP reg off);
5973 op_cost(0);
5974 format %{ "[$reg, $off]" %}
5975 interface(MEMORY_INTER) %{
5976 base($reg);
5977 index(0xffffffff);
5978 scale(0x0);
5979 disp($off);
5980 %}
5981 %}
5982
5983 //----------Special Memory Operands--------------------------------------------
5984 // Stack Slot Operand - This operand is used for loading and storing temporary
5985 // values on the stack where a match requires a value to
5986 // flow through memory.
5987 operand stackSlotP(sRegP reg)
5988 %{
5989 constraint(ALLOC_IN_RC(stack_slots));
5990 op_cost(100);
5991 // No match rule because this operand is only generated in matching
5992 // match(RegP);
5993 format %{ "[$reg]" %}
5994 interface(MEMORY_INTER) %{
5995 base(0x1e); // RSP
5996 index(0x0); // No Index
5997 scale(0x0); // No Scale
5998 disp($reg); // Stack Offset
5999 %}
6000 %}
6001
6002 operand stackSlotI(sRegI reg)
6003 %{
6004 constraint(ALLOC_IN_RC(stack_slots));
6005 // No match rule because this operand is only generated in matching
6006 // match(RegI);
6007 format %{ "[$reg]" %}
6008 interface(MEMORY_INTER) %{
6009 base(0x1e); // RSP
6010 index(0x0); // No Index
6011 scale(0x0); // No Scale
6012 disp($reg); // Stack Offset
6013 %}
6014 %}
6015
6016 operand stackSlotF(sRegF reg)
6017 %{
6018 constraint(ALLOC_IN_RC(stack_slots));
6019 // No match rule because this operand is only generated in matching
6020 // match(RegF);
6021 format %{ "[$reg]" %}
6022 interface(MEMORY_INTER) %{
6023 base(0x1e); // RSP
6024 index(0x0); // No Index
6025 scale(0x0); // No Scale
6026 disp($reg); // Stack Offset
6027 %}
6028 %}
6029
6030 operand stackSlotD(sRegD reg)
6031 %{
6032 constraint(ALLOC_IN_RC(stack_slots));
6033 // No match rule because this operand is only generated in matching
6034 // match(RegD);
6035 format %{ "[$reg]" %}
6036 interface(MEMORY_INTER) %{
6037 base(0x1e); // RSP
6038 index(0x0); // No Index
6039 scale(0x0); // No Scale
6040 disp($reg); // Stack Offset
6041 %}
6042 %}
6043
6044 operand stackSlotL(sRegL reg)
6045 %{
6046 constraint(ALLOC_IN_RC(stack_slots));
6047 // No match rule because this operand is only generated in matching
6048 // match(RegL);
6049 format %{ "[$reg]" %}
6050 interface(MEMORY_INTER) %{
6051 base(0x1e); // RSP
6052 index(0x0); // No Index
6053 scale(0x0); // No Scale
6054 disp($reg); // Stack Offset
6055 %}
6056 %}
6057
6058 // Operands for expressing Control Flow
6059 // NOTE: Label is a predefined operand which should not be redefined in
6060 // the AD file. It is generically handled within the ADLC.
6061
6062 //----------Conditional Branch Operands----------------------------------------
6063 // Comparison Op - This is the operation of the comparison, and is limited to
6064 // the following set of codes:
6065 // L (<), LE (<=), G (>), GE (>=), E (==), NE (!=)
6066 //
6067 // Other attributes of the comparison, such as unsignedness, are specified
6068 // by the comparison instruction that sets a condition code flags register.
6069 // That result is represented by a flags operand whose subtype is appropriate
6070 // to the unsignedness (etc.) of the comparison.
6071 //
6072 // Later, the instruction which matches both the Comparison Op (a Bool) and
6073 // the flags (produced by the Cmp) specifies the coding of the comparison op
6074 // by matching a specific subtype of Bool operand below, such as cmpOpU.
6075
6076 // used for signed integral comparisons and fp comparisons
6077
6078 operand cmpOp()
6079 %{
6080 match(Bool);
6081
6082 format %{ "" %}
6083 interface(COND_INTER) %{
6084 equal(0x0, "eq");
6085 not_equal(0x1, "ne");
6086 less(0xb, "lt");
6087 greater_equal(0xa, "ge");
6088 less_equal(0xd, "le");
6089 greater(0xc, "gt");
6090 overflow(0x6, "vs");
6091 no_overflow(0x7, "vc");
6092 %}
6093 %}
6094
6095 // used for unsigned integral comparisons
6096
6097 operand cmpOpU()
6098 %{
6099 match(Bool);
6100
6101 format %{ "" %}
6102 interface(COND_INTER) %{
6103 equal(0x0, "eq");
6104 not_equal(0x1, "ne");
6105 less(0x3, "lo");
6106 greater_equal(0x2, "hs");
6107 less_equal(0x9, "ls");
6108 greater(0x8, "hi");
6109 overflow(0x6, "vs");
6110 no_overflow(0x7, "vc");
6111 %}
6112 %}
6113
6114 // used for certain integral comparisons which can be
6115 // converted to cbxx or tbxx instructions
6116
6117 operand cmpOpEqNe()
6118 %{
6119 match(Bool);
6120 op_cost(0);
6121 predicate(n->as_Bool()->_test._test == BoolTest::ne
6122 || n->as_Bool()->_test._test == BoolTest::eq);
6123
6124 format %{ "" %}
6125 interface(COND_INTER) %{
6126 equal(0x0, "eq");
6127 not_equal(0x1, "ne");
6128 less(0xb, "lt");
6129 greater_equal(0xa, "ge");
6130 less_equal(0xd, "le");
6131 greater(0xc, "gt");
6132 overflow(0x6, "vs");
6133 no_overflow(0x7, "vc");
6134 %}
6135 %}
6136
6137 // used for certain integral comparisons which can be
6138 // converted to cbxx or tbxx instructions
6139
6140 operand cmpOpLtGe()
6141 %{
6142 match(Bool);
6143 op_cost(0);
6144
6145 predicate(n->as_Bool()->_test._test == BoolTest::lt
6146 || n->as_Bool()->_test._test == BoolTest::ge);
6147
6148 format %{ "" %}
6149 interface(COND_INTER) %{
6150 equal(0x0, "eq");
6151 not_equal(0x1, "ne");
6152 less(0xb, "lt");
6153 greater_equal(0xa, "ge");
6154 less_equal(0xd, "le");
6155 greater(0xc, "gt");
6156 overflow(0x6, "vs");
6157 no_overflow(0x7, "vc");
6158 %}
6159 %}
6160
6161 // used for certain unsigned integral comparisons which can be
6162 // converted to cbxx or tbxx instructions
6163
6164 operand cmpOpUEqNeLtGe()
6165 %{
6166 match(Bool);
6167 op_cost(0);
6168
6169 predicate(n->as_Bool()->_test._test == BoolTest::eq
6170 || n->as_Bool()->_test._test == BoolTest::ne
6171 || n->as_Bool()->_test._test == BoolTest::lt
6172 || n->as_Bool()->_test._test == BoolTest::ge);
6173
6174 format %{ "" %}
6175 interface(COND_INTER) %{
6176 equal(0x0, "eq");
6177 not_equal(0x1, "ne");
6178 less(0xb, "lt");
6179 greater_equal(0xa, "ge");
6180 less_equal(0xd, "le");
6181 greater(0xc, "gt");
6182 overflow(0x6, "vs");
6183 no_overflow(0x7, "vc");
6184 %}
6185 %}
6186
6187 // Special operand allowing long args to int ops to be truncated for free
6188
6189 operand iRegL2I(iRegL reg) %{
6190
6191 op_cost(0);
6192
6193 match(ConvL2I reg);
6194
6195 format %{ "l2i($reg)" %}
6196
6197 interface(REG_INTER)
6198 %}
6199
6200 opclass vmem2(indirect, indIndex, indOffI2, indOffL2);
6201 opclass vmem4(indirect, indIndex, indOffI4, indOffL4);
6202 opclass vmem8(indirect, indIndex, indOffI8, indOffL8);
6203 opclass vmem16(indirect, indIndex, indOffI16, indOffL16);
6204
6205 //----------OPERAND CLASSES----------------------------------------------------
6206 // Operand Classes are groups of operands that are used as to simplify
6207 // instruction definitions by not requiring the AD writer to specify
6208 // separate instructions for every form of operand when the
6209 // instruction accepts multiple operand types with the same basic
6210 // encoding and format. The classic case of this is memory operands.
6211
6212 // memory is used to define read/write location for load/store
6213 // instruction defs. we can turn a memory op into an Address
6214
6215 opclass memory1(indirect, indIndexScaled, indIndexScaledI2L, indIndexI2L, indIndex, indOffI1, indOffL1,
6216 indirectN, indIndexScaledN, indIndexScaledI2LN, indIndexI2LN, indIndexN);
6217
6218 opclass memory2(indirect, indIndexScaled, indIndexScaledI2L, indIndexI2L, indIndex, indOffI2, indOffL2,
6219 indirectN, indIndexScaledN, indIndexScaledI2LN, indIndexI2LN, indIndexN);
6220
6221 opclass memory4(indirect, indIndexScaled, indIndexScaledI2L, indIndexI2L, indIndex, indOffI4, indOffL4,
6222 indirectN, indIndexScaledN, indIndexScaledI2LN, indIndexI2LN, indIndexN, indOffIN, indOffLN);
6223
6224 opclass memory8(indirect, indIndexScaled, indIndexScaledI2L, indIndexI2L, indIndex, indOffI8, indOffL8,
6225 indirectN, indIndexScaledN, indIndexScaledI2LN, indIndexI2LN, indIndexN, indOffIN, indOffLN);
6226
6227 // All of the memory operands. For the pipeline description.
6228 opclass memory(indirect, indIndexScaled, indIndexScaledI2L, indIndexI2L, indIndex,
6229 indOffI1, indOffL1, indOffI2, indOffL2, indOffI4, indOffL4, indOffI8, indOffL8,
6230 indirectN, indIndexScaledN, indIndexScaledI2LN, indIndexI2LN, indIndexN, indOffIN, indOffLN);
6231
6232
6233 // iRegIorL2I is used for src inputs in rules for 32 bit int (I)
6234 // operations. it allows the src to be either an iRegI or a (ConvL2I
6235 // iRegL). in the latter case the l2i normally planted for a ConvL2I
6236 // can be elided because the 32-bit instruction will just employ the
6237 // lower 32 bits anyway.
6238 //
6239 // n.b. this does not elide all L2I conversions. if the truncated
6240 // value is consumed by more than one operation then the ConvL2I
6241 // cannot be bundled into the consuming nodes so an l2i gets planted
6242 // (actually a movw $dst $src) and the downstream instructions consume
6243 // the result of the l2i as an iRegI input. That's a shame since the
6244 // movw is actually redundant but its not too costly.
6245
6246 opclass iRegIorL2I(iRegI, iRegL2I);
6247
6248 //----------PIPELINE-----------------------------------------------------------
6249 // Rules which define the behavior of the target architectures pipeline.
6250
6251 // For specific pipelines, eg A53, define the stages of that pipeline
6252 //pipe_desc(ISS, EX1, EX2, WR);
6253 #define ISS S0
6254 #define EX1 S1
6255 #define EX2 S2
6256 #define WR S3
6257
6258 // Integer ALU reg operation
6259 pipeline %{
6260
6261 attributes %{
6262 // ARM instructions are of fixed length
6263 fixed_size_instructions; // Fixed size instructions TODO does
6264 max_instructions_per_bundle = 4; // A53 = 2, A57 = 4
6265 // ARM instructions come in 32-bit word units
6266 instruction_unit_size = 4; // An instruction is 4 bytes long
6267 instruction_fetch_unit_size = 64; // The processor fetches one line
6268 instruction_fetch_units = 1; // of 64 bytes
6269
6270 // List of nop instructions
6271 nops( MachNop );
6272 %}
6273
6274 // We don't use an actual pipeline model so don't care about resources
6275 // or description. we do use pipeline classes to introduce fixed
6276 // latencies
6277
6278 //----------RESOURCES----------------------------------------------------------
6279 // Resources are the functional units available to the machine
6280
6281 resources( INS0, INS1, INS01 = INS0 | INS1,
6282 ALU0, ALU1, ALU = ALU0 | ALU1,
6283 MAC,
6284 DIV,
6285 BRANCH,
6286 LDST,
6287 NEON_FP);
6288
6289 //----------PIPELINE DESCRIPTION-----------------------------------------------
6290 // Pipeline Description specifies the stages in the machine's pipeline
6291
6292 // Define the pipeline as a generic 6 stage pipeline
6293 pipe_desc(S0, S1, S2, S3, S4, S5);
6294
6295 //----------PIPELINE CLASSES---------------------------------------------------
6296 // Pipeline Classes describe the stages in which input and output are
6297 // referenced by the hardware pipeline.
6298
6299 pipe_class fp_dop_reg_reg_s(vRegF dst, vRegF src1, vRegF src2)
6300 %{
6301 single_instruction;
6302 src1 : S1(read);
6303 src2 : S2(read);
6304 dst : S5(write);
6305 INS01 : ISS;
6306 NEON_FP : S5;
6307 %}
6308
6309 pipe_class fp_dop_reg_reg_d(vRegD dst, vRegD src1, vRegD src2)
6310 %{
6311 single_instruction;
6312 src1 : S1(read);
6313 src2 : S2(read);
6314 dst : S5(write);
6315 INS01 : ISS;
6316 NEON_FP : S5;
6317 %}
6318
6319 pipe_class fp_uop_s(vRegF dst, vRegF src)
6320 %{
6321 single_instruction;
6322 src : S1(read);
6323 dst : S5(write);
6324 INS01 : ISS;
6325 NEON_FP : S5;
6326 %}
6327
6328 pipe_class fp_uop_d(vRegD dst, vRegD src)
6329 %{
6330 single_instruction;
6331 src : S1(read);
6332 dst : S5(write);
6333 INS01 : ISS;
6334 NEON_FP : S5;
6335 %}
6336
6337 pipe_class fp_d2f(vRegF dst, vRegD src)
6338 %{
6339 single_instruction;
6340 src : S1(read);
6341 dst : S5(write);
6342 INS01 : ISS;
6343 NEON_FP : S5;
6344 %}
6345
6346 pipe_class fp_f2d(vRegD dst, vRegF src)
6347 %{
6348 single_instruction;
6349 src : S1(read);
6350 dst : S5(write);
6351 INS01 : ISS;
6352 NEON_FP : S5;
6353 %}
6354
6355 pipe_class fp_f2i(iRegINoSp dst, vRegF src)
6356 %{
6357 single_instruction;
6358 src : S1(read);
6359 dst : S5(write);
6360 INS01 : ISS;
6361 NEON_FP : S5;
6362 %}
6363
6364 pipe_class fp_f2l(iRegLNoSp dst, vRegF src)
6365 %{
6366 single_instruction;
6367 src : S1(read);
6368 dst : S5(write);
6369 INS01 : ISS;
6370 NEON_FP : S5;
6371 %}
6372
6373 pipe_class fp_i2f(vRegF dst, iRegIorL2I src)
6374 %{
6375 single_instruction;
6376 src : S1(read);
6377 dst : S5(write);
6378 INS01 : ISS;
6379 NEON_FP : S5;
6380 %}
6381
6382 pipe_class fp_l2f(vRegF dst, iRegL src)
6383 %{
6384 single_instruction;
6385 src : S1(read);
6386 dst : S5(write);
6387 INS01 : ISS;
6388 NEON_FP : S5;
6389 %}
6390
6391 pipe_class fp_d2i(iRegINoSp dst, vRegD src)
6392 %{
6393 single_instruction;
6394 src : S1(read);
6395 dst : S5(write);
6396 INS01 : ISS;
6397 NEON_FP : S5;
6398 %}
6399
6400 pipe_class fp_d2l(iRegLNoSp dst, vRegD src)
6401 %{
6402 single_instruction;
6403 src : S1(read);
6404 dst : S5(write);
6405 INS01 : ISS;
6406 NEON_FP : S5;
6407 %}
6408
6409 pipe_class fp_i2d(vRegD dst, iRegIorL2I src)
6410 %{
6411 single_instruction;
6412 src : S1(read);
6413 dst : S5(write);
6414 INS01 : ISS;
6415 NEON_FP : S5;
6416 %}
6417
6418 pipe_class fp_l2d(vRegD dst, iRegIorL2I src)
6419 %{
6420 single_instruction;
6421 src : S1(read);
6422 dst : S5(write);
6423 INS01 : ISS;
6424 NEON_FP : S5;
6425 %}
6426
6427 pipe_class fp_div_s(vRegF dst, vRegF src1, vRegF src2)
6428 %{
6429 single_instruction;
6430 src1 : S1(read);
6431 src2 : S2(read);
6432 dst : S5(write);
6433 INS0 : ISS;
6434 NEON_FP : S5;
6435 %}
6436
6437 pipe_class fp_div_d(vRegD dst, vRegD src1, vRegD src2)
6438 %{
6439 single_instruction;
6440 src1 : S1(read);
6441 src2 : S2(read);
6442 dst : S5(write);
6443 INS0 : ISS;
6444 NEON_FP : S5;
6445 %}
6446
6447 pipe_class fp_cond_reg_reg_s(vRegF dst, vRegF src1, vRegF src2, rFlagsReg cr)
6448 %{
6449 single_instruction;
6450 cr : S1(read);
6451 src1 : S1(read);
6452 src2 : S1(read);
6453 dst : S3(write);
6454 INS01 : ISS;
6455 NEON_FP : S3;
6456 %}
6457
6458 pipe_class fp_cond_reg_reg_d(vRegD dst, vRegD src1, vRegD src2, rFlagsReg cr)
6459 %{
6460 single_instruction;
6461 cr : S1(read);
6462 src1 : S1(read);
6463 src2 : S1(read);
6464 dst : S3(write);
6465 INS01 : ISS;
6466 NEON_FP : S3;
6467 %}
6468
6469 pipe_class fp_imm_s(vRegF dst)
6470 %{
6471 single_instruction;
6472 dst : S3(write);
6473 INS01 : ISS;
6474 NEON_FP : S3;
6475 %}
6476
6477 pipe_class fp_imm_d(vRegD dst)
6478 %{
6479 single_instruction;
6480 dst : S3(write);
6481 INS01 : ISS;
6482 NEON_FP : S3;
6483 %}
6484
6485 pipe_class fp_load_constant_s(vRegF dst)
6486 %{
6487 single_instruction;
6488 dst : S4(write);
6489 INS01 : ISS;
6490 NEON_FP : S4;
6491 %}
6492
6493 pipe_class fp_load_constant_d(vRegD dst)
6494 %{
6495 single_instruction;
6496 dst : S4(write);
6497 INS01 : ISS;
6498 NEON_FP : S4;
6499 %}
6500
6501 pipe_class vmul64(vecD dst, vecD src1, vecD src2)
6502 %{
6503 single_instruction;
6504 dst : S5(write);
6505 src1 : S1(read);
6506 src2 : S1(read);
6507 INS01 : ISS;
6508 NEON_FP : S5;
6509 %}
6510
6511 pipe_class vmul128(vecX dst, vecX src1, vecX src2)
6512 %{
6513 single_instruction;
6514 dst : S5(write);
6515 src1 : S1(read);
6516 src2 : S1(read);
6517 INS0 : ISS;
6518 NEON_FP : S5;
6519 %}
6520
6521 pipe_class vmla64(vecD dst, vecD src1, vecD src2)
6522 %{
6523 single_instruction;
6524 dst : S5(write);
6525 src1 : S1(read);
6526 src2 : S1(read);
6527 dst : S1(read);
6528 INS01 : ISS;
6529 NEON_FP : S5;
6530 %}
6531
6532 pipe_class vmla128(vecX dst, vecX src1, vecX src2)
6533 %{
6534 single_instruction;
6535 dst : S5(write);
6536 src1 : S1(read);
6537 src2 : S1(read);
6538 dst : S1(read);
6539 INS0 : ISS;
6540 NEON_FP : S5;
6541 %}
6542
6543 pipe_class vdop64(vecD dst, vecD src1, vecD src2)
6544 %{
6545 single_instruction;
6546 dst : S4(write);
6547 src1 : S2(read);
6548 src2 : S2(read);
6549 INS01 : ISS;
6550 NEON_FP : S4;
6551 %}
6552
6553 pipe_class vdop128(vecX dst, vecX src1, vecX src2)
6554 %{
6555 single_instruction;
6556 dst : S4(write);
6557 src1 : S2(read);
6558 src2 : S2(read);
6559 INS0 : ISS;
6560 NEON_FP : S4;
6561 %}
6562
6563 pipe_class vlogical64(vecD dst, vecD src1, vecD src2)
6564 %{
6565 single_instruction;
6566 dst : S3(write);
6567 src1 : S2(read);
6568 src2 : S2(read);
6569 INS01 : ISS;
6570 NEON_FP : S3;
6571 %}
6572
6573 pipe_class vlogical128(vecX dst, vecX src1, vecX src2)
6574 %{
6575 single_instruction;
6576 dst : S3(write);
6577 src1 : S2(read);
6578 src2 : S2(read);
6579 INS0 : ISS;
6580 NEON_FP : S3;
6581 %}
6582
6583 pipe_class vshift64(vecD dst, vecD src, vecX shift)
6584 %{
6585 single_instruction;
6586 dst : S3(write);
6587 src : S1(read);
6588 shift : S1(read);
6589 INS01 : ISS;
6590 NEON_FP : S3;
6591 %}
6592
6593 pipe_class vshift128(vecX dst, vecX src, vecX shift)
6594 %{
6595 single_instruction;
6596 dst : S3(write);
6597 src : S1(read);
6598 shift : S1(read);
6599 INS0 : ISS;
6600 NEON_FP : S3;
6601 %}
6602
6603 pipe_class vshift64_imm(vecD dst, vecD src, immI shift)
6604 %{
6605 single_instruction;
6606 dst : S3(write);
6607 src : S1(read);
6608 INS01 : ISS;
6609 NEON_FP : S3;
6610 %}
6611
6612 pipe_class vshift128_imm(vecX dst, vecX src, immI shift)
6613 %{
6614 single_instruction;
6615 dst : S3(write);
6616 src : S1(read);
6617 INS0 : ISS;
6618 NEON_FP : S3;
6619 %}
6620
6621 pipe_class vdop_fp64(vecD dst, vecD src1, vecD src2)
6622 %{
6623 single_instruction;
6624 dst : S5(write);
6625 src1 : S1(read);
6626 src2 : S1(read);
6627 INS01 : ISS;
6628 NEON_FP : S5;
6629 %}
6630
6631 pipe_class vdop_fp128(vecX dst, vecX src1, vecX src2)
6632 %{
6633 single_instruction;
6634 dst : S5(write);
6635 src1 : S1(read);
6636 src2 : S1(read);
6637 INS0 : ISS;
6638 NEON_FP : S5;
6639 %}
6640
6641 pipe_class vmuldiv_fp64(vecD dst, vecD src1, vecD src2)
6642 %{
6643 single_instruction;
6644 dst : S5(write);
6645 src1 : S1(read);
6646 src2 : S1(read);
6647 INS0 : ISS;
6648 NEON_FP : S5;
6649 %}
6650
6651 pipe_class vmuldiv_fp128(vecX dst, vecX src1, vecX src2)
6652 %{
6653 single_instruction;
6654 dst : S5(write);
6655 src1 : S1(read);
6656 src2 : S1(read);
6657 INS0 : ISS;
6658 NEON_FP : S5;
6659 %}
6660
6661 pipe_class vsqrt_fp128(vecX dst, vecX src)
6662 %{
6663 single_instruction;
6664 dst : S5(write);
6665 src : S1(read);
6666 INS0 : ISS;
6667 NEON_FP : S5;
6668 %}
6669
6670 pipe_class vunop_fp64(vecD dst, vecD src)
6671 %{
6672 single_instruction;
6673 dst : S5(write);
6674 src : S1(read);
6675 INS01 : ISS;
6676 NEON_FP : S5;
6677 %}
6678
6679 pipe_class vunop_fp128(vecX dst, vecX src)
6680 %{
6681 single_instruction;
6682 dst : S5(write);
6683 src : S1(read);
6684 INS0 : ISS;
6685 NEON_FP : S5;
6686 %}
6687
6688 pipe_class vdup_reg_reg64(vecD dst, iRegI src)
6689 %{
6690 single_instruction;
6691 dst : S3(write);
6692 src : S1(read);
6693 INS01 : ISS;
6694 NEON_FP : S3;
6695 %}
6696
6697 pipe_class vdup_reg_reg128(vecX dst, iRegI src)
6698 %{
6699 single_instruction;
6700 dst : S3(write);
6701 src : S1(read);
6702 INS01 : ISS;
6703 NEON_FP : S3;
6704 %}
6705
6706 pipe_class vdup_reg_freg64(vecD dst, vRegF src)
6707 %{
6708 single_instruction;
6709 dst : S3(write);
6710 src : S1(read);
6711 INS01 : ISS;
6712 NEON_FP : S3;
6713 %}
6714
6715 pipe_class vdup_reg_freg128(vecX dst, vRegF src)
6716 %{
6717 single_instruction;
6718 dst : S3(write);
6719 src : S1(read);
6720 INS01 : ISS;
6721 NEON_FP : S3;
6722 %}
6723
6724 pipe_class vdup_reg_dreg128(vecX dst, vRegD src)
6725 %{
6726 single_instruction;
6727 dst : S3(write);
6728 src : S1(read);
6729 INS01 : ISS;
6730 NEON_FP : S3;
6731 %}
6732
6733 pipe_class vmovi_reg_imm64(vecD dst)
6734 %{
6735 single_instruction;
6736 dst : S3(write);
6737 INS01 : ISS;
6738 NEON_FP : S3;
6739 %}
6740
6741 pipe_class vmovi_reg_imm128(vecX dst)
6742 %{
6743 single_instruction;
6744 dst : S3(write);
6745 INS0 : ISS;
6746 NEON_FP : S3;
6747 %}
6748
6749 pipe_class vload_reg_mem64(vecD dst, vmem8 mem)
6750 %{
6751 single_instruction;
6752 dst : S5(write);
6753 mem : ISS(read);
6754 INS01 : ISS;
6755 NEON_FP : S3;
6756 %}
6757
6758 pipe_class vload_reg_mem128(vecX dst, vmem16 mem)
6759 %{
6760 single_instruction;
6761 dst : S5(write);
6762 mem : ISS(read);
6763 INS01 : ISS;
6764 NEON_FP : S3;
6765 %}
6766
6767 pipe_class vstore_reg_mem64(vecD src, vmem8 mem)
6768 %{
6769 single_instruction;
6770 mem : ISS(read);
6771 src : S2(read);
6772 INS01 : ISS;
6773 NEON_FP : S3;
6774 %}
6775
6776 pipe_class vstore_reg_mem128(vecD src, vmem16 mem)
6777 %{
6778 single_instruction;
6779 mem : ISS(read);
6780 src : S2(read);
6781 INS01 : ISS;
6782 NEON_FP : S3;
6783 %}
6784
6785 //------- Integer ALU operations --------------------------
6786
6787 // Integer ALU reg-reg operation
6788 // Operands needed in EX1, result generated in EX2
6789 // Eg. ADD x0, x1, x2
6790 pipe_class ialu_reg_reg(iRegI dst, iRegI src1, iRegI src2)
6791 %{
6792 single_instruction;
6793 dst : EX2(write);
6794 src1 : EX1(read);
6795 src2 : EX1(read);
6796 INS01 : ISS; // Dual issue as instruction 0 or 1
6797 ALU : EX2;
6798 %}
6799
6800 // Integer ALU reg-reg operation with constant shift
6801 // Shifted register must be available in LATE_ISS instead of EX1
6802 // Eg. ADD x0, x1, x2, LSL #2
6803 pipe_class ialu_reg_reg_shift(iRegI dst, iRegI src1, iRegI src2, immI shift)
6804 %{
6805 single_instruction;
6806 dst : EX2(write);
6807 src1 : EX1(read);
6808 src2 : ISS(read);
6809 INS01 : ISS;
6810 ALU : EX2;
6811 %}
6812
6813 // Integer ALU reg operation with constant shift
6814 // Eg. LSL x0, x1, #shift
6815 pipe_class ialu_reg_shift(iRegI dst, iRegI src1)
6816 %{
6817 single_instruction;
6818 dst : EX2(write);
6819 src1 : ISS(read);
6820 INS01 : ISS;
6821 ALU : EX2;
6822 %}
6823
6824 // Integer ALU reg-reg operation with variable shift
6825 // Both operands must be available in LATE_ISS instead of EX1
6826 // Result is available in EX1 instead of EX2
6827 // Eg. LSLV x0, x1, x2
6828 pipe_class ialu_reg_reg_vshift(iRegI dst, iRegI src1, iRegI src2)
6829 %{
6830 single_instruction;
6831 dst : EX1(write);
6832 src1 : ISS(read);
6833 src2 : ISS(read);
6834 INS01 : ISS;
6835 ALU : EX1;
6836 %}
6837
6838 // Integer ALU reg-reg operation with extract
6839 // As for _vshift above, but result generated in EX2
6840 // Eg. EXTR x0, x1, x2, #N
6841 pipe_class ialu_reg_reg_extr(iRegI dst, iRegI src1, iRegI src2)
6842 %{
6843 single_instruction;
6844 dst : EX2(write);
6845 src1 : ISS(read);
6846 src2 : ISS(read);
6847 INS1 : ISS; // Can only dual issue as Instruction 1
6848 ALU : EX1;
6849 %}
6850
6851 // Integer ALU reg operation
6852 // Eg. NEG x0, x1
6853 pipe_class ialu_reg(iRegI dst, iRegI src)
6854 %{
6855 single_instruction;
6856 dst : EX2(write);
6857 src : EX1(read);
6858 INS01 : ISS;
6859 ALU : EX2;
6860 %}
6861
6862 // Integer ALU reg mmediate operation
6863 // Eg. ADD x0, x1, #N
6864 pipe_class ialu_reg_imm(iRegI dst, iRegI src1)
6865 %{
6866 single_instruction;
6867 dst : EX2(write);
6868 src1 : EX1(read);
6869 INS01 : ISS;
6870 ALU : EX2;
6871 %}
6872
6873 // Integer ALU immediate operation (no source operands)
6874 // Eg. MOV x0, #N
6875 pipe_class ialu_imm(iRegI dst)
6876 %{
6877 single_instruction;
6878 dst : EX1(write);
6879 INS01 : ISS;
6880 ALU : EX1;
6881 %}
6882
6883 //------- Compare operation -------------------------------
6884
6885 // Compare reg-reg
6886 // Eg. CMP x0, x1
6887 pipe_class icmp_reg_reg(rFlagsReg cr, iRegI op1, iRegI op2)
6888 %{
6889 single_instruction;
6890 // fixed_latency(16);
6891 cr : EX2(write);
6892 op1 : EX1(read);
6893 op2 : EX1(read);
6894 INS01 : ISS;
6895 ALU : EX2;
6896 %}
6897
6898 // Compare reg-reg
6899 // Eg. CMP x0, #N
6900 pipe_class icmp_reg_imm(rFlagsReg cr, iRegI op1)
6901 %{
6902 single_instruction;
6903 // fixed_latency(16);
6904 cr : EX2(write);
6905 op1 : EX1(read);
6906 INS01 : ISS;
6907 ALU : EX2;
6908 %}
6909
6910 //------- Conditional instructions ------------------------
6911
6912 // Conditional no operands
6913 // Eg. CSINC x0, zr, zr, <cond>
6914 pipe_class icond_none(iRegI dst, rFlagsReg cr)
6915 %{
6916 single_instruction;
6917 cr : EX1(read);
6918 dst : EX2(write);
6919 INS01 : ISS;
6920 ALU : EX2;
6921 %}
6922
6923 // Conditional 2 operand
6924 // EG. CSEL X0, X1, X2, <cond>
6925 pipe_class icond_reg_reg(iRegI dst, iRegI src1, iRegI src2, rFlagsReg cr)
6926 %{
6927 single_instruction;
6928 cr : EX1(read);
6929 src1 : EX1(read);
6930 src2 : EX1(read);
6931 dst : EX2(write);
6932 INS01 : ISS;
6933 ALU : EX2;
6934 %}
6935
6936 // Conditional 2 operand
6937 // EG. CSEL X0, X1, X2, <cond>
6938 pipe_class icond_reg(iRegI dst, iRegI src, rFlagsReg cr)
6939 %{
6940 single_instruction;
6941 cr : EX1(read);
6942 src : EX1(read);
6943 dst : EX2(write);
6944 INS01 : ISS;
6945 ALU : EX2;
6946 %}
6947
6948 //------- Multiply pipeline operations --------------------
6949
6950 // Multiply reg-reg
6951 // Eg. MUL w0, w1, w2
6952 pipe_class imul_reg_reg(iRegI dst, iRegI src1, iRegI src2)
6953 %{
6954 single_instruction;
6955 dst : WR(write);
6956 src1 : ISS(read);
6957 src2 : ISS(read);
6958 INS01 : ISS;
6959 MAC : WR;
6960 %}
6961
6962 // Multiply accumulate
6963 // Eg. MADD w0, w1, w2, w3
6964 pipe_class imac_reg_reg(iRegI dst, iRegI src1, iRegI src2, iRegI src3)
6965 %{
6966 single_instruction;
6967 dst : WR(write);
6968 src1 : ISS(read);
6969 src2 : ISS(read);
6970 src3 : ISS(read);
6971 INS01 : ISS;
6972 MAC : WR;
6973 %}
6974
6975 // Eg. MUL w0, w1, w2
6976 pipe_class lmul_reg_reg(iRegI dst, iRegI src1, iRegI src2)
6977 %{
6978 single_instruction;
6979 fixed_latency(3); // Maximum latency for 64 bit mul
6980 dst : WR(write);
6981 src1 : ISS(read);
6982 src2 : ISS(read);
6983 INS01 : ISS;
6984 MAC : WR;
6985 %}
6986
6987 // Multiply accumulate
6988 // Eg. MADD w0, w1, w2, w3
6989 pipe_class lmac_reg_reg(iRegI dst, iRegI src1, iRegI src2, iRegI src3)
6990 %{
6991 single_instruction;
6992 fixed_latency(3); // Maximum latency for 64 bit mul
6993 dst : WR(write);
6994 src1 : ISS(read);
6995 src2 : ISS(read);
6996 src3 : ISS(read);
6997 INS01 : ISS;
6998 MAC : WR;
6999 %}
7000
7001 //------- Divide pipeline operations --------------------
7002
7003 // Eg. SDIV w0, w1, w2
7004 pipe_class idiv_reg_reg(iRegI dst, iRegI src1, iRegI src2)
7005 %{
7006 single_instruction;
7007 fixed_latency(8); // Maximum latency for 32 bit divide
7008 dst : WR(write);
7009 src1 : ISS(read);
7010 src2 : ISS(read);
7011 INS0 : ISS; // Can only dual issue as instruction 0
7012 DIV : WR;
7013 %}
7014
7015 // Eg. SDIV x0, x1, x2
7016 pipe_class ldiv_reg_reg(iRegI dst, iRegI src1, iRegI src2)
7017 %{
7018 single_instruction;
7019 fixed_latency(16); // Maximum latency for 64 bit divide
7020 dst : WR(write);
7021 src1 : ISS(read);
7022 src2 : ISS(read);
7023 INS0 : ISS; // Can only dual issue as instruction 0
7024 DIV : WR;
7025 %}
7026
7027 //------- Load pipeline operations ------------------------
7028
7029 // Load - prefetch
7030 // Eg. PFRM <mem>
7031 pipe_class iload_prefetch(memory mem)
7032 %{
7033 single_instruction;
7034 mem : ISS(read);
7035 INS01 : ISS;
7036 LDST : WR;
7037 %}
7038
7039 // Load - reg, mem
7040 // Eg. LDR x0, <mem>
7041 pipe_class iload_reg_mem(iRegI dst, memory mem)
7042 %{
7043 single_instruction;
7044 dst : WR(write);
7045 mem : ISS(read);
7046 INS01 : ISS;
7047 LDST : WR;
7048 %}
7049
7050 // Load - reg, reg
7051 // Eg. LDR x0, [sp, x1]
7052 pipe_class iload_reg_reg(iRegI dst, iRegI src)
7053 %{
7054 single_instruction;
7055 dst : WR(write);
7056 src : ISS(read);
7057 INS01 : ISS;
7058 LDST : WR;
7059 %}
7060
7061 //------- Store pipeline operations -----------------------
7062
7063 // Store - zr, mem
7064 // Eg. STR zr, <mem>
7065 pipe_class istore_mem(memory mem)
7066 %{
7067 single_instruction;
7068 mem : ISS(read);
7069 INS01 : ISS;
7070 LDST : WR;
7071 %}
7072
7073 // Store - reg, mem
7074 // Eg. STR x0, <mem>
7075 pipe_class istore_reg_mem(iRegI src, memory mem)
7076 %{
7077 single_instruction;
7078 mem : ISS(read);
7079 src : EX2(read);
7080 INS01 : ISS;
7081 LDST : WR;
7082 %}
7083
7084 // Store - reg, reg
7085 // Eg. STR x0, [sp, x1]
7086 pipe_class istore_reg_reg(iRegI dst, iRegI src)
7087 %{
7088 single_instruction;
7089 dst : ISS(read);
7090 src : EX2(read);
7091 INS01 : ISS;
7092 LDST : WR;
7093 %}
7094
7095 //------- Store pipeline operations -----------------------
7096
7097 // Branch
7098 pipe_class pipe_branch()
7099 %{
7100 single_instruction;
7101 INS01 : ISS;
7102 BRANCH : EX1;
7103 %}
7104
7105 // Conditional branch
7106 pipe_class pipe_branch_cond(rFlagsReg cr)
7107 %{
7108 single_instruction;
7109 cr : EX1(read);
7110 INS01 : ISS;
7111 BRANCH : EX1;
7112 %}
7113
7114 // Compare & Branch
7115 // EG. CBZ/CBNZ
7116 pipe_class pipe_cmp_branch(iRegI op1)
7117 %{
7118 single_instruction;
7119 op1 : EX1(read);
7120 INS01 : ISS;
7121 BRANCH : EX1;
7122 %}
7123
7124 //------- Synchronisation operations ----------------------
7125
7126 // Any operation requiring serialization.
7127 // EG. DMB/Atomic Ops/Load Acquire/Str Release
7128 pipe_class pipe_serial()
7129 %{
7130 single_instruction;
7131 force_serialization;
7132 fixed_latency(16);
7133 INS01 : ISS(2); // Cannot dual issue with any other instruction
7134 LDST : WR;
7135 %}
7136
7137 // Generic big/slow expanded idiom - also serialized
7138 pipe_class pipe_slow()
7139 %{
7140 instruction_count(10);
7141 multiple_bundles;
7142 force_serialization;
7143 fixed_latency(16);
7144 INS01 : ISS(2); // Cannot dual issue with any other instruction
7145 LDST : WR;
7146 %}
7147
7148 // Empty pipeline class
7149 pipe_class pipe_class_empty()
7150 %{
7151 single_instruction;
7152 fixed_latency(0);
7153 %}
7154
7155 // Default pipeline class.
7156 pipe_class pipe_class_default()
7157 %{
7158 single_instruction;
7159 fixed_latency(2);
7160 %}
7161
7162 // Pipeline class for compares.
7163 pipe_class pipe_class_compare()
7164 %{
7165 single_instruction;
7166 fixed_latency(16);
7167 %}
7168
7169 // Pipeline class for memory operations.
7170 pipe_class pipe_class_memory()
7171 %{
7172 single_instruction;
7173 fixed_latency(16);
7174 %}
7175
7176 // Pipeline class for call.
7177 pipe_class pipe_class_call()
7178 %{
7179 single_instruction;
7180 fixed_latency(100);
7181 %}
7182
7183 // Define the class for the Nop node.
7184 define %{
7185 MachNop = pipe_class_empty;
7186 %}
7187
7188 %}
7189 //----------INSTRUCTIONS-------------------------------------------------------
7190 //
7191 // match -- States which machine-independent subtree may be replaced
7192 // by this instruction.
7193 // ins_cost -- The estimated cost of this instruction is used by instruction
7194 // selection to identify a minimum cost tree of machine
7195 // instructions that matches a tree of machine-independent
7196 // instructions.
7197 // format -- A string providing the disassembly for this instruction.
7198 // The value of an instruction's operand may be inserted
7199 // by referring to it with a '$' prefix.
7200 // opcode -- Three instruction opcodes may be provided. These are referred
7201 // to within an encode class as $primary, $secondary, and $tertiary
7202 // rrspectively. The primary opcode is commonly used to
7203 // indicate the type of machine instruction, while secondary
7204 // and tertiary are often used for prefix options or addressing
7205 // modes.
7206 // ins_encode -- A list of encode classes with parameters. The encode class
7207 // name must have been defined in an 'enc_class' specification
7208 // in the encode section of the architecture description.
7209
7210 // ============================================================================
7211 // Memory (Load/Store) Instructions
7212
7213 // Load Instructions
7214
7215 // Load Byte (8 bit signed)
7216 instruct loadB(iRegINoSp dst, memory1 mem)
7217 %{
7218 match(Set dst (LoadB mem));
7219 predicate(!needs_acquiring_load(n));
7220
7221 ins_cost(4 * INSN_COST);
7222 format %{ "ldrsbw $dst, $mem\t# byte" %}
7223
7224 ins_encode(aarch64_enc_ldrsbw(dst, mem));
7225
7226 ins_pipe(iload_reg_mem);
7227 %}
7228
7229 // Load Byte (8 bit signed) into long
7230 instruct loadB2L(iRegLNoSp dst, memory1 mem)
7231 %{
7232 match(Set dst (ConvI2L (LoadB mem)));
7233 predicate(!needs_acquiring_load(n->in(1)));
7234
7235 ins_cost(4 * INSN_COST);
7236 format %{ "ldrsb $dst, $mem\t# byte" %}
7237
7238 ins_encode(aarch64_enc_ldrsb(dst, mem));
7239
7240 ins_pipe(iload_reg_mem);
7241 %}
7242
7243 // Load Byte (8 bit unsigned)
7244 instruct loadUB(iRegINoSp dst, memory1 mem)
7245 %{
7246 match(Set dst (LoadUB mem));
7247 predicate(!needs_acquiring_load(n));
7248
7249 ins_cost(4 * INSN_COST);
7250 format %{ "ldrbw $dst, $mem\t# byte" %}
7251
7252 ins_encode(aarch64_enc_ldrb(dst, mem));
7253
7254 ins_pipe(iload_reg_mem);
7255 %}
7256
7257 // Load Byte (8 bit unsigned) into long
7258 instruct loadUB2L(iRegLNoSp dst, memory1 mem)
7259 %{
7260 match(Set dst (ConvI2L (LoadUB mem)));
7261 predicate(!needs_acquiring_load(n->in(1)));
7262
7263 ins_cost(4 * INSN_COST);
7264 format %{ "ldrb $dst, $mem\t# byte" %}
7265
7266 ins_encode(aarch64_enc_ldrb(dst, mem));
7267
7268 ins_pipe(iload_reg_mem);
7269 %}
7270
7271 // Load Short (16 bit signed)
7272 instruct loadS(iRegINoSp dst, memory2 mem)
7273 %{
7274 match(Set dst (LoadS mem));
7275 predicate(!needs_acquiring_load(n));
7276
7277 ins_cost(4 * INSN_COST);
7278 format %{ "ldrshw $dst, $mem\t# short" %}
7279
7280 ins_encode(aarch64_enc_ldrshw(dst, mem));
7281
7282 ins_pipe(iload_reg_mem);
7283 %}
7284
7285 // Load Short (16 bit signed) into long
7286 instruct loadS2L(iRegLNoSp dst, memory2 mem)
7287 %{
7288 match(Set dst (ConvI2L (LoadS mem)));
7289 predicate(!needs_acquiring_load(n->in(1)));
7290
7291 ins_cost(4 * INSN_COST);
7292 format %{ "ldrsh $dst, $mem\t# short" %}
7293
7294 ins_encode(aarch64_enc_ldrsh(dst, mem));
7295
7296 ins_pipe(iload_reg_mem);
7297 %}
7298
7299 // Load Char (16 bit unsigned)
7300 instruct loadUS(iRegINoSp dst, memory2 mem)
7301 %{
7302 match(Set dst (LoadUS mem));
7303 predicate(!needs_acquiring_load(n));
7304
7305 ins_cost(4 * INSN_COST);
7306 format %{ "ldrh $dst, $mem\t# short" %}
7307
7308 ins_encode(aarch64_enc_ldrh(dst, mem));
7309
7310 ins_pipe(iload_reg_mem);
7311 %}
7312
7313 // Load Short/Char (16 bit unsigned) into long
7314 instruct loadUS2L(iRegLNoSp dst, memory2 mem)
7315 %{
7316 match(Set dst (ConvI2L (LoadUS mem)));
7317 predicate(!needs_acquiring_load(n->in(1)));
7318
7319 ins_cost(4 * INSN_COST);
7320 format %{ "ldrh $dst, $mem\t# short" %}
7321
7322 ins_encode(aarch64_enc_ldrh(dst, mem));
7323
7324 ins_pipe(iload_reg_mem);
7325 %}
7326
7327 // Load Integer (32 bit signed)
7328 instruct loadI(iRegINoSp dst, memory4 mem)
7329 %{
7330 match(Set dst (LoadI mem));
7331 predicate(!needs_acquiring_load(n));
7332
7333 ins_cost(4 * INSN_COST);
7334 format %{ "ldrw $dst, $mem\t# int" %}
7335
7336 ins_encode(aarch64_enc_ldrw(dst, mem));
7337
7338 ins_pipe(iload_reg_mem);
7339 %}
7340
7341 // Load Integer (32 bit signed) into long
7342 instruct loadI2L(iRegLNoSp dst, memory4 mem)
7343 %{
7344 match(Set dst (ConvI2L (LoadI mem)));
7345 predicate(!needs_acquiring_load(n->in(1)));
7346
7347 ins_cost(4 * INSN_COST);
7348 format %{ "ldrsw $dst, $mem\t# int" %}
7349
7350 ins_encode(aarch64_enc_ldrsw(dst, mem));
7351
7352 ins_pipe(iload_reg_mem);
7353 %}
7354
7355 // Load Integer (32 bit unsigned) into long
7356 instruct loadUI2L(iRegLNoSp dst, memory4 mem, immL_32bits mask)
7357 %{
7358 match(Set dst (AndL (ConvI2L (LoadI mem)) mask));
7359 predicate(!needs_acquiring_load(n->in(1)->in(1)->as_Load()));
7360
7361 ins_cost(4 * INSN_COST);
7362 format %{ "ldrw $dst, $mem\t# int" %}
7363
7364 ins_encode(aarch64_enc_ldrw(dst, mem));
7365
7366 ins_pipe(iload_reg_mem);
7367 %}
7368
7369 // Load Long (64 bit signed)
7370 instruct loadL(iRegLNoSp dst, memory8 mem)
7371 %{
7372 match(Set dst (LoadL mem));
7373 predicate(!needs_acquiring_load(n));
7374
7375 ins_cost(4 * INSN_COST);
7376 format %{ "ldr $dst, $mem\t# int" %}
7377
7378 ins_encode(aarch64_enc_ldr(dst, mem));
7379
7380 ins_pipe(iload_reg_mem);
7381 %}
7382
7383 // Load Range
7384 instruct loadRange(iRegINoSp dst, memory4 mem)
7385 %{
7386 match(Set dst (LoadRange mem));
7387
7388 ins_cost(4 * INSN_COST);
7389 format %{ "ldrw $dst, $mem\t# range" %}
7390
7391 ins_encode(aarch64_enc_ldrw(dst, mem));
7392
7393 ins_pipe(iload_reg_mem);
7394 %}
7395
7396 // Load Pointer
7397 instruct loadP(iRegPNoSp dst, memory8 mem)
7398 %{
7399 match(Set dst (LoadP mem));
7400 predicate(!needs_acquiring_load(n) && (n->as_Load()->barrier_data() == 0));
7401
7402 ins_cost(4 * INSN_COST);
7403 format %{ "ldr $dst, $mem\t# ptr" %}
7404
7405 ins_encode(aarch64_enc_ldr(dst, mem));
7406
7407 ins_pipe(iload_reg_mem);
7408 %}
7409
7410 // Load Compressed Pointer
7411 instruct loadN(iRegNNoSp dst, memory4 mem)
7412 %{
7413 match(Set dst (LoadN mem));
7414 predicate(!needs_acquiring_load(n));
7415
7416 ins_cost(4 * INSN_COST);
7417 format %{ "ldrw $dst, $mem\t# compressed ptr" %}
7418
7419 ins_encode(aarch64_enc_ldrw(dst, mem));
7420
7421 ins_pipe(iload_reg_mem);
7422 %}
7423
7424 // Load Klass Pointer
7425 instruct loadKlass(iRegPNoSp dst, memory8 mem)
7426 %{
7427 match(Set dst (LoadKlass mem));
7428 predicate(!needs_acquiring_load(n));
7429
7430 ins_cost(4 * INSN_COST);
7431 format %{ "ldr $dst, $mem\t# class" %}
7432
7433 ins_encode(aarch64_enc_ldr(dst, mem));
7434
7435 ins_pipe(iload_reg_mem);
7436 %}
7437
7438 // Load Narrow Klass Pointer
7439 instruct loadNKlass(iRegNNoSp dst, memory4 mem)
7440 %{
7441 match(Set dst (LoadNKlass mem));
7442 predicate(!needs_acquiring_load(n));
7443
7444 ins_cost(4 * INSN_COST);
7445 format %{ "ldrw $dst, $mem\t# compressed class ptr" %}
7446
7447 ins_encode(aarch64_enc_ldrw(dst, mem));
7448
7449 ins_pipe(iload_reg_mem);
7450 %}
7451
7452 // Load Float
7453 instruct loadF(vRegF dst, memory4 mem)
7454 %{
7455 match(Set dst (LoadF mem));
7456 predicate(!needs_acquiring_load(n));
7457
7458 ins_cost(4 * INSN_COST);
7459 format %{ "ldrs $dst, $mem\t# float" %}
7460
7461 ins_encode( aarch64_enc_ldrs(dst, mem) );
7462
7463 ins_pipe(pipe_class_memory);
7464 %}
7465
7466 // Load Double
7467 instruct loadD(vRegD dst, memory8 mem)
7468 %{
7469 match(Set dst (LoadD mem));
7470 predicate(!needs_acquiring_load(n));
7471
7472 ins_cost(4 * INSN_COST);
7473 format %{ "ldrd $dst, $mem\t# double" %}
7474
7475 ins_encode( aarch64_enc_ldrd(dst, mem) );
7476
7477 ins_pipe(pipe_class_memory);
7478 %}
7479
7480
7481 // Load Int Constant
7482 instruct loadConI(iRegINoSp dst, immI src)
7483 %{
7484 match(Set dst src);
7485
7486 ins_cost(INSN_COST);
7487 format %{ "mov $dst, $src\t# int" %}
7488
7489 ins_encode( aarch64_enc_movw_imm(dst, src) );
7490
7491 ins_pipe(ialu_imm);
7492 %}
7493
7494 // Load Long Constant
7495 instruct loadConL(iRegLNoSp dst, immL src)
7496 %{
7497 match(Set dst src);
7498
7499 ins_cost(INSN_COST);
7500 format %{ "mov $dst, $src\t# long" %}
7501
7502 ins_encode( aarch64_enc_mov_imm(dst, src) );
7503
7504 ins_pipe(ialu_imm);
7505 %}
7506
7507 // Load Pointer Constant
7508
7509 instruct loadConP(iRegPNoSp dst, immP con)
7510 %{
7511 match(Set dst con);
7512
7513 ins_cost(INSN_COST * 4);
7514 format %{
7515 "mov $dst, $con\t# ptr\n\t"
7516 %}
7517
7518 ins_encode(aarch64_enc_mov_p(dst, con));
7519
7520 ins_pipe(ialu_imm);
7521 %}
7522
7523 // Load Null Pointer Constant
7524
7525 instruct loadConP0(iRegPNoSp dst, immP0 con)
7526 %{
7527 match(Set dst con);
7528
7529 ins_cost(INSN_COST);
7530 format %{ "mov $dst, $con\t# NULL ptr" %}
7531
7532 ins_encode(aarch64_enc_mov_p0(dst, con));
7533
7534 ins_pipe(ialu_imm);
7535 %}
7536
7537 // Load Pointer Constant One
7538
7539 instruct loadConP1(iRegPNoSp dst, immP_1 con)
7540 %{
7541 match(Set dst con);
7542
7543 ins_cost(INSN_COST);
7544 format %{ "mov $dst, $con\t# NULL ptr" %}
7545
7546 ins_encode(aarch64_enc_mov_p1(dst, con));
7547
7548 ins_pipe(ialu_imm);
7549 %}
7550
7551 // Load Byte Map Base Constant
7552
7553 instruct loadByteMapBase(iRegPNoSp dst, immByteMapBase con)
7554 %{
7555 match(Set dst con);
7556
7557 ins_cost(INSN_COST);
7558 format %{ "adr $dst, $con\t# Byte Map Base" %}
7559
7560 ins_encode(aarch64_enc_mov_byte_map_base(dst, con));
7561
7562 ins_pipe(ialu_imm);
7563 %}
7564
7565 // Load Narrow Pointer Constant
7566
7567 instruct loadConN(iRegNNoSp dst, immN con)
7568 %{
7569 match(Set dst con);
7570
7571 ins_cost(INSN_COST * 4);
7572 format %{ "mov $dst, $con\t# compressed ptr" %}
7573
7574 ins_encode(aarch64_enc_mov_n(dst, con));
7575
7576 ins_pipe(ialu_imm);
7577 %}
7578
7579 // Load Narrow Null Pointer Constant
7580
7581 instruct loadConN0(iRegNNoSp dst, immN0 con)
7582 %{
7583 match(Set dst con);
7584
7585 ins_cost(INSN_COST);
7586 format %{ "mov $dst, $con\t# compressed NULL ptr" %}
7587
7588 ins_encode(aarch64_enc_mov_n0(dst, con));
7589
7590 ins_pipe(ialu_imm);
7591 %}
7592
7593 // Load Narrow Klass Constant
7594
7595 instruct loadConNKlass(iRegNNoSp dst, immNKlass con)
7596 %{
7597 match(Set dst con);
7598
7599 ins_cost(INSN_COST);
7600 format %{ "mov $dst, $con\t# compressed klass ptr" %}
7601
7602 ins_encode(aarch64_enc_mov_nk(dst, con));
7603
7604 ins_pipe(ialu_imm);
7605 %}
7606
7607 // Load Packed Float Constant
7608
7609 instruct loadConF_packed(vRegF dst, immFPacked con) %{
7610 match(Set dst con);
7611 ins_cost(INSN_COST * 4);
7612 format %{ "fmovs $dst, $con"%}
7613 ins_encode %{
7614 __ fmovs(as_FloatRegister($dst$$reg), (double)$con$$constant);
7615 %}
7616
7617 ins_pipe(fp_imm_s);
7618 %}
7619
7620 // Load Float Constant
7621
7622 instruct loadConF(vRegF dst, immF con) %{
7623 match(Set dst con);
7624
7625 ins_cost(INSN_COST * 4);
7626
7627 format %{
7628 "ldrs $dst, [$constantaddress]\t# load from constant table: float=$con\n\t"
7629 %}
7630
7631 ins_encode %{
7632 __ ldrs(as_FloatRegister($dst$$reg), $constantaddress($con));
7633 %}
7634
7635 ins_pipe(fp_load_constant_s);
7636 %}
7637
7638 // Load Packed Double Constant
7639
7640 instruct loadConD_packed(vRegD dst, immDPacked con) %{
7641 match(Set dst con);
7642 ins_cost(INSN_COST);
7643 format %{ "fmovd $dst, $con"%}
7644 ins_encode %{
7645 __ fmovd(as_FloatRegister($dst$$reg), $con$$constant);
7646 %}
7647
7648 ins_pipe(fp_imm_d);
7649 %}
7650
7651 // Load Double Constant
7652
7653 instruct loadConD(vRegD dst, immD con) %{
7654 match(Set dst con);
7655
7656 ins_cost(INSN_COST * 5);
7657 format %{
7658 "ldrd $dst, [$constantaddress]\t# load from constant table: float=$con\n\t"
7659 %}
7660
7661 ins_encode %{
7662 __ ldrd(as_FloatRegister($dst$$reg), $constantaddress($con));
7663 %}
7664
7665 ins_pipe(fp_load_constant_d);
7666 %}
7667
7668 // Store Instructions
7669
7670 // Store CMS card-mark Immediate
7671 instruct storeimmCM0(immI0 zero, memory1 mem)
7672 %{
7673 match(Set mem (StoreCM mem zero));
7674
7675 ins_cost(INSN_COST);
7676 format %{ "storestore (elided)\n\t"
7677 "strb zr, $mem\t# byte" %}
7678
7679 ins_encode(aarch64_enc_strb0(mem));
7680
7681 ins_pipe(istore_mem);
7682 %}
7683
7684 // Store CMS card-mark Immediate with intervening StoreStore
7685 // needed when using CMS with no conditional card marking
7686 instruct storeimmCM0_ordered(immI0 zero, memory1 mem)
7687 %{
7688 match(Set mem (StoreCM mem zero));
7689
7690 ins_cost(INSN_COST * 2);
7691 format %{ "storestore\n\t"
7692 "dmb ishst"
7693 "\n\tstrb zr, $mem\t# byte" %}
7694
7695 ins_encode(aarch64_enc_strb0_ordered(mem));
7696
7697 ins_pipe(istore_mem);
7698 %}
7699
7700 // Store Byte
7701 instruct storeB(iRegIorL2I src, memory1 mem)
7702 %{
7703 match(Set mem (StoreB mem src));
7704 predicate(!needs_releasing_store(n));
7705
7706 ins_cost(INSN_COST);
7707 format %{ "strb $src, $mem\t# byte" %}
7708
7709 ins_encode(aarch64_enc_strb(src, mem));
7710
7711 ins_pipe(istore_reg_mem);
7712 %}
7713
7714
7715 instruct storeimmB0(immI0 zero, memory1 mem)
7716 %{
7717 match(Set mem (StoreB mem zero));
7718 predicate(!needs_releasing_store(n));
7719
7720 ins_cost(INSN_COST);
7721 format %{ "strb rscractch2, $mem\t# byte" %}
7722
7723 ins_encode(aarch64_enc_strb0(mem));
7724
7725 ins_pipe(istore_mem);
7726 %}
7727
7728 // Store Char/Short
7729 instruct storeC(iRegIorL2I src, memory2 mem)
7730 %{
7731 match(Set mem (StoreC mem src));
7732 predicate(!needs_releasing_store(n));
7733
7734 ins_cost(INSN_COST);
7735 format %{ "strh $src, $mem\t# short" %}
7736
7737 ins_encode(aarch64_enc_strh(src, mem));
7738
7739 ins_pipe(istore_reg_mem);
7740 %}
7741
7742 instruct storeimmC0(immI0 zero, memory2 mem)
7743 %{
7744 match(Set mem (StoreC mem zero));
7745 predicate(!needs_releasing_store(n));
7746
7747 ins_cost(INSN_COST);
7748 format %{ "strh zr, $mem\t# short" %}
7749
7750 ins_encode(aarch64_enc_strh0(mem));
7751
7752 ins_pipe(istore_mem);
7753 %}
7754
7755 // Store Integer
7756
7757 instruct storeI(iRegIorL2I src, memory4 mem)
7758 %{
7759 match(Set mem(StoreI mem src));
7760 predicate(!needs_releasing_store(n));
7761
7762 ins_cost(INSN_COST);
7763 format %{ "strw $src, $mem\t# int" %}
7764
7765 ins_encode(aarch64_enc_strw(src, mem));
7766
7767 ins_pipe(istore_reg_mem);
7768 %}
7769
7770 instruct storeimmI0(immI0 zero, memory4 mem)
7771 %{
7772 match(Set mem(StoreI mem zero));
7773 predicate(!needs_releasing_store(n));
7774
7775 ins_cost(INSN_COST);
7776 format %{ "strw zr, $mem\t# int" %}
7777
7778 ins_encode(aarch64_enc_strw0(mem));
7779
7780 ins_pipe(istore_mem);
7781 %}
7782
7783 // Store Long (64 bit signed)
7784 instruct storeL(iRegL src, memory8 mem)
7785 %{
7786 match(Set mem (StoreL mem src));
7787 predicate(!needs_releasing_store(n));
7788
7789 ins_cost(INSN_COST);
7790 format %{ "str $src, $mem\t# int" %}
7791
7792 ins_encode(aarch64_enc_str(src, mem));
7793
7794 ins_pipe(istore_reg_mem);
7795 %}
7796
7797 // Store Long (64 bit signed)
7798 instruct storeimmL0(immL0 zero, memory8 mem)
7799 %{
7800 match(Set mem (StoreL mem zero));
7801 predicate(!needs_releasing_store(n));
7802
7803 ins_cost(INSN_COST);
7804 format %{ "str zr, $mem\t# int" %}
7805
7806 ins_encode(aarch64_enc_str0(mem));
7807
7808 ins_pipe(istore_mem);
7809 %}
7810
7811 // Store Pointer
7812 instruct storeP(iRegP src, memory8 mem)
7813 %{
7814 match(Set mem (StoreP mem src));
7815 predicate(!needs_releasing_store(n));
7816
7817 ins_cost(INSN_COST);
7818 format %{ "str $src, $mem\t# ptr" %}
7819
7820 ins_encode(aarch64_enc_str(src, mem));
7821
7822 ins_pipe(istore_reg_mem);
7823 %}
7824
7825 // Store Pointer
7826 instruct storeimmP0(immP0 zero, memory8 mem)
7827 %{
7828 match(Set mem (StoreP mem zero));
7829 predicate(!needs_releasing_store(n));
7830
7831 ins_cost(INSN_COST);
7832 format %{ "str zr, $mem\t# ptr" %}
7833
7834 ins_encode(aarch64_enc_str0(mem));
7835
7836 ins_pipe(istore_mem);
7837 %}
7838
7839 // Store Compressed Pointer
7840 instruct storeN(iRegN src, memory4 mem)
7841 %{
7842 match(Set mem (StoreN mem src));
7843 predicate(!needs_releasing_store(n));
7844
7845 ins_cost(INSN_COST);
7846 format %{ "strw $src, $mem\t# compressed ptr" %}
7847
7848 ins_encode(aarch64_enc_strw(src, mem));
7849
7850 ins_pipe(istore_reg_mem);
7851 %}
7852
7853 instruct storeImmN0(immN0 zero, memory4 mem)
7854 %{
7855 match(Set mem (StoreN mem zero));
7856 predicate(!needs_releasing_store(n));
7857
7858 ins_cost(INSN_COST);
7859 format %{ "strw zr, $mem\t# compressed ptr" %}
7860
7861 ins_encode(aarch64_enc_strw0(mem));
7862
7863 ins_pipe(istore_mem);
7864 %}
7865
7866 // Store Float
7867 instruct storeF(vRegF src, memory4 mem)
7868 %{
7869 match(Set mem (StoreF mem src));
7870 predicate(!needs_releasing_store(n));
7871
7872 ins_cost(INSN_COST);
7873 format %{ "strs $src, $mem\t# float" %}
7874
7875 ins_encode( aarch64_enc_strs(src, mem) );
7876
7877 ins_pipe(pipe_class_memory);
7878 %}
7879
7880 // TODO
7881 // implement storeImmF0 and storeFImmPacked
7882
7883 // Store Double
7884 instruct storeD(vRegD src, memory8 mem)
7885 %{
7886 match(Set mem (StoreD mem src));
7887 predicate(!needs_releasing_store(n));
7888
7889 ins_cost(INSN_COST);
7890 format %{ "strd $src, $mem\t# double" %}
7891
7892 ins_encode( aarch64_enc_strd(src, mem) );
7893
7894 ins_pipe(pipe_class_memory);
7895 %}
7896
7897 // Store Compressed Klass Pointer
7898 instruct storeNKlass(iRegN src, memory4 mem)
7899 %{
7900 predicate(!needs_releasing_store(n));
7901 match(Set mem (StoreNKlass mem src));
7902
7903 ins_cost(INSN_COST);
7904 format %{ "strw $src, $mem\t# compressed klass ptr" %}
7905
7906 ins_encode(aarch64_enc_strw(src, mem));
7907
7908 ins_pipe(istore_reg_mem);
7909 %}
7910
7911 // TODO
7912 // implement storeImmD0 and storeDImmPacked
7913
7914 // prefetch instructions
7915 // Must be safe to execute with invalid address (cannot fault).
7916
7917 instruct prefetchalloc( memory8 mem ) %{
7918 match(PrefetchAllocation mem);
7919
7920 ins_cost(INSN_COST);
7921 format %{ "prfm $mem, PSTL1KEEP\t# Prefetch into level 1 cache write keep" %}
7922
7923 ins_encode( aarch64_enc_prefetchw(mem) );
7924
7925 ins_pipe(iload_prefetch);
7926 %}
7927
7928 // ---------------- volatile loads and stores ----------------
7929
7930 // Load Byte (8 bit signed)
7931 instruct loadB_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
7932 %{
7933 match(Set dst (LoadB mem));
7934
7935 ins_cost(VOLATILE_REF_COST);
7936 format %{ "ldarsb $dst, $mem\t# byte" %}
7937
7938 ins_encode(aarch64_enc_ldarsb(dst, mem));
7939
7940 ins_pipe(pipe_serial);
7941 %}
7942
7943 // Load Byte (8 bit signed) into long
7944 instruct loadB2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
7945 %{
7946 match(Set dst (ConvI2L (LoadB mem)));
7947
7948 ins_cost(VOLATILE_REF_COST);
7949 format %{ "ldarsb $dst, $mem\t# byte" %}
7950
7951 ins_encode(aarch64_enc_ldarsb(dst, mem));
7952
7953 ins_pipe(pipe_serial);
7954 %}
7955
7956 // Load Byte (8 bit unsigned)
7957 instruct loadUB_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
7958 %{
7959 match(Set dst (LoadUB mem));
7960
7961 ins_cost(VOLATILE_REF_COST);
7962 format %{ "ldarb $dst, $mem\t# byte" %}
7963
7964 ins_encode(aarch64_enc_ldarb(dst, mem));
7965
7966 ins_pipe(pipe_serial);
7967 %}
7968
7969 // Load Byte (8 bit unsigned) into long
7970 instruct loadUB2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
7971 %{
7972 match(Set dst (ConvI2L (LoadUB mem)));
7973
7974 ins_cost(VOLATILE_REF_COST);
7975 format %{ "ldarb $dst, $mem\t# byte" %}
7976
7977 ins_encode(aarch64_enc_ldarb(dst, mem));
7978
7979 ins_pipe(pipe_serial);
7980 %}
7981
7982 // Load Short (16 bit signed)
7983 instruct loadS_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
7984 %{
7985 match(Set dst (LoadS mem));
7986
7987 ins_cost(VOLATILE_REF_COST);
7988 format %{ "ldarshw $dst, $mem\t# short" %}
7989
7990 ins_encode(aarch64_enc_ldarshw(dst, mem));
7991
7992 ins_pipe(pipe_serial);
7993 %}
7994
7995 instruct loadUS_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
7996 %{
7997 match(Set dst (LoadUS mem));
7998
7999 ins_cost(VOLATILE_REF_COST);
8000 format %{ "ldarhw $dst, $mem\t# short" %}
8001
8002 ins_encode(aarch64_enc_ldarhw(dst, mem));
8003
8004 ins_pipe(pipe_serial);
8005 %}
8006
8007 // Load Short/Char (16 bit unsigned) into long
8008 instruct loadUS2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
8009 %{
8010 match(Set dst (ConvI2L (LoadUS mem)));
8011
8012 ins_cost(VOLATILE_REF_COST);
8013 format %{ "ldarh $dst, $mem\t# short" %}
8014
8015 ins_encode(aarch64_enc_ldarh(dst, mem));
8016
8017 ins_pipe(pipe_serial);
8018 %}
8019
8020 // Load Short/Char (16 bit signed) into long
8021 instruct loadS2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
8022 %{
8023 match(Set dst (ConvI2L (LoadS mem)));
8024
8025 ins_cost(VOLATILE_REF_COST);
8026 format %{ "ldarh $dst, $mem\t# short" %}
8027
8028 ins_encode(aarch64_enc_ldarsh(dst, mem));
8029
8030 ins_pipe(pipe_serial);
8031 %}
8032
8033 // Load Integer (32 bit signed)
8034 instruct loadI_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
8035 %{
8036 match(Set dst (LoadI mem));
8037
8038 ins_cost(VOLATILE_REF_COST);
8039 format %{ "ldarw $dst, $mem\t# int" %}
8040
8041 ins_encode(aarch64_enc_ldarw(dst, mem));
8042
8043 ins_pipe(pipe_serial);
8044 %}
8045
8046 // Load Integer (32 bit unsigned) into long
8047 instruct loadUI2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem, immL_32bits mask)
8048 %{
8049 match(Set dst (AndL (ConvI2L (LoadI mem)) mask));
8050
8051 ins_cost(VOLATILE_REF_COST);
8052 format %{ "ldarw $dst, $mem\t# int" %}
8053
8054 ins_encode(aarch64_enc_ldarw(dst, mem));
8055
8056 ins_pipe(pipe_serial);
8057 %}
8058
8059 // Load Long (64 bit signed)
8060 instruct loadL_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
8061 %{
8062 match(Set dst (LoadL mem));
8063
8064 ins_cost(VOLATILE_REF_COST);
8065 format %{ "ldar $dst, $mem\t# int" %}
8066
8067 ins_encode(aarch64_enc_ldar(dst, mem));
8068
8069 ins_pipe(pipe_serial);
8070 %}
8071
8072 // Load Pointer
8073 instruct loadP_volatile(iRegPNoSp dst, /* sync_memory*/indirect mem)
8074 %{
8075 match(Set dst (LoadP mem));
8076 predicate(n->as_Load()->barrier_data() == 0);
8077
8078 ins_cost(VOLATILE_REF_COST);
8079 format %{ "ldar $dst, $mem\t# ptr" %}
8080
8081 ins_encode(aarch64_enc_ldar(dst, mem));
8082
8083 ins_pipe(pipe_serial);
8084 %}
8085
8086 // Load Compressed Pointer
8087 instruct loadN_volatile(iRegNNoSp dst, /* sync_memory*/indirect mem)
8088 %{
8089 match(Set dst (LoadN mem));
8090
8091 ins_cost(VOLATILE_REF_COST);
8092 format %{ "ldarw $dst, $mem\t# compressed ptr" %}
8093
8094 ins_encode(aarch64_enc_ldarw(dst, mem));
8095
8096 ins_pipe(pipe_serial);
8097 %}
8098
8099 // Load Float
8100 instruct loadF_volatile(vRegF dst, /* sync_memory*/indirect mem)
8101 %{
8102 match(Set dst (LoadF mem));
8103
8104 ins_cost(VOLATILE_REF_COST);
8105 format %{ "ldars $dst, $mem\t# float" %}
8106
8107 ins_encode( aarch64_enc_fldars(dst, mem) );
8108
8109 ins_pipe(pipe_serial);
8110 %}
8111
8112 // Load Double
8113 instruct loadD_volatile(vRegD dst, /* sync_memory*/indirect mem)
8114 %{
8115 match(Set dst (LoadD mem));
8116
8117 ins_cost(VOLATILE_REF_COST);
8118 format %{ "ldard $dst, $mem\t# double" %}
8119
8120 ins_encode( aarch64_enc_fldard(dst, mem) );
8121
8122 ins_pipe(pipe_serial);
8123 %}
8124
8125 // Store Byte
8126 instruct storeB_volatile(iRegIorL2I src, /* sync_memory*/indirect mem)
8127 %{
8128 match(Set mem (StoreB mem src));
8129
8130 ins_cost(VOLATILE_REF_COST);
8131 format %{ "stlrb $src, $mem\t# byte" %}
8132
8133 ins_encode(aarch64_enc_stlrb(src, mem));
8134
8135 ins_pipe(pipe_class_memory);
8136 %}
8137
8138 instruct storeimmB0_volatile(immI0 zero, /* sync_memory*/indirect mem)
8139 %{
8140 match(Set mem (StoreB mem zero));
8141
8142 ins_cost(VOLATILE_REF_COST);
8143 format %{ "stlrb zr, $mem\t# byte" %}
8144
8145 ins_encode(aarch64_enc_stlrb0(mem));
8146
8147 ins_pipe(pipe_class_memory);
8148 %}
8149
8150 // Store Char/Short
8151 instruct storeC_volatile(iRegIorL2I src, /* sync_memory*/indirect mem)
8152 %{
8153 match(Set mem (StoreC mem src));
8154
8155 ins_cost(VOLATILE_REF_COST);
8156 format %{ "stlrh $src, $mem\t# short" %}
8157
8158 ins_encode(aarch64_enc_stlrh(src, mem));
8159
8160 ins_pipe(pipe_class_memory);
8161 %}
8162
8163 instruct storeimmC0_volatile(immI0 zero, /* sync_memory*/indirect mem)
8164 %{
8165 match(Set mem (StoreC mem zero));
8166
8167 ins_cost(VOLATILE_REF_COST);
8168 format %{ "stlrh zr, $mem\t# short" %}
8169
8170 ins_encode(aarch64_enc_stlrh0(mem));
8171
8172 ins_pipe(pipe_class_memory);
8173 %}
8174
8175 // Store Integer
8176
8177 instruct storeI_volatile(iRegIorL2I src, /* sync_memory*/indirect mem)
8178 %{
8179 match(Set mem(StoreI mem src));
8180
8181 ins_cost(VOLATILE_REF_COST);
8182 format %{ "stlrw $src, $mem\t# int" %}
8183
8184 ins_encode(aarch64_enc_stlrw(src, mem));
8185
8186 ins_pipe(pipe_class_memory);
8187 %}
8188
8189 instruct storeimmI0_volatile(immI0 zero, /* sync_memory*/indirect mem)
8190 %{
8191 match(Set mem(StoreI mem zero));
8192
8193 ins_cost(VOLATILE_REF_COST);
8194 format %{ "stlrw zr, $mem\t# int" %}
8195
8196 ins_encode(aarch64_enc_stlrw0(mem));
8197
8198 ins_pipe(pipe_class_memory);
8199 %}
8200
8201 // Store Long (64 bit signed)
8202 instruct storeL_volatile(iRegL src, /* sync_memory*/indirect mem)
8203 %{
8204 match(Set mem (StoreL mem src));
8205
8206 ins_cost(VOLATILE_REF_COST);
8207 format %{ "stlr $src, $mem\t# int" %}
8208
8209 ins_encode(aarch64_enc_stlr(src, mem));
8210
8211 ins_pipe(pipe_class_memory);
8212 %}
8213
8214 instruct storeimmL0_volatile(immL0 zero, /* sync_memory*/indirect mem)
8215 %{
8216 match(Set mem (StoreL mem zero));
8217
8218 ins_cost(VOLATILE_REF_COST);
8219 format %{ "stlr zr, $mem\t# int" %}
8220
8221 ins_encode(aarch64_enc_stlr0(mem));
8222
8223 ins_pipe(pipe_class_memory);
8224 %}
8225
8226 // Store Pointer
8227 instruct storeP_volatile(iRegP src, /* sync_memory*/indirect mem)
8228 %{
8229 match(Set mem (StoreP mem src));
8230
8231 ins_cost(VOLATILE_REF_COST);
8232 format %{ "stlr $src, $mem\t# ptr" %}
8233
8234 ins_encode(aarch64_enc_stlr(src, mem));
8235
8236 ins_pipe(pipe_class_memory);
8237 %}
8238
8239 instruct storeimmP0_volatile(immP0 zero, /* sync_memory*/indirect mem)
8240 %{
8241 match(Set mem (StoreP mem zero));
8242
8243 ins_cost(VOLATILE_REF_COST);
8244 format %{ "stlr zr, $mem\t# ptr" %}
8245
8246 ins_encode(aarch64_enc_stlr0(mem));
8247
8248 ins_pipe(pipe_class_memory);
8249 %}
8250
8251 // Store Compressed Pointer
8252 instruct storeN_volatile(iRegN src, /* sync_memory*/indirect mem)
8253 %{
8254 match(Set mem (StoreN mem src));
8255
8256 ins_cost(VOLATILE_REF_COST);
8257 format %{ "stlrw $src, $mem\t# compressed ptr" %}
8258
8259 ins_encode(aarch64_enc_stlrw(src, mem));
8260
8261 ins_pipe(pipe_class_memory);
8262 %}
8263
8264 instruct storeimmN0_volatile(immN0 zero, /* sync_memory*/indirect mem)
8265 %{
8266 match(Set mem (StoreN mem zero));
8267
8268 ins_cost(VOLATILE_REF_COST);
8269 format %{ "stlrw zr, $mem\t# compressed ptr" %}
8270
8271 ins_encode(aarch64_enc_stlrw0(mem));
8272
8273 ins_pipe(pipe_class_memory);
8274 %}
8275
8276 // Store Float
8277 instruct storeF_volatile(vRegF src, /* sync_memory*/indirect mem)
8278 %{
8279 match(Set mem (StoreF mem src));
8280
8281 ins_cost(VOLATILE_REF_COST);
8282 format %{ "stlrs $src, $mem\t# float" %}
8283
8284 ins_encode( aarch64_enc_fstlrs(src, mem) );
8285
8286 ins_pipe(pipe_class_memory);
8287 %}
8288
8289 // TODO
8290 // implement storeImmF0 and storeFImmPacked
8291
8292 // Store Double
8293 instruct storeD_volatile(vRegD src, /* sync_memory*/indirect mem)
8294 %{
8295 match(Set mem (StoreD mem src));
8296
8297 ins_cost(VOLATILE_REF_COST);
8298 format %{ "stlrd $src, $mem\t# double" %}
8299
8300 ins_encode( aarch64_enc_fstlrd(src, mem) );
8301
8302 ins_pipe(pipe_class_memory);
8303 %}
8304
8305 // ---------------- end of volatile loads and stores ----------------
8306
8307 instruct cacheWB(indirect addr)
8308 %{
8309 predicate(VM_Version::supports_data_cache_line_flush());
8310 match(CacheWB addr);
8311
8312 ins_cost(100);
8313 format %{"cache wb $addr" %}
8314 ins_encode %{
8315 assert($addr->index_position() < 0, "should be");
8316 assert($addr$$disp == 0, "should be");
8317 __ cache_wb(Address($addr$$base$$Register, 0));
8318 %}
8319 ins_pipe(pipe_slow); // XXX
8320 %}
8321
8322 instruct cacheWBPreSync()
8323 %{
8324 predicate(VM_Version::supports_data_cache_line_flush());
8325 match(CacheWBPreSync);
8326
8327 ins_cost(100);
8328 format %{"cache wb presync" %}
8329 ins_encode %{
8330 __ cache_wbsync(true);
8331 %}
8332 ins_pipe(pipe_slow); // XXX
8333 %}
8334
8335 instruct cacheWBPostSync()
8336 %{
8337 predicate(VM_Version::supports_data_cache_line_flush());
8338 match(CacheWBPostSync);
8339
8340 ins_cost(100);
8341 format %{"cache wb postsync" %}
8342 ins_encode %{
8343 __ cache_wbsync(false);
8344 %}
8345 ins_pipe(pipe_slow); // XXX
8346 %}
8347
8348 // ============================================================================
8349 // BSWAP Instructions
8350
8351 instruct bytes_reverse_int(iRegINoSp dst, iRegIorL2I src) %{
8352 match(Set dst (ReverseBytesI src));
8353
8354 ins_cost(INSN_COST);
8355 format %{ "revw $dst, $src" %}
8356
8357 ins_encode %{
8358 __ revw(as_Register($dst$$reg), as_Register($src$$reg));
8359 %}
8360
8361 ins_pipe(ialu_reg);
8362 %}
8363
8364 instruct bytes_reverse_long(iRegLNoSp dst, iRegL src) %{
8365 match(Set dst (ReverseBytesL src));
8366
8367 ins_cost(INSN_COST);
8368 format %{ "rev $dst, $src" %}
8369
8370 ins_encode %{
8371 __ rev(as_Register($dst$$reg), as_Register($src$$reg));
8372 %}
8373
8374 ins_pipe(ialu_reg);
8375 %}
8376
8377 instruct bytes_reverse_unsigned_short(iRegINoSp dst, iRegIorL2I src) %{
8378 match(Set dst (ReverseBytesUS src));
8379
8380 ins_cost(INSN_COST);
8381 format %{ "rev16w $dst, $src" %}
8382
8383 ins_encode %{
8384 __ rev16w(as_Register($dst$$reg), as_Register($src$$reg));
8385 %}
8386
8387 ins_pipe(ialu_reg);
8388 %}
8389
8390 instruct bytes_reverse_short(iRegINoSp dst, iRegIorL2I src) %{
8391 match(Set dst (ReverseBytesS src));
8392
8393 ins_cost(INSN_COST);
8394 format %{ "rev16w $dst, $src\n\t"
8395 "sbfmw $dst, $dst, #0, #15" %}
8396
8397 ins_encode %{
8398 __ rev16w(as_Register($dst$$reg), as_Register($src$$reg));
8399 __ sbfmw(as_Register($dst$$reg), as_Register($dst$$reg), 0U, 15U);
8400 %}
8401
8402 ins_pipe(ialu_reg);
8403 %}
8404
8405 // ============================================================================
8406 // Zero Count Instructions
8407
8408 instruct countLeadingZerosI(iRegINoSp dst, iRegIorL2I src) %{
8409 match(Set dst (CountLeadingZerosI src));
8410
8411 ins_cost(INSN_COST);
8412 format %{ "clzw $dst, $src" %}
8413 ins_encode %{
8414 __ clzw(as_Register($dst$$reg), as_Register($src$$reg));
8415 %}
8416
8417 ins_pipe(ialu_reg);
8418 %}
8419
8420 instruct countLeadingZerosL(iRegINoSp dst, iRegL src) %{
8421 match(Set dst (CountLeadingZerosL src));
8422
8423 ins_cost(INSN_COST);
8424 format %{ "clz $dst, $src" %}
8425 ins_encode %{
8426 __ clz(as_Register($dst$$reg), as_Register($src$$reg));
8427 %}
8428
8429 ins_pipe(ialu_reg);
8430 %}
8431
8432 instruct countTrailingZerosI(iRegINoSp dst, iRegIorL2I src) %{
8433 match(Set dst (CountTrailingZerosI src));
8434
8435 ins_cost(INSN_COST * 2);
8436 format %{ "rbitw $dst, $src\n\t"
8437 "clzw $dst, $dst" %}
8438 ins_encode %{
8439 __ rbitw(as_Register($dst$$reg), as_Register($src$$reg));
8440 __ clzw(as_Register($dst$$reg), as_Register($dst$$reg));
8441 %}
8442
8443 ins_pipe(ialu_reg);
8444 %}
8445
8446 instruct countTrailingZerosL(iRegINoSp dst, iRegL src) %{
8447 match(Set dst (CountTrailingZerosL src));
8448
8449 ins_cost(INSN_COST * 2);
8450 format %{ "rbit $dst, $src\n\t"
8451 "clz $dst, $dst" %}
8452 ins_encode %{
8453 __ rbit(as_Register($dst$$reg), as_Register($src$$reg));
8454 __ clz(as_Register($dst$$reg), as_Register($dst$$reg));
8455 %}
8456
8457 ins_pipe(ialu_reg);
8458 %}
8459
8460 //---------- Population Count Instructions -------------------------------------
8461 //
8462
8463 instruct popCountI(iRegINoSp dst, iRegIorL2I src, vRegF tmp) %{
8464 predicate(UsePopCountInstruction);
8465 match(Set dst (PopCountI src));
8466 effect(TEMP tmp);
8467 ins_cost(INSN_COST * 13);
8468
8469 format %{ "movw $src, $src\n\t"
8470 "mov $tmp, $src\t# vector (1D)\n\t"
8471 "cnt $tmp, $tmp\t# vector (8B)\n\t"
8472 "addv $tmp, $tmp\t# vector (8B)\n\t"
8473 "mov $dst, $tmp\t# vector (1D)" %}
8474 ins_encode %{
8475 __ movw($src$$Register, $src$$Register); // ensure top 32 bits 0
8476 __ mov($tmp$$FloatRegister, __ T1D, 0, $src$$Register);
8477 __ cnt($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
8478 __ addv($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
8479 __ mov($dst$$Register, $tmp$$FloatRegister, __ T1D, 0);
8480 %}
8481
8482 ins_pipe(pipe_class_default);
8483 %}
8484
8485 instruct popCountI_mem(iRegINoSp dst, memory4 mem, vRegF tmp) %{
8486 predicate(UsePopCountInstruction);
8487 match(Set dst (PopCountI (LoadI mem)));
8488 effect(TEMP tmp);
8489 ins_cost(INSN_COST * 13);
8490
8491 format %{ "ldrs $tmp, $mem\n\t"
8492 "cnt $tmp, $tmp\t# vector (8B)\n\t"
8493 "addv $tmp, $tmp\t# vector (8B)\n\t"
8494 "mov $dst, $tmp\t# vector (1D)" %}
8495 ins_encode %{
8496 FloatRegister tmp_reg = as_FloatRegister($tmp$$reg);
8497 loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrs, tmp_reg, $mem->opcode(),
8498 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
8499 __ cnt($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
8500 __ addv($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
8501 __ mov($dst$$Register, $tmp$$FloatRegister, __ T1D, 0);
8502 %}
8503
8504 ins_pipe(pipe_class_default);
8505 %}
8506
8507 // Note: Long.bitCount(long) returns an int.
8508 instruct popCountL(iRegINoSp dst, iRegL src, vRegD tmp) %{
8509 predicate(UsePopCountInstruction);
8510 match(Set dst (PopCountL src));
8511 effect(TEMP tmp);
8512 ins_cost(INSN_COST * 13);
8513
8514 format %{ "mov $tmp, $src\t# vector (1D)\n\t"
8515 "cnt $tmp, $tmp\t# vector (8B)\n\t"
8516 "addv $tmp, $tmp\t# vector (8B)\n\t"
8517 "mov $dst, $tmp\t# vector (1D)" %}
8518 ins_encode %{
8519 __ mov($tmp$$FloatRegister, __ T1D, 0, $src$$Register);
8520 __ cnt($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
8521 __ addv($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
8522 __ mov($dst$$Register, $tmp$$FloatRegister, __ T1D, 0);
8523 %}
8524
8525 ins_pipe(pipe_class_default);
8526 %}
8527
8528 instruct popCountL_mem(iRegINoSp dst, memory8 mem, vRegD tmp) %{
8529 predicate(UsePopCountInstruction);
8530 match(Set dst (PopCountL (LoadL mem)));
8531 effect(TEMP tmp);
8532 ins_cost(INSN_COST * 13);
8533
8534 format %{ "ldrd $tmp, $mem\n\t"
8535 "cnt $tmp, $tmp\t# vector (8B)\n\t"
8536 "addv $tmp, $tmp\t# vector (8B)\n\t"
8537 "mov $dst, $tmp\t# vector (1D)" %}
8538 ins_encode %{
8539 FloatRegister tmp_reg = as_FloatRegister($tmp$$reg);
8540 loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrd, tmp_reg, $mem->opcode(),
8541 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
8542 __ cnt($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
8543 __ addv($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
8544 __ mov($dst$$Register, $tmp$$FloatRegister, __ T1D, 0);
8545 %}
8546
8547 ins_pipe(pipe_class_default);
8548 %}
8549
8550 // ============================================================================
8551 // MemBar Instruction
8552
8553 instruct load_fence() %{
8554 match(LoadFence);
8555 ins_cost(VOLATILE_REF_COST);
8556
8557 format %{ "load_fence" %}
8558
8559 ins_encode %{
8560 __ membar(Assembler::LoadLoad|Assembler::LoadStore);
8561 %}
8562 ins_pipe(pipe_serial);
8563 %}
8564
8565 instruct unnecessary_membar_acquire() %{
8566 predicate(unnecessary_acquire(n));
8567 match(MemBarAcquire);
8568 ins_cost(0);
8569
8570 format %{ "membar_acquire (elided)" %}
8571
8572 ins_encode %{
8573 __ block_comment("membar_acquire (elided)");
8574 %}
8575
8576 ins_pipe(pipe_class_empty);
8577 %}
8578
8579 instruct membar_acquire() %{
8580 match(MemBarAcquire);
8581 ins_cost(VOLATILE_REF_COST);
8582
8583 format %{ "membar_acquire\n\t"
8584 "dmb ish" %}
8585
8586 ins_encode %{
8587 __ block_comment("membar_acquire");
8588 __ membar(Assembler::LoadLoad|Assembler::LoadStore);
8589 %}
8590
8591 ins_pipe(pipe_serial);
8592 %}
8593
8594
8595 instruct membar_acquire_lock() %{
8596 match(MemBarAcquireLock);
8597 ins_cost(VOLATILE_REF_COST);
8598
8599 format %{ "membar_acquire_lock (elided)" %}
8600
8601 ins_encode %{
8602 __ block_comment("membar_acquire_lock (elided)");
8603 %}
8604
8605 ins_pipe(pipe_serial);
8606 %}
8607
8608 instruct store_fence() %{
8609 match(StoreFence);
8610 ins_cost(VOLATILE_REF_COST);
8611
8612 format %{ "store_fence" %}
8613
8614 ins_encode %{
8615 __ membar(Assembler::LoadStore|Assembler::StoreStore);
8616 %}
8617 ins_pipe(pipe_serial);
8618 %}
8619
8620 instruct unnecessary_membar_release() %{
8621 predicate(unnecessary_release(n));
8622 match(MemBarRelease);
8623 ins_cost(0);
8624
8625 format %{ "membar_release (elided)" %}
8626
8627 ins_encode %{
8628 __ block_comment("membar_release (elided)");
8629 %}
8630 ins_pipe(pipe_serial);
8631 %}
8632
8633 instruct membar_release() %{
8634 match(MemBarRelease);
8635 ins_cost(VOLATILE_REF_COST);
8636
8637 format %{ "membar_release\n\t"
8638 "dmb ish" %}
8639
8640 ins_encode %{
8641 __ block_comment("membar_release");
8642 __ membar(Assembler::LoadStore|Assembler::StoreStore);
8643 %}
8644 ins_pipe(pipe_serial);
8645 %}
8646
8647 instruct membar_storestore() %{
8648 match(MemBarStoreStore);
8649 match(StoreStoreFence);
8650 ins_cost(VOLATILE_REF_COST);
8651
8652 format %{ "MEMBAR-store-store" %}
8653
8654 ins_encode %{
8655 __ membar(Assembler::StoreStore);
8656 %}
8657 ins_pipe(pipe_serial);
8658 %}
8659
8660 instruct membar_release_lock() %{
8661 match(MemBarReleaseLock);
8662 ins_cost(VOLATILE_REF_COST);
8663
8664 format %{ "membar_release_lock (elided)" %}
8665
8666 ins_encode %{
8667 __ block_comment("membar_release_lock (elided)");
8668 %}
8669
8670 ins_pipe(pipe_serial);
8671 %}
8672
8673 instruct unnecessary_membar_volatile() %{
8674 predicate(unnecessary_volatile(n));
8675 match(MemBarVolatile);
8676 ins_cost(0);
8677
8678 format %{ "membar_volatile (elided)" %}
8679
8680 ins_encode %{
8681 __ block_comment("membar_volatile (elided)");
8682 %}
8683
8684 ins_pipe(pipe_serial);
8685 %}
8686
8687 instruct membar_volatile() %{
8688 match(MemBarVolatile);
8689 ins_cost(VOLATILE_REF_COST*100);
8690
8691 format %{ "membar_volatile\n\t"
8692 "dmb ish"%}
8693
8694 ins_encode %{
8695 __ block_comment("membar_volatile");
8696 __ membar(Assembler::StoreLoad);
8697 %}
8698
8699 ins_pipe(pipe_serial);
8700 %}
8701
8702 // ============================================================================
8703 // Cast/Convert Instructions
8704
8705 instruct castX2P(iRegPNoSp dst, iRegL src) %{
8706 match(Set dst (CastX2P src));
8707
8708 ins_cost(INSN_COST);
8709 format %{ "mov $dst, $src\t# long -> ptr" %}
8710
8711 ins_encode %{
8712 if ($dst$$reg != $src$$reg) {
8713 __ mov(as_Register($dst$$reg), as_Register($src$$reg));
8714 }
8715 %}
8716
8717 ins_pipe(ialu_reg);
8718 %}
8719
8720 instruct castP2X(iRegLNoSp dst, iRegP src) %{
8721 match(Set dst (CastP2X src));
8722
8723 ins_cost(INSN_COST);
8724 format %{ "mov $dst, $src\t# ptr -> long" %}
8725
8726 ins_encode %{
8727 if ($dst$$reg != $src$$reg) {
8728 __ mov(as_Register($dst$$reg), as_Register($src$$reg));
8729 }
8730 %}
8731
8732 ins_pipe(ialu_reg);
8733 %}
8734
8735 // Convert oop into int for vectors alignment masking
8736 instruct convP2I(iRegINoSp dst, iRegP src) %{
8737 match(Set dst (ConvL2I (CastP2X src)));
8738
8739 ins_cost(INSN_COST);
8740 format %{ "movw $dst, $src\t# ptr -> int" %}
8741 ins_encode %{
8742 __ movw($dst$$Register, $src$$Register);
8743 %}
8744
8745 ins_pipe(ialu_reg);
8746 %}
8747
8748 // Convert compressed oop into int for vectors alignment masking
8749 // in case of 32bit oops (heap < 4Gb).
8750 instruct convN2I(iRegINoSp dst, iRegN src)
8751 %{
8752 predicate(CompressedOops::shift() == 0);
8753 match(Set dst (ConvL2I (CastP2X (DecodeN src))));
8754
8755 ins_cost(INSN_COST);
8756 format %{ "mov dst, $src\t# compressed ptr -> int" %}
8757 ins_encode %{
8758 __ movw($dst$$Register, $src$$Register);
8759 %}
8760
8761 ins_pipe(ialu_reg);
8762 %}
8763
8764
8765 // Convert oop pointer into compressed form
8766 instruct encodeHeapOop(iRegNNoSp dst, iRegP src, rFlagsReg cr) %{
8767 predicate(n->bottom_type()->make_ptr()->ptr() != TypePtr::NotNull);
8768 match(Set dst (EncodeP src));
8769 effect(KILL cr);
8770 ins_cost(INSN_COST * 3);
8771 format %{ "encode_heap_oop $dst, $src" %}
8772 ins_encode %{
8773 Register s = $src$$Register;
8774 Register d = $dst$$Register;
8775 __ encode_heap_oop(d, s);
8776 %}
8777 ins_pipe(ialu_reg);
8778 %}
8779
8780 instruct encodeHeapOop_not_null(iRegNNoSp dst, iRegP src, rFlagsReg cr) %{
8781 predicate(n->bottom_type()->make_ptr()->ptr() == TypePtr::NotNull);
8782 match(Set dst (EncodeP src));
8783 ins_cost(INSN_COST * 3);
8784 format %{ "encode_heap_oop_not_null $dst, $src" %}
8785 ins_encode %{
8786 __ encode_heap_oop_not_null($dst$$Register, $src$$Register);
8787 %}
8788 ins_pipe(ialu_reg);
8789 %}
8790
8791 instruct decodeHeapOop(iRegPNoSp dst, iRegN src, rFlagsReg cr) %{
8792 predicate(n->bottom_type()->is_ptr()->ptr() != TypePtr::NotNull &&
8793 n->bottom_type()->is_ptr()->ptr() != TypePtr::Constant);
8794 match(Set dst (DecodeN src));
8795 ins_cost(INSN_COST * 3);
8796 format %{ "decode_heap_oop $dst, $src" %}
8797 ins_encode %{
8798 Register s = $src$$Register;
8799 Register d = $dst$$Register;
8800 __ decode_heap_oop(d, s);
8801 %}
8802 ins_pipe(ialu_reg);
8803 %}
8804
8805 instruct decodeHeapOop_not_null(iRegPNoSp dst, iRegN src, rFlagsReg cr) %{
8806 predicate(n->bottom_type()->is_ptr()->ptr() == TypePtr::NotNull ||
8807 n->bottom_type()->is_ptr()->ptr() == TypePtr::Constant);
8808 match(Set dst (DecodeN src));
8809 ins_cost(INSN_COST * 3);
8810 format %{ "decode_heap_oop_not_null $dst, $src" %}
8811 ins_encode %{
8812 Register s = $src$$Register;
8813 Register d = $dst$$Register;
8814 __ decode_heap_oop_not_null(d, s);
8815 %}
8816 ins_pipe(ialu_reg);
8817 %}
8818
8819 // n.b. AArch64 implementations of encode_klass_not_null and
8820 // decode_klass_not_null do not modify the flags register so, unlike
8821 // Intel, we don't kill CR as a side effect here
8822
8823 instruct encodeKlass_not_null(iRegNNoSp dst, iRegP src) %{
8824 match(Set dst (EncodePKlass src));
8825
8826 ins_cost(INSN_COST * 3);
8827 format %{ "encode_klass_not_null $dst,$src" %}
8828
8829 ins_encode %{
8830 Register src_reg = as_Register($src$$reg);
8831 Register dst_reg = as_Register($dst$$reg);
8832 __ encode_klass_not_null(dst_reg, src_reg);
8833 %}
8834
8835 ins_pipe(ialu_reg);
8836 %}
8837
8838 instruct decodeKlass_not_null(iRegPNoSp dst, iRegN src) %{
8839 match(Set dst (DecodeNKlass src));
8840
8841 ins_cost(INSN_COST * 3);
8842 format %{ "decode_klass_not_null $dst,$src" %}
8843
8844 ins_encode %{
8845 Register src_reg = as_Register($src$$reg);
8846 Register dst_reg = as_Register($dst$$reg);
8847 if (dst_reg != src_reg) {
8848 __ decode_klass_not_null(dst_reg, src_reg);
8849 } else {
8850 __ decode_klass_not_null(dst_reg);
8851 }
8852 %}
8853
8854 ins_pipe(ialu_reg);
8855 %}
8856
8857 instruct checkCastPP(iRegPNoSp dst)
8858 %{
8859 match(Set dst (CheckCastPP dst));
8860
8861 size(0);
8862 format %{ "# checkcastPP of $dst" %}
8863 ins_encode(/* empty encoding */);
8864 ins_pipe(pipe_class_empty);
8865 %}
8866
8867 instruct castPP(iRegPNoSp dst)
8868 %{
8869 match(Set dst (CastPP dst));
8870
8871 size(0);
8872 format %{ "# castPP of $dst" %}
8873 ins_encode(/* empty encoding */);
8874 ins_pipe(pipe_class_empty);
8875 %}
8876
8877 instruct castII(iRegI dst)
8878 %{
8879 match(Set dst (CastII dst));
8880
8881 size(0);
8882 format %{ "# castII of $dst" %}
8883 ins_encode(/* empty encoding */);
8884 ins_cost(0);
8885 ins_pipe(pipe_class_empty);
8886 %}
8887
8888 instruct castLL(iRegL dst)
8889 %{
8890 match(Set dst (CastLL dst));
8891
8892 size(0);
8893 format %{ "# castLL of $dst" %}
8894 ins_encode(/* empty encoding */);
8895 ins_cost(0);
8896 ins_pipe(pipe_class_empty);
8897 %}
8898
8899 instruct castFF(vRegF dst)
8900 %{
8901 match(Set dst (CastFF dst));
8902
8903 size(0);
8904 format %{ "# castFF of $dst" %}
8905 ins_encode(/* empty encoding */);
8906 ins_cost(0);
8907 ins_pipe(pipe_class_empty);
8908 %}
8909
8910 instruct castDD(vRegD dst)
8911 %{
8912 match(Set dst (CastDD dst));
8913
8914 size(0);
8915 format %{ "# castDD of $dst" %}
8916 ins_encode(/* empty encoding */);
8917 ins_cost(0);
8918 ins_pipe(pipe_class_empty);
8919 %}
8920
8921 instruct castVVD(vecD dst)
8922 %{
8923 match(Set dst (CastVV dst));
8924
8925 size(0);
8926 format %{ "# castVV of $dst" %}
8927 ins_encode(/* empty encoding */);
8928 ins_cost(0);
8929 ins_pipe(pipe_class_empty);
8930 %}
8931
8932 instruct castVVX(vecX dst)
8933 %{
8934 match(Set dst (CastVV dst));
8935
8936 size(0);
8937 format %{ "# castVV of $dst" %}
8938 ins_encode(/* empty encoding */);
8939 ins_cost(0);
8940 ins_pipe(pipe_class_empty);
8941 %}
8942
8943 instruct castVV(vReg dst)
8944 %{
8945 match(Set dst (CastVV dst));
8946
8947 size(0);
8948 format %{ "# castVV of $dst" %}
8949 ins_encode(/* empty encoding */);
8950 ins_cost(0);
8951 ins_pipe(pipe_class_empty);
8952 %}
8953
8954 // ============================================================================
8955 // Atomic operation instructions
8956 //
8957 // Intel and SPARC both implement Ideal Node LoadPLocked and
8958 // Store{PIL}Conditional instructions using a normal load for the
8959 // LoadPLocked and a CAS for the Store{PIL}Conditional.
8960 //
8961 // The ideal code appears only to use LoadPLocked/StorePLocked as a
8962 // pair to lock object allocations from Eden space when not using
8963 // TLABs.
8964 //
8965 // There does not appear to be a Load{IL}Locked Ideal Node and the
8966 // Ideal code appears to use Store{IL}Conditional as an alias for CAS
8967 // and to use StoreIConditional only for 32-bit and StoreLConditional
8968 // only for 64-bit.
8969 //
8970 // We implement LoadPLocked and StorePLocked instructions using,
8971 // respectively the AArch64 hw load-exclusive and store-conditional
8972 // instructions. Whereas we must implement each of
8973 // Store{IL}Conditional using a CAS which employs a pair of
8974 // instructions comprising a load-exclusive followed by a
8975 // store-conditional.
8976
8977
8978 // Locked-load (linked load) of the current heap-top
8979 // used when updating the eden heap top
8980 // implemented using ldaxr on AArch64
8981
8982 instruct loadPLocked(iRegPNoSp dst, indirect mem)
8983 %{
8984 match(Set dst (LoadPLocked mem));
8985
8986 ins_cost(VOLATILE_REF_COST);
8987
8988 format %{ "ldaxr $dst, $mem\t# ptr linked acquire" %}
8989
8990 ins_encode(aarch64_enc_ldaxr(dst, mem));
8991
8992 ins_pipe(pipe_serial);
8993 %}
8994
8995 // Conditional-store of the updated heap-top.
8996 // Used during allocation of the shared heap.
8997 // Sets flag (EQ) on success.
8998 // implemented using stlxr on AArch64.
8999
9000 instruct storePConditional(memory8 heap_top_ptr, iRegP oldval, iRegP newval, rFlagsReg cr)
9001 %{
9002 match(Set cr (StorePConditional heap_top_ptr (Binary oldval newval)));
9003
9004 ins_cost(VOLATILE_REF_COST);
9005
9006 // TODO
9007 // do we need to do a store-conditional release or can we just use a
9008 // plain store-conditional?
9009
9010 format %{
9011 "stlxr rscratch1, $newval, $heap_top_ptr\t# ptr cond release"
9012 "cmpw rscratch1, zr\t# EQ on successful write"
9013 %}
9014
9015 ins_encode(aarch64_enc_stlxr(newval, heap_top_ptr));
9016
9017 ins_pipe(pipe_serial);
9018 %}
9019
9020
9021 // storeLConditional is used by PhaseMacroExpand::expand_lock_node
9022 // when attempting to rebias a lock towards the current thread. We
9023 // must use the acquire form of cmpxchg in order to guarantee acquire
9024 // semantics in this case.
9025 instruct storeLConditional(indirect mem, iRegLNoSp oldval, iRegLNoSp newval, rFlagsReg cr)
9026 %{
9027 match(Set cr (StoreLConditional mem (Binary oldval newval)));
9028
9029 ins_cost(VOLATILE_REF_COST);
9030
9031 format %{
9032 "cmpxchg rscratch1, $mem, $oldval, $newval, $mem\t# if $mem == $oldval then $mem <-- $newval"
9033 "cmpw rscratch1, zr\t# EQ on successful write"
9034 %}
9035
9036 ins_encode(aarch64_enc_cmpxchg_acq(mem, oldval, newval));
9037
9038 ins_pipe(pipe_slow);
9039 %}
9040
9041 // storeIConditional also has acquire semantics, for no better reason
9042 // than matching storeLConditional. At the time of writing this
9043 // comment storeIConditional was not used anywhere by AArch64.
9044 instruct storeIConditional(indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr)
9045 %{
9046 match(Set cr (StoreIConditional mem (Binary oldval newval)));
9047
9048 ins_cost(VOLATILE_REF_COST);
9049
9050 format %{
9051 "cmpxchgw rscratch1, $mem, $oldval, $newval, $mem\t# if $mem == $oldval then $mem <-- $newval"
9052 "cmpw rscratch1, zr\t# EQ on successful write"
9053 %}
9054
9055 ins_encode(aarch64_enc_cmpxchgw_acq(mem, oldval, newval));
9056
9057 ins_pipe(pipe_slow);
9058 %}
9059
9060 // standard CompareAndSwapX when we are using barriers
9061 // these have higher priority than the rules selected by a predicate
9062
9063 // XXX No flag versions for CompareAndSwap{I,L,P,N} because matcher
9064 // can't match them
9065
9066 instruct compareAndSwapB(iRegINoSp res, indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr) %{
9067
9068 match(Set res (CompareAndSwapB mem (Binary oldval newval)));
9069 ins_cost(2 * VOLATILE_REF_COST);
9070
9071 effect(KILL cr);
9072
9073 format %{
9074 "cmpxchgb $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval"
9075 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
9076 %}
9077
9078 ins_encode(aarch64_enc_cmpxchgb(mem, oldval, newval),
9079 aarch64_enc_cset_eq(res));
9080
9081 ins_pipe(pipe_slow);
9082 %}
9083
9084 instruct compareAndSwapS(iRegINoSp res, indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr) %{
9085
9086 match(Set res (CompareAndSwapS mem (Binary oldval newval)));
9087 ins_cost(2 * VOLATILE_REF_COST);
9088
9089 effect(KILL cr);
9090
9091 format %{
9092 "cmpxchgs $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval"
9093 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
9094 %}
9095
9096 ins_encode(aarch64_enc_cmpxchgs(mem, oldval, newval),
9097 aarch64_enc_cset_eq(res));
9098
9099 ins_pipe(pipe_slow);
9100 %}
9101
9102 instruct compareAndSwapI(iRegINoSp res, indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr) %{
9103
9104 match(Set res (CompareAndSwapI mem (Binary oldval newval)));
9105 ins_cost(2 * VOLATILE_REF_COST);
9106
9107 effect(KILL cr);
9108
9109 format %{
9110 "cmpxchgw $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval"
9111 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
9112 %}
9113
9114 ins_encode(aarch64_enc_cmpxchgw(mem, oldval, newval),
9115 aarch64_enc_cset_eq(res));
9116
9117 ins_pipe(pipe_slow);
9118 %}
9119
9120 instruct compareAndSwapL(iRegINoSp res, indirect mem, iRegLNoSp oldval, iRegLNoSp newval, rFlagsReg cr) %{
9121
9122 match(Set res (CompareAndSwapL mem (Binary oldval newval)));
9123 ins_cost(2 * VOLATILE_REF_COST);
9124
9125 effect(KILL cr);
9126
9127 format %{
9128 "cmpxchg $mem, $oldval, $newval\t# (long) if $mem == $oldval then $mem <-- $newval"
9129 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
9130 %}
9131
9132 ins_encode(aarch64_enc_cmpxchg(mem, oldval, newval),
9133 aarch64_enc_cset_eq(res));
9134
9135 ins_pipe(pipe_slow);
9136 %}
9137
9138 instruct compareAndSwapP(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
9139
9140 match(Set res (CompareAndSwapP mem (Binary oldval newval)));
9141 predicate(n->as_LoadStore()->barrier_data() == 0);
9142 ins_cost(2 * VOLATILE_REF_COST);
9143
9144 effect(KILL cr);
9145
9146 format %{
9147 "cmpxchg $mem, $oldval, $newval\t# (ptr) if $mem == $oldval then $mem <-- $newval"
9148 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
9149 %}
9150
9151 ins_encode(aarch64_enc_cmpxchg(mem, oldval, newval),
9152 aarch64_enc_cset_eq(res));
9153
9154 ins_pipe(pipe_slow);
9155 %}
9156
9157 instruct compareAndSwapN(iRegINoSp res, indirect mem, iRegNNoSp oldval, iRegNNoSp newval, rFlagsReg cr) %{
9158
9159 match(Set res (CompareAndSwapN mem (Binary oldval newval)));
9160 ins_cost(2 * VOLATILE_REF_COST);
9161
9162 effect(KILL cr);
9163
9164 format %{
9165 "cmpxchgw $mem, $oldval, $newval\t# (narrow oop) if $mem == $oldval then $mem <-- $newval"
9166 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
9167 %}
9168
9169 ins_encode(aarch64_enc_cmpxchgw(mem, oldval, newval),
9170 aarch64_enc_cset_eq(res));
9171
9172 ins_pipe(pipe_slow);
9173 %}
9174
9175 // alternative CompareAndSwapX when we are eliding barriers
9176
9177 instruct compareAndSwapBAcq(iRegINoSp res, indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr) %{
9178
9179 predicate(needs_acquiring_load_exclusive(n));
9180 match(Set res (CompareAndSwapB mem (Binary oldval newval)));
9181 ins_cost(VOLATILE_REF_COST);
9182
9183 effect(KILL cr);
9184
9185 format %{
9186 "cmpxchgb_acq $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval"
9187 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
9188 %}
9189
9190 ins_encode(aarch64_enc_cmpxchgb_acq(mem, oldval, newval),
9191 aarch64_enc_cset_eq(res));
9192
9193 ins_pipe(pipe_slow);
9194 %}
9195
9196 instruct compareAndSwapSAcq(iRegINoSp res, indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr) %{
9197
9198 predicate(needs_acquiring_load_exclusive(n));
9199 match(Set res (CompareAndSwapS mem (Binary oldval newval)));
9200 ins_cost(VOLATILE_REF_COST);
9201
9202 effect(KILL cr);
9203
9204 format %{
9205 "cmpxchgs_acq $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval"
9206 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
9207 %}
9208
9209 ins_encode(aarch64_enc_cmpxchgs_acq(mem, oldval, newval),
9210 aarch64_enc_cset_eq(res));
9211
9212 ins_pipe(pipe_slow);
9213 %}
9214
9215 instruct compareAndSwapIAcq(iRegINoSp res, indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr) %{
9216
9217 predicate(needs_acquiring_load_exclusive(n));
9218 match(Set res (CompareAndSwapI mem (Binary oldval newval)));
9219 ins_cost(VOLATILE_REF_COST);
9220
9221 effect(KILL cr);
9222
9223 format %{
9224 "cmpxchgw_acq $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval"
9225 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
9226 %}
9227
9228 ins_encode(aarch64_enc_cmpxchgw_acq(mem, oldval, newval),
9229 aarch64_enc_cset_eq(res));
9230
9231 ins_pipe(pipe_slow);
9232 %}
9233
9234 instruct compareAndSwapLAcq(iRegINoSp res, indirect mem, iRegLNoSp oldval, iRegLNoSp newval, rFlagsReg cr) %{
9235
9236 predicate(needs_acquiring_load_exclusive(n));
9237 match(Set res (CompareAndSwapL mem (Binary oldval newval)));
9238 ins_cost(VOLATILE_REF_COST);
9239
9240 effect(KILL cr);
9241
9242 format %{
9243 "cmpxchg_acq $mem, $oldval, $newval\t# (long) if $mem == $oldval then $mem <-- $newval"
9244 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
9245 %}
9246
9247 ins_encode(aarch64_enc_cmpxchg_acq(mem, oldval, newval),
9248 aarch64_enc_cset_eq(res));
9249
9250 ins_pipe(pipe_slow);
9251 %}
9252
9253 instruct compareAndSwapPAcq(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
9254
9255 predicate(needs_acquiring_load_exclusive(n) && (n->as_LoadStore()->barrier_data() == 0));
9256 match(Set res (CompareAndSwapP mem (Binary oldval newval)));
9257 ins_cost(VOLATILE_REF_COST);
9258
9259 effect(KILL cr);
9260
9261 format %{
9262 "cmpxchg_acq $mem, $oldval, $newval\t# (ptr) if $mem == $oldval then $mem <-- $newval"
9263 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
9264 %}
9265
9266 ins_encode(aarch64_enc_cmpxchg_acq(mem, oldval, newval),
9267 aarch64_enc_cset_eq(res));
9268
9269 ins_pipe(pipe_slow);
9270 %}
9271
9272 instruct compareAndSwapNAcq(iRegINoSp res, indirect mem, iRegNNoSp oldval, iRegNNoSp newval, rFlagsReg cr) %{
9273
9274 predicate(needs_acquiring_load_exclusive(n));
9275 match(Set res (CompareAndSwapN mem (Binary oldval newval)));
9276 ins_cost(VOLATILE_REF_COST);
9277
9278 effect(KILL cr);
9279
9280 format %{
9281 "cmpxchgw_acq $mem, $oldval, $newval\t# (narrow oop) if $mem == $oldval then $mem <-- $newval"
9282 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
9283 %}
9284
9285 ins_encode(aarch64_enc_cmpxchgw_acq(mem, oldval, newval),
9286 aarch64_enc_cset_eq(res));
9287
9288 ins_pipe(pipe_slow);
9289 %}
9290
9291
9292 // ---------------------------------------------------------------------
9293
9294 // BEGIN This section of the file is automatically generated. Do not edit --------------
9295
9296 // Sundry CAS operations. Note that release is always true,
9297 // regardless of the memory ordering of the CAS. This is because we
9298 // need the volatile case to be sequentially consistent but there is
9299 // no trailing StoreLoad barrier emitted by C2. Unfortunately we
9300 // can't check the type of memory ordering here, so we always emit a
9301 // STLXR.
9302
9303 // This section is generated from aarch64_ad_cas.m4
9304
9305
9306
9307 // This pattern is generated automatically from cas.m4.
9308 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9309 instruct compareAndExchangeB(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
9310
9311 match(Set res (CompareAndExchangeB mem (Binary oldval newval)));
9312 ins_cost(2 * VOLATILE_REF_COST);
9313 effect(TEMP_DEF res, KILL cr);
9314 format %{
9315 "cmpxchgb $res = $mem, $oldval, $newval\t# (byte, weak) if $mem == $oldval then $mem <-- $newval"
9316 %}
9317 ins_encode %{
9318 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
9319 Assembler::byte, /*acquire*/ false, /*release*/ true,
9320 /*weak*/ false, $res$$Register);
9321 __ sxtbw($res$$Register, $res$$Register);
9322 %}
9323 ins_pipe(pipe_slow);
9324 %}
9325
9326 // This pattern is generated automatically from cas.m4.
9327 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9328 instruct compareAndExchangeS(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
9329
9330 match(Set res (CompareAndExchangeS mem (Binary oldval newval)));
9331 ins_cost(2 * VOLATILE_REF_COST);
9332 effect(TEMP_DEF res, KILL cr);
9333 format %{
9334 "cmpxchgs $res = $mem, $oldval, $newval\t# (short, weak) if $mem == $oldval then $mem <-- $newval"
9335 %}
9336 ins_encode %{
9337 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
9338 Assembler::halfword, /*acquire*/ false, /*release*/ true,
9339 /*weak*/ false, $res$$Register);
9340 __ sxthw($res$$Register, $res$$Register);
9341 %}
9342 ins_pipe(pipe_slow);
9343 %}
9344
9345 // This pattern is generated automatically from cas.m4.
9346 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9347 instruct compareAndExchangeI(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
9348
9349 match(Set res (CompareAndExchangeI mem (Binary oldval newval)));
9350 ins_cost(2 * VOLATILE_REF_COST);
9351 effect(TEMP_DEF res, KILL cr);
9352 format %{
9353 "cmpxchgw $res = $mem, $oldval, $newval\t# (int, weak) if $mem == $oldval then $mem <-- $newval"
9354 %}
9355 ins_encode %{
9356 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
9357 Assembler::word, /*acquire*/ false, /*release*/ true,
9358 /*weak*/ false, $res$$Register);
9359 %}
9360 ins_pipe(pipe_slow);
9361 %}
9362
9363 // This pattern is generated automatically from cas.m4.
9364 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9365 instruct compareAndExchangeL(iRegLNoSp res, indirect mem, iRegL oldval, iRegL newval, rFlagsReg cr) %{
9366
9367 match(Set res (CompareAndExchangeL mem (Binary oldval newval)));
9368 ins_cost(2 * VOLATILE_REF_COST);
9369 effect(TEMP_DEF res, KILL cr);
9370 format %{
9371 "cmpxchg $res = $mem, $oldval, $newval\t# (long, weak) if $mem == $oldval then $mem <-- $newval"
9372 %}
9373 ins_encode %{
9374 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
9375 Assembler::xword, /*acquire*/ false, /*release*/ true,
9376 /*weak*/ false, $res$$Register);
9377 %}
9378 ins_pipe(pipe_slow);
9379 %}
9380
9381 // This pattern is generated automatically from cas.m4.
9382 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9383 instruct compareAndExchangeN(iRegNNoSp res, indirect mem, iRegN oldval, iRegN newval, rFlagsReg cr) %{
9384
9385 match(Set res (CompareAndExchangeN mem (Binary oldval newval)));
9386 ins_cost(2 * VOLATILE_REF_COST);
9387 effect(TEMP_DEF res, KILL cr);
9388 format %{
9389 "cmpxchgw $res = $mem, $oldval, $newval\t# (narrow oop, weak) if $mem == $oldval then $mem <-- $newval"
9390 %}
9391 ins_encode %{
9392 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
9393 Assembler::word, /*acquire*/ false, /*release*/ true,
9394 /*weak*/ false, $res$$Register);
9395 %}
9396 ins_pipe(pipe_slow);
9397 %}
9398
9399 // This pattern is generated automatically from cas.m4.
9400 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9401 instruct compareAndExchangeP(iRegPNoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
9402 predicate(n->as_LoadStore()->barrier_data() == 0);
9403 match(Set res (CompareAndExchangeP mem (Binary oldval newval)));
9404 ins_cost(2 * VOLATILE_REF_COST);
9405 effect(TEMP_DEF res, KILL cr);
9406 format %{
9407 "cmpxchg $res = $mem, $oldval, $newval\t# (ptr, weak) if $mem == $oldval then $mem <-- $newval"
9408 %}
9409 ins_encode %{
9410 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
9411 Assembler::xword, /*acquire*/ false, /*release*/ true,
9412 /*weak*/ false, $res$$Register);
9413 %}
9414 ins_pipe(pipe_slow);
9415 %}
9416
9417 // This pattern is generated automatically from cas.m4.
9418 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9419 instruct compareAndExchangeBAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
9420 predicate(needs_acquiring_load_exclusive(n));
9421 match(Set res (CompareAndExchangeB mem (Binary oldval newval)));
9422 ins_cost(VOLATILE_REF_COST);
9423 effect(TEMP_DEF res, KILL cr);
9424 format %{
9425 "cmpxchgb_acq $res = $mem, $oldval, $newval\t# (byte, weak) if $mem == $oldval then $mem <-- $newval"
9426 %}
9427 ins_encode %{
9428 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
9429 Assembler::byte, /*acquire*/ true, /*release*/ true,
9430 /*weak*/ false, $res$$Register);
9431 __ sxtbw($res$$Register, $res$$Register);
9432 %}
9433 ins_pipe(pipe_slow);
9434 %}
9435
9436 // This pattern is generated automatically from cas.m4.
9437 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9438 instruct compareAndExchangeSAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
9439 predicate(needs_acquiring_load_exclusive(n));
9440 match(Set res (CompareAndExchangeS mem (Binary oldval newval)));
9441 ins_cost(VOLATILE_REF_COST);
9442 effect(TEMP_DEF res, KILL cr);
9443 format %{
9444 "cmpxchgs_acq $res = $mem, $oldval, $newval\t# (short, weak) if $mem == $oldval then $mem <-- $newval"
9445 %}
9446 ins_encode %{
9447 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
9448 Assembler::halfword, /*acquire*/ true, /*release*/ true,
9449 /*weak*/ false, $res$$Register);
9450 __ sxthw($res$$Register, $res$$Register);
9451 %}
9452 ins_pipe(pipe_slow);
9453 %}
9454
9455 // This pattern is generated automatically from cas.m4.
9456 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9457 instruct compareAndExchangeIAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
9458 predicate(needs_acquiring_load_exclusive(n));
9459 match(Set res (CompareAndExchangeI mem (Binary oldval newval)));
9460 ins_cost(VOLATILE_REF_COST);
9461 effect(TEMP_DEF res, KILL cr);
9462 format %{
9463 "cmpxchgw_acq $res = $mem, $oldval, $newval\t# (int, weak) if $mem == $oldval then $mem <-- $newval"
9464 %}
9465 ins_encode %{
9466 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
9467 Assembler::word, /*acquire*/ true, /*release*/ true,
9468 /*weak*/ false, $res$$Register);
9469 %}
9470 ins_pipe(pipe_slow);
9471 %}
9472
9473 // This pattern is generated automatically from cas.m4.
9474 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9475 instruct compareAndExchangeLAcq(iRegLNoSp res, indirect mem, iRegL oldval, iRegL newval, rFlagsReg cr) %{
9476 predicate(needs_acquiring_load_exclusive(n));
9477 match(Set res (CompareAndExchangeL mem (Binary oldval newval)));
9478 ins_cost(VOLATILE_REF_COST);
9479 effect(TEMP_DEF res, KILL cr);
9480 format %{
9481 "cmpxchg_acq $res = $mem, $oldval, $newval\t# (long, weak) if $mem == $oldval then $mem <-- $newval"
9482 %}
9483 ins_encode %{
9484 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
9485 Assembler::xword, /*acquire*/ true, /*release*/ true,
9486 /*weak*/ false, $res$$Register);
9487 %}
9488 ins_pipe(pipe_slow);
9489 %}
9490
9491 // This pattern is generated automatically from cas.m4.
9492 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9493 instruct compareAndExchangeNAcq(iRegNNoSp res, indirect mem, iRegN oldval, iRegN newval, rFlagsReg cr) %{
9494 predicate(needs_acquiring_load_exclusive(n));
9495 match(Set res (CompareAndExchangeN mem (Binary oldval newval)));
9496 ins_cost(VOLATILE_REF_COST);
9497 effect(TEMP_DEF res, KILL cr);
9498 format %{
9499 "cmpxchgw_acq $res = $mem, $oldval, $newval\t# (narrow oop, weak) if $mem == $oldval then $mem <-- $newval"
9500 %}
9501 ins_encode %{
9502 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
9503 Assembler::word, /*acquire*/ true, /*release*/ true,
9504 /*weak*/ false, $res$$Register);
9505 %}
9506 ins_pipe(pipe_slow);
9507 %}
9508
9509 // This pattern is generated automatically from cas.m4.
9510 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9511 instruct compareAndExchangePAcq(iRegPNoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
9512 predicate(needs_acquiring_load_exclusive(n) && (n->as_LoadStore()->barrier_data() == 0));
9513 match(Set res (CompareAndExchangeP mem (Binary oldval newval)));
9514 ins_cost(VOLATILE_REF_COST);
9515 effect(TEMP_DEF res, KILL cr);
9516 format %{
9517 "cmpxchg_acq $res = $mem, $oldval, $newval\t# (ptr, weak) if $mem == $oldval then $mem <-- $newval"
9518 %}
9519 ins_encode %{
9520 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
9521 Assembler::xword, /*acquire*/ true, /*release*/ true,
9522 /*weak*/ false, $res$$Register);
9523 %}
9524 ins_pipe(pipe_slow);
9525 %}
9526
9527 // This pattern is generated automatically from cas.m4.
9528 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9529 instruct weakCompareAndSwapB(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
9530
9531 match(Set res (WeakCompareAndSwapB mem (Binary oldval newval)));
9532 ins_cost(2 * VOLATILE_REF_COST);
9533 effect(KILL cr);
9534 format %{
9535 "cmpxchgb $res = $mem, $oldval, $newval\t# (byte, weak) if $mem == $oldval then $mem <-- $newval"
9536 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
9537 %}
9538 ins_encode %{
9539 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
9540 Assembler::byte, /*acquire*/ false, /*release*/ true,
9541 /*weak*/ true, noreg);
9542 __ csetw($res$$Register, Assembler::EQ);
9543 %}
9544 ins_pipe(pipe_slow);
9545 %}
9546
9547 // This pattern is generated automatically from cas.m4.
9548 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9549 instruct weakCompareAndSwapS(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
9550
9551 match(Set res (WeakCompareAndSwapS mem (Binary oldval newval)));
9552 ins_cost(2 * VOLATILE_REF_COST);
9553 effect(KILL cr);
9554 format %{
9555 "cmpxchgs $res = $mem, $oldval, $newval\t# (short, weak) if $mem == $oldval then $mem <-- $newval"
9556 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
9557 %}
9558 ins_encode %{
9559 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
9560 Assembler::halfword, /*acquire*/ false, /*release*/ true,
9561 /*weak*/ true, noreg);
9562 __ csetw($res$$Register, Assembler::EQ);
9563 %}
9564 ins_pipe(pipe_slow);
9565 %}
9566
9567 // This pattern is generated automatically from cas.m4.
9568 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9569 instruct weakCompareAndSwapI(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
9570
9571 match(Set res (WeakCompareAndSwapI mem (Binary oldval newval)));
9572 ins_cost(2 * VOLATILE_REF_COST);
9573 effect(KILL cr);
9574 format %{
9575 "cmpxchgw $res = $mem, $oldval, $newval\t# (int, weak) if $mem == $oldval then $mem <-- $newval"
9576 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
9577 %}
9578 ins_encode %{
9579 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
9580 Assembler::word, /*acquire*/ false, /*release*/ true,
9581 /*weak*/ true, noreg);
9582 __ csetw($res$$Register, Assembler::EQ);
9583 %}
9584 ins_pipe(pipe_slow);
9585 %}
9586
9587 // This pattern is generated automatically from cas.m4.
9588 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9589 instruct weakCompareAndSwapL(iRegINoSp res, indirect mem, iRegL oldval, iRegL newval, rFlagsReg cr) %{
9590
9591 match(Set res (WeakCompareAndSwapL mem (Binary oldval newval)));
9592 ins_cost(2 * VOLATILE_REF_COST);
9593 effect(KILL cr);
9594 format %{
9595 "cmpxchg $res = $mem, $oldval, $newval\t# (long, weak) if $mem == $oldval then $mem <-- $newval"
9596 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
9597 %}
9598 ins_encode %{
9599 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
9600 Assembler::xword, /*acquire*/ false, /*release*/ true,
9601 /*weak*/ true, noreg);
9602 __ csetw($res$$Register, Assembler::EQ);
9603 %}
9604 ins_pipe(pipe_slow);
9605 %}
9606
9607 // This pattern is generated automatically from cas.m4.
9608 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9609 instruct weakCompareAndSwapN(iRegINoSp res, indirect mem, iRegN oldval, iRegN newval, rFlagsReg cr) %{
9610
9611 match(Set res (WeakCompareAndSwapN mem (Binary oldval newval)));
9612 ins_cost(2 * VOLATILE_REF_COST);
9613 effect(KILL cr);
9614 format %{
9615 "cmpxchgw $res = $mem, $oldval, $newval\t# (narrow oop, weak) if $mem == $oldval then $mem <-- $newval"
9616 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
9617 %}
9618 ins_encode %{
9619 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
9620 Assembler::word, /*acquire*/ false, /*release*/ true,
9621 /*weak*/ true, noreg);
9622 __ csetw($res$$Register, Assembler::EQ);
9623 %}
9624 ins_pipe(pipe_slow);
9625 %}
9626
9627 // This pattern is generated automatically from cas.m4.
9628 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9629 instruct weakCompareAndSwapP(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
9630 predicate(n->as_LoadStore()->barrier_data() == 0);
9631 match(Set res (WeakCompareAndSwapP mem (Binary oldval newval)));
9632 ins_cost(2 * VOLATILE_REF_COST);
9633 effect(KILL cr);
9634 format %{
9635 "cmpxchg $res = $mem, $oldval, $newval\t# (ptr, weak) if $mem == $oldval then $mem <-- $newval"
9636 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
9637 %}
9638 ins_encode %{
9639 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
9640 Assembler::xword, /*acquire*/ false, /*release*/ true,
9641 /*weak*/ true, noreg);
9642 __ csetw($res$$Register, Assembler::EQ);
9643 %}
9644 ins_pipe(pipe_slow);
9645 %}
9646
9647 // This pattern is generated automatically from cas.m4.
9648 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9649 instruct weakCompareAndSwapBAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
9650 predicate(needs_acquiring_load_exclusive(n));
9651 match(Set res (WeakCompareAndSwapB mem (Binary oldval newval)));
9652 ins_cost(VOLATILE_REF_COST);
9653 effect(KILL cr);
9654 format %{
9655 "cmpxchgb_acq $res = $mem, $oldval, $newval\t# (byte, weak) if $mem == $oldval then $mem <-- $newval"
9656 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
9657 %}
9658 ins_encode %{
9659 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
9660 Assembler::byte, /*acquire*/ true, /*release*/ true,
9661 /*weak*/ true, noreg);
9662 __ csetw($res$$Register, Assembler::EQ);
9663 %}
9664 ins_pipe(pipe_slow);
9665 %}
9666
9667 // This pattern is generated automatically from cas.m4.
9668 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9669 instruct weakCompareAndSwapSAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
9670 predicate(needs_acquiring_load_exclusive(n));
9671 match(Set res (WeakCompareAndSwapS mem (Binary oldval newval)));
9672 ins_cost(VOLATILE_REF_COST);
9673 effect(KILL cr);
9674 format %{
9675 "cmpxchgs_acq $res = $mem, $oldval, $newval\t# (short, weak) if $mem == $oldval then $mem <-- $newval"
9676 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
9677 %}
9678 ins_encode %{
9679 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
9680 Assembler::halfword, /*acquire*/ true, /*release*/ true,
9681 /*weak*/ true, noreg);
9682 __ csetw($res$$Register, Assembler::EQ);
9683 %}
9684 ins_pipe(pipe_slow);
9685 %}
9686
9687 // This pattern is generated automatically from cas.m4.
9688 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9689 instruct weakCompareAndSwapIAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
9690 predicate(needs_acquiring_load_exclusive(n));
9691 match(Set res (WeakCompareAndSwapI mem (Binary oldval newval)));
9692 ins_cost(VOLATILE_REF_COST);
9693 effect(KILL cr);
9694 format %{
9695 "cmpxchgw_acq $res = $mem, $oldval, $newval\t# (int, weak) if $mem == $oldval then $mem <-- $newval"
9696 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
9697 %}
9698 ins_encode %{
9699 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
9700 Assembler::word, /*acquire*/ true, /*release*/ true,
9701 /*weak*/ true, noreg);
9702 __ csetw($res$$Register, Assembler::EQ);
9703 %}
9704 ins_pipe(pipe_slow);
9705 %}
9706
9707 // This pattern is generated automatically from cas.m4.
9708 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9709 instruct weakCompareAndSwapLAcq(iRegINoSp res, indirect mem, iRegL oldval, iRegL newval, rFlagsReg cr) %{
9710 predicate(needs_acquiring_load_exclusive(n));
9711 match(Set res (WeakCompareAndSwapL mem (Binary oldval newval)));
9712 ins_cost(VOLATILE_REF_COST);
9713 effect(KILL cr);
9714 format %{
9715 "cmpxchg_acq $res = $mem, $oldval, $newval\t# (long, weak) if $mem == $oldval then $mem <-- $newval"
9716 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
9717 %}
9718 ins_encode %{
9719 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
9720 Assembler::xword, /*acquire*/ true, /*release*/ true,
9721 /*weak*/ true, noreg);
9722 __ csetw($res$$Register, Assembler::EQ);
9723 %}
9724 ins_pipe(pipe_slow);
9725 %}
9726
9727 // This pattern is generated automatically from cas.m4.
9728 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9729 instruct weakCompareAndSwapNAcq(iRegINoSp res, indirect mem, iRegN oldval, iRegN newval, rFlagsReg cr) %{
9730 predicate(needs_acquiring_load_exclusive(n));
9731 match(Set res (WeakCompareAndSwapN mem (Binary oldval newval)));
9732 ins_cost(VOLATILE_REF_COST);
9733 effect(KILL cr);
9734 format %{
9735 "cmpxchgw_acq $res = $mem, $oldval, $newval\t# (narrow oop, weak) if $mem == $oldval then $mem <-- $newval"
9736 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
9737 %}
9738 ins_encode %{
9739 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
9740 Assembler::word, /*acquire*/ true, /*release*/ true,
9741 /*weak*/ true, noreg);
9742 __ csetw($res$$Register, Assembler::EQ);
9743 %}
9744 ins_pipe(pipe_slow);
9745 %}
9746
9747 // This pattern is generated automatically from cas.m4.
9748 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9749 instruct weakCompareAndSwapPAcq(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
9750 predicate(needs_acquiring_load_exclusive(n) && (n->as_LoadStore()->barrier_data() == 0));
9751 match(Set res (WeakCompareAndSwapP mem (Binary oldval newval)));
9752 ins_cost(VOLATILE_REF_COST);
9753 effect(KILL cr);
9754 format %{
9755 "cmpxchg_acq $res = $mem, $oldval, $newval\t# (ptr, weak) if $mem == $oldval then $mem <-- $newval"
9756 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
9757 %}
9758 ins_encode %{
9759 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
9760 Assembler::xword, /*acquire*/ true, /*release*/ true,
9761 /*weak*/ true, noreg);
9762 __ csetw($res$$Register, Assembler::EQ);
9763 %}
9764 ins_pipe(pipe_slow);
9765 %}
9766
9767 // END This section of the file is automatically generated. Do not edit --------------
9768 // ---------------------------------------------------------------------
9769
9770 instruct get_and_setI(indirect mem, iRegI newv, iRegINoSp prev) %{
9771 match(Set prev (GetAndSetI mem newv));
9772 ins_cost(2 * VOLATILE_REF_COST);
9773 format %{ "atomic_xchgw $prev, $newv, [$mem]" %}
9774 ins_encode %{
9775 __ atomic_xchgw($prev$$Register, $newv$$Register, as_Register($mem$$base));
9776 %}
9777 ins_pipe(pipe_serial);
9778 %}
9779
9780 instruct get_and_setL(indirect mem, iRegL newv, iRegLNoSp prev) %{
9781 match(Set prev (GetAndSetL mem newv));
9782 ins_cost(2 * VOLATILE_REF_COST);
9783 format %{ "atomic_xchg $prev, $newv, [$mem]" %}
9784 ins_encode %{
9785 __ atomic_xchg($prev$$Register, $newv$$Register, as_Register($mem$$base));
9786 %}
9787 ins_pipe(pipe_serial);
9788 %}
9789
9790 instruct get_and_setN(indirect mem, iRegN newv, iRegINoSp prev) %{
9791 match(Set prev (GetAndSetN mem newv));
9792 ins_cost(2 * VOLATILE_REF_COST);
9793 format %{ "atomic_xchgw $prev, $newv, [$mem]" %}
9794 ins_encode %{
9795 __ atomic_xchgw($prev$$Register, $newv$$Register, as_Register($mem$$base));
9796 %}
9797 ins_pipe(pipe_serial);
9798 %}
9799
9800 instruct get_and_setP(indirect mem, iRegP newv, iRegPNoSp prev) %{
9801 predicate(n->as_LoadStore()->barrier_data() == 0);
9802 match(Set prev (GetAndSetP mem newv));
9803 ins_cost(2 * VOLATILE_REF_COST);
9804 format %{ "atomic_xchg $prev, $newv, [$mem]" %}
9805 ins_encode %{
9806 __ atomic_xchg($prev$$Register, $newv$$Register, as_Register($mem$$base));
9807 %}
9808 ins_pipe(pipe_serial);
9809 %}
9810
9811 instruct get_and_setIAcq(indirect mem, iRegI newv, iRegINoSp prev) %{
9812 predicate(needs_acquiring_load_exclusive(n));
9813 match(Set prev (GetAndSetI mem newv));
9814 ins_cost(VOLATILE_REF_COST);
9815 format %{ "atomic_xchgw_acq $prev, $newv, [$mem]" %}
9816 ins_encode %{
9817 __ atomic_xchgalw($prev$$Register, $newv$$Register, as_Register($mem$$base));
9818 %}
9819 ins_pipe(pipe_serial);
9820 %}
9821
9822 instruct get_and_setLAcq(indirect mem, iRegL newv, iRegLNoSp prev) %{
9823 predicate(needs_acquiring_load_exclusive(n));
9824 match(Set prev (GetAndSetL mem newv));
9825 ins_cost(VOLATILE_REF_COST);
9826 format %{ "atomic_xchg_acq $prev, $newv, [$mem]" %}
9827 ins_encode %{
9828 __ atomic_xchgal($prev$$Register, $newv$$Register, as_Register($mem$$base));
9829 %}
9830 ins_pipe(pipe_serial);
9831 %}
9832
9833 instruct get_and_setNAcq(indirect mem, iRegN newv, iRegINoSp prev) %{
9834 predicate(needs_acquiring_load_exclusive(n));
9835 match(Set prev (GetAndSetN mem newv));
9836 ins_cost(VOLATILE_REF_COST);
9837 format %{ "atomic_xchgw_acq $prev, $newv, [$mem]" %}
9838 ins_encode %{
9839 __ atomic_xchgalw($prev$$Register, $newv$$Register, as_Register($mem$$base));
9840 %}
9841 ins_pipe(pipe_serial);
9842 %}
9843
9844 instruct get_and_setPAcq(indirect mem, iRegP newv, iRegPNoSp prev) %{
9845 predicate(needs_acquiring_load_exclusive(n) && (n->as_LoadStore()->barrier_data() == 0));
9846 match(Set prev (GetAndSetP mem newv));
9847 ins_cost(VOLATILE_REF_COST);
9848 format %{ "atomic_xchg_acq $prev, $newv, [$mem]" %}
9849 ins_encode %{
9850 __ atomic_xchgal($prev$$Register, $newv$$Register, as_Register($mem$$base));
9851 %}
9852 ins_pipe(pipe_serial);
9853 %}
9854
9855
9856 instruct get_and_addL(indirect mem, iRegLNoSp newval, iRegL incr) %{
9857 match(Set newval (GetAndAddL mem incr));
9858 ins_cost(2 * VOLATILE_REF_COST + 1);
9859 format %{ "get_and_addL $newval, [$mem], $incr" %}
9860 ins_encode %{
9861 __ atomic_add($newval$$Register, $incr$$Register, as_Register($mem$$base));
9862 %}
9863 ins_pipe(pipe_serial);
9864 %}
9865
9866 instruct get_and_addL_no_res(indirect mem, Universe dummy, iRegL incr) %{
9867 predicate(n->as_LoadStore()->result_not_used());
9868 match(Set dummy (GetAndAddL mem incr));
9869 ins_cost(2 * VOLATILE_REF_COST);
9870 format %{ "get_and_addL [$mem], $incr" %}
9871 ins_encode %{
9872 __ atomic_add(noreg, $incr$$Register, as_Register($mem$$base));
9873 %}
9874 ins_pipe(pipe_serial);
9875 %}
9876
9877 instruct get_and_addLi(indirect mem, iRegLNoSp newval, immLAddSub incr) %{
9878 match(Set newval (GetAndAddL mem incr));
9879 ins_cost(2 * VOLATILE_REF_COST + 1);
9880 format %{ "get_and_addL $newval, [$mem], $incr" %}
9881 ins_encode %{
9882 __ atomic_add($newval$$Register, $incr$$constant, as_Register($mem$$base));
9883 %}
9884 ins_pipe(pipe_serial);
9885 %}
9886
9887 instruct get_and_addLi_no_res(indirect mem, Universe dummy, immLAddSub incr) %{
9888 predicate(n->as_LoadStore()->result_not_used());
9889 match(Set dummy (GetAndAddL mem incr));
9890 ins_cost(2 * VOLATILE_REF_COST);
9891 format %{ "get_and_addL [$mem], $incr" %}
9892 ins_encode %{
9893 __ atomic_add(noreg, $incr$$constant, as_Register($mem$$base));
9894 %}
9895 ins_pipe(pipe_serial);
9896 %}
9897
9898 instruct get_and_addI(indirect mem, iRegINoSp newval, iRegIorL2I incr) %{
9899 match(Set newval (GetAndAddI mem incr));
9900 ins_cost(2 * VOLATILE_REF_COST + 1);
9901 format %{ "get_and_addI $newval, [$mem], $incr" %}
9902 ins_encode %{
9903 __ atomic_addw($newval$$Register, $incr$$Register, as_Register($mem$$base));
9904 %}
9905 ins_pipe(pipe_serial);
9906 %}
9907
9908 instruct get_and_addI_no_res(indirect mem, Universe dummy, iRegIorL2I incr) %{
9909 predicate(n->as_LoadStore()->result_not_used());
9910 match(Set dummy (GetAndAddI mem incr));
9911 ins_cost(2 * VOLATILE_REF_COST);
9912 format %{ "get_and_addI [$mem], $incr" %}
9913 ins_encode %{
9914 __ atomic_addw(noreg, $incr$$Register, as_Register($mem$$base));
9915 %}
9916 ins_pipe(pipe_serial);
9917 %}
9918
9919 instruct get_and_addIi(indirect mem, iRegINoSp newval, immIAddSub incr) %{
9920 match(Set newval (GetAndAddI mem incr));
9921 ins_cost(2 * VOLATILE_REF_COST + 1);
9922 format %{ "get_and_addI $newval, [$mem], $incr" %}
9923 ins_encode %{
9924 __ atomic_addw($newval$$Register, $incr$$constant, as_Register($mem$$base));
9925 %}
9926 ins_pipe(pipe_serial);
9927 %}
9928
9929 instruct get_and_addIi_no_res(indirect mem, Universe dummy, immIAddSub incr) %{
9930 predicate(n->as_LoadStore()->result_not_used());
9931 match(Set dummy (GetAndAddI mem incr));
9932 ins_cost(2 * VOLATILE_REF_COST);
9933 format %{ "get_and_addI [$mem], $incr" %}
9934 ins_encode %{
9935 __ atomic_addw(noreg, $incr$$constant, as_Register($mem$$base));
9936 %}
9937 ins_pipe(pipe_serial);
9938 %}
9939
9940 instruct get_and_addLAcq(indirect mem, iRegLNoSp newval, iRegL incr) %{
9941 predicate(needs_acquiring_load_exclusive(n));
9942 match(Set newval (GetAndAddL mem incr));
9943 ins_cost(VOLATILE_REF_COST + 1);
9944 format %{ "get_and_addL_acq $newval, [$mem], $incr" %}
9945 ins_encode %{
9946 __ atomic_addal($newval$$Register, $incr$$Register, as_Register($mem$$base));
9947 %}
9948 ins_pipe(pipe_serial);
9949 %}
9950
9951 instruct get_and_addL_no_resAcq(indirect mem, Universe dummy, iRegL incr) %{
9952 predicate(n->as_LoadStore()->result_not_used() && needs_acquiring_load_exclusive(n));
9953 match(Set dummy (GetAndAddL mem incr));
9954 ins_cost(VOLATILE_REF_COST);
9955 format %{ "get_and_addL_acq [$mem], $incr" %}
9956 ins_encode %{
9957 __ atomic_addal(noreg, $incr$$Register, as_Register($mem$$base));
9958 %}
9959 ins_pipe(pipe_serial);
9960 %}
9961
9962 instruct get_and_addLiAcq(indirect mem, iRegLNoSp newval, immLAddSub incr) %{
9963 predicate(needs_acquiring_load_exclusive(n));
9964 match(Set newval (GetAndAddL mem incr));
9965 ins_cost(VOLATILE_REF_COST + 1);
9966 format %{ "get_and_addL_acq $newval, [$mem], $incr" %}
9967 ins_encode %{
9968 __ atomic_addal($newval$$Register, $incr$$constant, as_Register($mem$$base));
9969 %}
9970 ins_pipe(pipe_serial);
9971 %}
9972
9973 instruct get_and_addLi_no_resAcq(indirect mem, Universe dummy, immLAddSub incr) %{
9974 predicate(n->as_LoadStore()->result_not_used() && needs_acquiring_load_exclusive(n));
9975 match(Set dummy (GetAndAddL mem incr));
9976 ins_cost(VOLATILE_REF_COST);
9977 format %{ "get_and_addL_acq [$mem], $incr" %}
9978 ins_encode %{
9979 __ atomic_addal(noreg, $incr$$constant, as_Register($mem$$base));
9980 %}
9981 ins_pipe(pipe_serial);
9982 %}
9983
9984 instruct get_and_addIAcq(indirect mem, iRegINoSp newval, iRegIorL2I incr) %{
9985 predicate(needs_acquiring_load_exclusive(n));
9986 match(Set newval (GetAndAddI mem incr));
9987 ins_cost(VOLATILE_REF_COST + 1);
9988 format %{ "get_and_addI_acq $newval, [$mem], $incr" %}
9989 ins_encode %{
9990 __ atomic_addalw($newval$$Register, $incr$$Register, as_Register($mem$$base));
9991 %}
9992 ins_pipe(pipe_serial);
9993 %}
9994
9995 instruct get_and_addI_no_resAcq(indirect mem, Universe dummy, iRegIorL2I incr) %{
9996 predicate(n->as_LoadStore()->result_not_used() && needs_acquiring_load_exclusive(n));
9997 match(Set dummy (GetAndAddI mem incr));
9998 ins_cost(VOLATILE_REF_COST);
9999 format %{ "get_and_addI_acq [$mem], $incr" %}
10000 ins_encode %{
10001 __ atomic_addalw(noreg, $incr$$Register, as_Register($mem$$base));
10002 %}
10003 ins_pipe(pipe_serial);
10004 %}
10005
10006 instruct get_and_addIiAcq(indirect mem, iRegINoSp newval, immIAddSub incr) %{
10007 predicate(needs_acquiring_load_exclusive(n));
10008 match(Set newval (GetAndAddI mem incr));
10009 ins_cost(VOLATILE_REF_COST + 1);
10010 format %{ "get_and_addI_acq $newval, [$mem], $incr" %}
10011 ins_encode %{
10012 __ atomic_addalw($newval$$Register, $incr$$constant, as_Register($mem$$base));
10013 %}
10014 ins_pipe(pipe_serial);
10015 %}
10016
10017 instruct get_and_addIi_no_resAcq(indirect mem, Universe dummy, immIAddSub incr) %{
10018 predicate(n->as_LoadStore()->result_not_used() && needs_acquiring_load_exclusive(n));
10019 match(Set dummy (GetAndAddI mem incr));
10020 ins_cost(VOLATILE_REF_COST);
10021 format %{ "get_and_addI_acq [$mem], $incr" %}
10022 ins_encode %{
10023 __ atomic_addalw(noreg, $incr$$constant, as_Register($mem$$base));
10024 %}
10025 ins_pipe(pipe_serial);
10026 %}
10027
10028 // Manifest a CmpL result in an integer register.
10029 // (src1 < src2) ? -1 : ((src1 > src2) ? 1 : 0)
10030 instruct cmpL3_reg_reg(iRegINoSp dst, iRegL src1, iRegL src2, rFlagsReg flags)
10031 %{
10032 match(Set dst (CmpL3 src1 src2));
10033 effect(KILL flags);
10034
10035 ins_cost(INSN_COST * 6);
10036 format %{
10037 "cmp $src1, $src2"
10038 "csetw $dst, ne"
10039 "cnegw $dst, lt"
10040 %}
10041 // format %{ "CmpL3 $dst, $src1, $src2" %}
10042 ins_encode %{
10043 __ cmp($src1$$Register, $src2$$Register);
10044 __ csetw($dst$$Register, Assembler::NE);
10045 __ cnegw($dst$$Register, $dst$$Register, Assembler::LT);
10046 %}
10047
10048 ins_pipe(pipe_class_default);
10049 %}
10050
10051 instruct cmpL3_reg_imm(iRegINoSp dst, iRegL src1, immLAddSub src2, rFlagsReg flags)
10052 %{
10053 match(Set dst (CmpL3 src1 src2));
10054 effect(KILL flags);
10055
10056 ins_cost(INSN_COST * 6);
10057 format %{
10058 "cmp $src1, $src2"
10059 "csetw $dst, ne"
10060 "cnegw $dst, lt"
10061 %}
10062 ins_encode %{
10063 int32_t con = (int32_t)$src2$$constant;
10064 if (con < 0) {
10065 __ adds(zr, $src1$$Register, -con);
10066 } else {
10067 __ subs(zr, $src1$$Register, con);
10068 }
10069 __ csetw($dst$$Register, Assembler::NE);
10070 __ cnegw($dst$$Register, $dst$$Register, Assembler::LT);
10071 %}
10072
10073 ins_pipe(pipe_class_default);
10074 %}
10075
10076 // ============================================================================
10077 // Conditional Move Instructions
10078
10079 // n.b. we have identical rules for both a signed compare op (cmpOp)
10080 // and an unsigned compare op (cmpOpU). it would be nice if we could
10081 // define an op class which merged both inputs and use it to type the
10082 // argument to a single rule. unfortunatelyt his fails because the
10083 // opclass does not live up to the COND_INTER interface of its
10084 // component operands. When the generic code tries to negate the
10085 // operand it ends up running the generci Machoper::negate method
10086 // which throws a ShouldNotHappen. So, we have to provide two flavours
10087 // of each rule, one for a cmpOp and a second for a cmpOpU (sigh).
10088
10089 instruct cmovI_reg_reg(cmpOp cmp, rFlagsReg cr, iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10090 match(Set dst (CMoveI (Binary cmp cr) (Binary src1 src2)));
10091
10092 ins_cost(INSN_COST * 2);
10093 format %{ "cselw $dst, $src2, $src1 $cmp\t# signed, int" %}
10094
10095 ins_encode %{
10096 __ cselw(as_Register($dst$$reg),
10097 as_Register($src2$$reg),
10098 as_Register($src1$$reg),
10099 (Assembler::Condition)$cmp$$cmpcode);
10100 %}
10101
10102 ins_pipe(icond_reg_reg);
10103 %}
10104
10105 instruct cmovUI_reg_reg(cmpOpU cmp, rFlagsRegU cr, iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10106 match(Set dst (CMoveI (Binary cmp cr) (Binary src1 src2)));
10107
10108 ins_cost(INSN_COST * 2);
10109 format %{ "cselw $dst, $src2, $src1 $cmp\t# unsigned, int" %}
10110
10111 ins_encode %{
10112 __ cselw(as_Register($dst$$reg),
10113 as_Register($src2$$reg),
10114 as_Register($src1$$reg),
10115 (Assembler::Condition)$cmp$$cmpcode);
10116 %}
10117
10118 ins_pipe(icond_reg_reg);
10119 %}
10120
10121 // special cases where one arg is zero
10122
10123 // n.b. this is selected in preference to the rule above because it
10124 // avoids loading constant 0 into a source register
10125
10126 // TODO
10127 // we ought only to be able to cull one of these variants as the ideal
10128 // transforms ought always to order the zero consistently (to left/right?)
10129
10130 instruct cmovI_zero_reg(cmpOp cmp, rFlagsReg cr, iRegINoSp dst, immI0 zero, iRegIorL2I src) %{
10131 match(Set dst (CMoveI (Binary cmp cr) (Binary zero src)));
10132
10133 ins_cost(INSN_COST * 2);
10134 format %{ "cselw $dst, $src, zr $cmp\t# signed, int" %}
10135
10136 ins_encode %{
10137 __ cselw(as_Register($dst$$reg),
10138 as_Register($src$$reg),
10139 zr,
10140 (Assembler::Condition)$cmp$$cmpcode);
10141 %}
10142
10143 ins_pipe(icond_reg);
10144 %}
10145
10146 instruct cmovUI_zero_reg(cmpOpU cmp, rFlagsRegU cr, iRegINoSp dst, immI0 zero, iRegIorL2I src) %{
10147 match(Set dst (CMoveI (Binary cmp cr) (Binary zero src)));
10148
10149 ins_cost(INSN_COST * 2);
10150 format %{ "cselw $dst, $src, zr $cmp\t# unsigned, int" %}
10151
10152 ins_encode %{
10153 __ cselw(as_Register($dst$$reg),
10154 as_Register($src$$reg),
10155 zr,
10156 (Assembler::Condition)$cmp$$cmpcode);
10157 %}
10158
10159 ins_pipe(icond_reg);
10160 %}
10161
10162 instruct cmovI_reg_zero(cmpOp cmp, rFlagsReg cr, iRegINoSp dst, iRegIorL2I src, immI0 zero) %{
10163 match(Set dst (CMoveI (Binary cmp cr) (Binary src zero)));
10164
10165 ins_cost(INSN_COST * 2);
10166 format %{ "cselw $dst, zr, $src $cmp\t# signed, int" %}
10167
10168 ins_encode %{
10169 __ cselw(as_Register($dst$$reg),
10170 zr,
10171 as_Register($src$$reg),
10172 (Assembler::Condition)$cmp$$cmpcode);
10173 %}
10174
10175 ins_pipe(icond_reg);
10176 %}
10177
10178 instruct cmovUI_reg_zero(cmpOpU cmp, rFlagsRegU cr, iRegINoSp dst, iRegIorL2I src, immI0 zero) %{
10179 match(Set dst (CMoveI (Binary cmp cr) (Binary src zero)));
10180
10181 ins_cost(INSN_COST * 2);
10182 format %{ "cselw $dst, zr, $src $cmp\t# unsigned, int" %}
10183
10184 ins_encode %{
10185 __ cselw(as_Register($dst$$reg),
10186 zr,
10187 as_Register($src$$reg),
10188 (Assembler::Condition)$cmp$$cmpcode);
10189 %}
10190
10191 ins_pipe(icond_reg);
10192 %}
10193
10194 // special case for creating a boolean 0 or 1
10195
10196 // n.b. this is selected in preference to the rule above because it
10197 // avoids loading constants 0 and 1 into a source register
10198
10199 instruct cmovI_reg_zero_one(cmpOp cmp, rFlagsReg cr, iRegINoSp dst, immI0 zero, immI_1 one) %{
10200 match(Set dst (CMoveI (Binary cmp cr) (Binary one zero)));
10201
10202 ins_cost(INSN_COST * 2);
10203 format %{ "csincw $dst, zr, zr $cmp\t# signed, int" %}
10204
10205 ins_encode %{
10206 // equivalently
10207 // cset(as_Register($dst$$reg),
10208 // negate_condition((Assembler::Condition)$cmp$$cmpcode));
10209 __ csincw(as_Register($dst$$reg),
10210 zr,
10211 zr,
10212 (Assembler::Condition)$cmp$$cmpcode);
10213 %}
10214
10215 ins_pipe(icond_none);
10216 %}
10217
10218 instruct cmovUI_reg_zero_one(cmpOpU cmp, rFlagsRegU cr, iRegINoSp dst, immI0 zero, immI_1 one) %{
10219 match(Set dst (CMoveI (Binary cmp cr) (Binary one zero)));
10220
10221 ins_cost(INSN_COST * 2);
10222 format %{ "csincw $dst, zr, zr $cmp\t# unsigned, int" %}
10223
10224 ins_encode %{
10225 // equivalently
10226 // cset(as_Register($dst$$reg),
10227 // negate_condition((Assembler::Condition)$cmp$$cmpcode));
10228 __ csincw(as_Register($dst$$reg),
10229 zr,
10230 zr,
10231 (Assembler::Condition)$cmp$$cmpcode);
10232 %}
10233
10234 ins_pipe(icond_none);
10235 %}
10236
10237 instruct cmovL_reg_reg(cmpOp cmp, rFlagsReg cr, iRegLNoSp dst, iRegL src1, iRegL src2) %{
10238 match(Set dst (CMoveL (Binary cmp cr) (Binary src1 src2)));
10239
10240 ins_cost(INSN_COST * 2);
10241 format %{ "csel $dst, $src2, $src1 $cmp\t# signed, long" %}
10242
10243 ins_encode %{
10244 __ csel(as_Register($dst$$reg),
10245 as_Register($src2$$reg),
10246 as_Register($src1$$reg),
10247 (Assembler::Condition)$cmp$$cmpcode);
10248 %}
10249
10250 ins_pipe(icond_reg_reg);
10251 %}
10252
10253 instruct cmovUL_reg_reg(cmpOpU cmp, rFlagsRegU cr, iRegLNoSp dst, iRegL src1, iRegL src2) %{
10254 match(Set dst (CMoveL (Binary cmp cr) (Binary src1 src2)));
10255
10256 ins_cost(INSN_COST * 2);
10257 format %{ "csel $dst, $src2, $src1 $cmp\t# unsigned, long" %}
10258
10259 ins_encode %{
10260 __ csel(as_Register($dst$$reg),
10261 as_Register($src2$$reg),
10262 as_Register($src1$$reg),
10263 (Assembler::Condition)$cmp$$cmpcode);
10264 %}
10265
10266 ins_pipe(icond_reg_reg);
10267 %}
10268
10269 // special cases where one arg is zero
10270
10271 instruct cmovL_reg_zero(cmpOp cmp, rFlagsReg cr, iRegLNoSp dst, iRegL src, immL0 zero) %{
10272 match(Set dst (CMoveL (Binary cmp cr) (Binary src zero)));
10273
10274 ins_cost(INSN_COST * 2);
10275 format %{ "csel $dst, zr, $src $cmp\t# signed, long" %}
10276
10277 ins_encode %{
10278 __ csel(as_Register($dst$$reg),
10279 zr,
10280 as_Register($src$$reg),
10281 (Assembler::Condition)$cmp$$cmpcode);
10282 %}
10283
10284 ins_pipe(icond_reg);
10285 %}
10286
10287 instruct cmovUL_reg_zero(cmpOpU cmp, rFlagsRegU cr, iRegLNoSp dst, iRegL src, immL0 zero) %{
10288 match(Set dst (CMoveL (Binary cmp cr) (Binary src zero)));
10289
10290 ins_cost(INSN_COST * 2);
10291 format %{ "csel $dst, zr, $src $cmp\t# unsigned, long" %}
10292
10293 ins_encode %{
10294 __ csel(as_Register($dst$$reg),
10295 zr,
10296 as_Register($src$$reg),
10297 (Assembler::Condition)$cmp$$cmpcode);
10298 %}
10299
10300 ins_pipe(icond_reg);
10301 %}
10302
10303 instruct cmovL_zero_reg(cmpOp cmp, rFlagsReg cr, iRegLNoSp dst, immL0 zero, iRegL src) %{
10304 match(Set dst (CMoveL (Binary cmp cr) (Binary zero src)));
10305
10306 ins_cost(INSN_COST * 2);
10307 format %{ "csel $dst, $src, zr $cmp\t# signed, long" %}
10308
10309 ins_encode %{
10310 __ csel(as_Register($dst$$reg),
10311 as_Register($src$$reg),
10312 zr,
10313 (Assembler::Condition)$cmp$$cmpcode);
10314 %}
10315
10316 ins_pipe(icond_reg);
10317 %}
10318
10319 instruct cmovUL_zero_reg(cmpOpU cmp, rFlagsRegU cr, iRegLNoSp dst, immL0 zero, iRegL src) %{
10320 match(Set dst (CMoveL (Binary cmp cr) (Binary zero src)));
10321
10322 ins_cost(INSN_COST * 2);
10323 format %{ "csel $dst, $src, zr $cmp\t# unsigned, long" %}
10324
10325 ins_encode %{
10326 __ csel(as_Register($dst$$reg),
10327 as_Register($src$$reg),
10328 zr,
10329 (Assembler::Condition)$cmp$$cmpcode);
10330 %}
10331
10332 ins_pipe(icond_reg);
10333 %}
10334
10335 instruct cmovP_reg_reg(cmpOp cmp, rFlagsReg cr, iRegPNoSp dst, iRegP src1, iRegP src2) %{
10336 match(Set dst (CMoveP (Binary cmp cr) (Binary src1 src2)));
10337
10338 ins_cost(INSN_COST * 2);
10339 format %{ "csel $dst, $src2, $src1 $cmp\t# signed, ptr" %}
10340
10341 ins_encode %{
10342 __ csel(as_Register($dst$$reg),
10343 as_Register($src2$$reg),
10344 as_Register($src1$$reg),
10345 (Assembler::Condition)$cmp$$cmpcode);
10346 %}
10347
10348 ins_pipe(icond_reg_reg);
10349 %}
10350
10351 instruct cmovUP_reg_reg(cmpOpU cmp, rFlagsRegU cr, iRegPNoSp dst, iRegP src1, iRegP src2) %{
10352 match(Set dst (CMoveP (Binary cmp cr) (Binary src1 src2)));
10353
10354 ins_cost(INSN_COST * 2);
10355 format %{ "csel $dst, $src2, $src1 $cmp\t# unsigned, ptr" %}
10356
10357 ins_encode %{
10358 __ csel(as_Register($dst$$reg),
10359 as_Register($src2$$reg),
10360 as_Register($src1$$reg),
10361 (Assembler::Condition)$cmp$$cmpcode);
10362 %}
10363
10364 ins_pipe(icond_reg_reg);
10365 %}
10366
10367 // special cases where one arg is zero
10368
10369 instruct cmovP_reg_zero(cmpOp cmp, rFlagsReg cr, iRegPNoSp dst, iRegP src, immP0 zero) %{
10370 match(Set dst (CMoveP (Binary cmp cr) (Binary src zero)));
10371
10372 ins_cost(INSN_COST * 2);
10373 format %{ "csel $dst, zr, $src $cmp\t# signed, ptr" %}
10374
10375 ins_encode %{
10376 __ csel(as_Register($dst$$reg),
10377 zr,
10378 as_Register($src$$reg),
10379 (Assembler::Condition)$cmp$$cmpcode);
10380 %}
10381
10382 ins_pipe(icond_reg);
10383 %}
10384
10385 instruct cmovUP_reg_zero(cmpOpU cmp, rFlagsRegU cr, iRegPNoSp dst, iRegP src, immP0 zero) %{
10386 match(Set dst (CMoveP (Binary cmp cr) (Binary src zero)));
10387
10388 ins_cost(INSN_COST * 2);
10389 format %{ "csel $dst, zr, $src $cmp\t# unsigned, ptr" %}
10390
10391 ins_encode %{
10392 __ csel(as_Register($dst$$reg),
10393 zr,
10394 as_Register($src$$reg),
10395 (Assembler::Condition)$cmp$$cmpcode);
10396 %}
10397
10398 ins_pipe(icond_reg);
10399 %}
10400
10401 instruct cmovP_zero_reg(cmpOp cmp, rFlagsReg cr, iRegPNoSp dst, immP0 zero, iRegP src) %{
10402 match(Set dst (CMoveP (Binary cmp cr) (Binary zero src)));
10403
10404 ins_cost(INSN_COST * 2);
10405 format %{ "csel $dst, $src, zr $cmp\t# signed, ptr" %}
10406
10407 ins_encode %{
10408 __ csel(as_Register($dst$$reg),
10409 as_Register($src$$reg),
10410 zr,
10411 (Assembler::Condition)$cmp$$cmpcode);
10412 %}
10413
10414 ins_pipe(icond_reg);
10415 %}
10416
10417 instruct cmovUP_zero_reg(cmpOpU cmp, rFlagsRegU cr, iRegPNoSp dst, immP0 zero, iRegP src) %{
10418 match(Set dst (CMoveP (Binary cmp cr) (Binary zero src)));
10419
10420 ins_cost(INSN_COST * 2);
10421 format %{ "csel $dst, $src, zr $cmp\t# unsigned, ptr" %}
10422
10423 ins_encode %{
10424 __ csel(as_Register($dst$$reg),
10425 as_Register($src$$reg),
10426 zr,
10427 (Assembler::Condition)$cmp$$cmpcode);
10428 %}
10429
10430 ins_pipe(icond_reg);
10431 %}
10432
10433 instruct cmovN_reg_reg(cmpOp cmp, rFlagsReg cr, iRegNNoSp dst, iRegN src1, iRegN src2) %{
10434 match(Set dst (CMoveN (Binary cmp cr) (Binary src1 src2)));
10435
10436 ins_cost(INSN_COST * 2);
10437 format %{ "cselw $dst, $src2, $src1 $cmp\t# signed, compressed ptr" %}
10438
10439 ins_encode %{
10440 __ cselw(as_Register($dst$$reg),
10441 as_Register($src2$$reg),
10442 as_Register($src1$$reg),
10443 (Assembler::Condition)$cmp$$cmpcode);
10444 %}
10445
10446 ins_pipe(icond_reg_reg);
10447 %}
10448
10449 instruct cmovUN_reg_reg(cmpOpU cmp, rFlagsRegU cr, iRegNNoSp dst, iRegN src1, iRegN src2) %{
10450 match(Set dst (CMoveN (Binary cmp cr) (Binary src1 src2)));
10451
10452 ins_cost(INSN_COST * 2);
10453 format %{ "cselw $dst, $src2, $src1 $cmp\t# signed, compressed ptr" %}
10454
10455 ins_encode %{
10456 __ cselw(as_Register($dst$$reg),
10457 as_Register($src2$$reg),
10458 as_Register($src1$$reg),
10459 (Assembler::Condition)$cmp$$cmpcode);
10460 %}
10461
10462 ins_pipe(icond_reg_reg);
10463 %}
10464
10465 // special cases where one arg is zero
10466
10467 instruct cmovN_reg_zero(cmpOp cmp, rFlagsReg cr, iRegNNoSp dst, iRegN src, immN0 zero) %{
10468 match(Set dst (CMoveN (Binary cmp cr) (Binary src zero)));
10469
10470 ins_cost(INSN_COST * 2);
10471 format %{ "cselw $dst, zr, $src $cmp\t# signed, compressed ptr" %}
10472
10473 ins_encode %{
10474 __ cselw(as_Register($dst$$reg),
10475 zr,
10476 as_Register($src$$reg),
10477 (Assembler::Condition)$cmp$$cmpcode);
10478 %}
10479
10480 ins_pipe(icond_reg);
10481 %}
10482
10483 instruct cmovUN_reg_zero(cmpOpU cmp, rFlagsRegU cr, iRegNNoSp dst, iRegN src, immN0 zero) %{
10484 match(Set dst (CMoveN (Binary cmp cr) (Binary src zero)));
10485
10486 ins_cost(INSN_COST * 2);
10487 format %{ "cselw $dst, zr, $src $cmp\t# unsigned, compressed ptr" %}
10488
10489 ins_encode %{
10490 __ cselw(as_Register($dst$$reg),
10491 zr,
10492 as_Register($src$$reg),
10493 (Assembler::Condition)$cmp$$cmpcode);
10494 %}
10495
10496 ins_pipe(icond_reg);
10497 %}
10498
10499 instruct cmovN_zero_reg(cmpOp cmp, rFlagsReg cr, iRegNNoSp dst, immN0 zero, iRegN src) %{
10500 match(Set dst (CMoveN (Binary cmp cr) (Binary zero src)));
10501
10502 ins_cost(INSN_COST * 2);
10503 format %{ "cselw $dst, $src, zr $cmp\t# signed, compressed ptr" %}
10504
10505 ins_encode %{
10506 __ cselw(as_Register($dst$$reg),
10507 as_Register($src$$reg),
10508 zr,
10509 (Assembler::Condition)$cmp$$cmpcode);
10510 %}
10511
10512 ins_pipe(icond_reg);
10513 %}
10514
10515 instruct cmovUN_zero_reg(cmpOpU cmp, rFlagsRegU cr, iRegNNoSp dst, immN0 zero, iRegN src) %{
10516 match(Set dst (CMoveN (Binary cmp cr) (Binary zero src)));
10517
10518 ins_cost(INSN_COST * 2);
10519 format %{ "cselw $dst, $src, zr $cmp\t# unsigned, compressed ptr" %}
10520
10521 ins_encode %{
10522 __ cselw(as_Register($dst$$reg),
10523 as_Register($src$$reg),
10524 zr,
10525 (Assembler::Condition)$cmp$$cmpcode);
10526 %}
10527
10528 ins_pipe(icond_reg);
10529 %}
10530
10531 instruct cmovF_reg(cmpOp cmp, rFlagsReg cr, vRegF dst, vRegF src1, vRegF src2)
10532 %{
10533 match(Set dst (CMoveF (Binary cmp cr) (Binary src1 src2)));
10534
10535 ins_cost(INSN_COST * 3);
10536
10537 format %{ "fcsels $dst, $src1, $src2, $cmp\t# signed cmove float\n\t" %}
10538 ins_encode %{
10539 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
10540 __ fcsels(as_FloatRegister($dst$$reg),
10541 as_FloatRegister($src2$$reg),
10542 as_FloatRegister($src1$$reg),
10543 cond);
10544 %}
10545
10546 ins_pipe(fp_cond_reg_reg_s);
10547 %}
10548
10549 instruct cmovUF_reg(cmpOpU cmp, rFlagsRegU cr, vRegF dst, vRegF src1, vRegF src2)
10550 %{
10551 match(Set dst (CMoveF (Binary cmp cr) (Binary src1 src2)));
10552
10553 ins_cost(INSN_COST * 3);
10554
10555 format %{ "fcsels $dst, $src1, $src2, $cmp\t# unsigned cmove float\n\t" %}
10556 ins_encode %{
10557 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
10558 __ fcsels(as_FloatRegister($dst$$reg),
10559 as_FloatRegister($src2$$reg),
10560 as_FloatRegister($src1$$reg),
10561 cond);
10562 %}
10563
10564 ins_pipe(fp_cond_reg_reg_s);
10565 %}
10566
10567 instruct cmovD_reg(cmpOp cmp, rFlagsReg cr, vRegD dst, vRegD src1, vRegD src2)
10568 %{
10569 match(Set dst (CMoveD (Binary cmp cr) (Binary src1 src2)));
10570
10571 ins_cost(INSN_COST * 3);
10572
10573 format %{ "fcseld $dst, $src1, $src2, $cmp\t# signed cmove float\n\t" %}
10574 ins_encode %{
10575 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
10576 __ fcseld(as_FloatRegister($dst$$reg),
10577 as_FloatRegister($src2$$reg),
10578 as_FloatRegister($src1$$reg),
10579 cond);
10580 %}
10581
10582 ins_pipe(fp_cond_reg_reg_d);
10583 %}
10584
10585 instruct cmovUD_reg(cmpOpU cmp, rFlagsRegU cr, vRegD dst, vRegD src1, vRegD src2)
10586 %{
10587 match(Set dst (CMoveD (Binary cmp cr) (Binary src1 src2)));
10588
10589 ins_cost(INSN_COST * 3);
10590
10591 format %{ "fcseld $dst, $src1, $src2, $cmp\t# unsigned cmove float\n\t" %}
10592 ins_encode %{
10593 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
10594 __ fcseld(as_FloatRegister($dst$$reg),
10595 as_FloatRegister($src2$$reg),
10596 as_FloatRegister($src1$$reg),
10597 cond);
10598 %}
10599
10600 ins_pipe(fp_cond_reg_reg_d);
10601 %}
10602
10603 // ============================================================================
10604 // Arithmetic Instructions
10605 //
10606
10607 // Integer Addition
10608
10609 // TODO
10610 // these currently employ operations which do not set CR and hence are
10611 // not flagged as killing CR but we would like to isolate the cases
10612 // where we want to set flags from those where we don't. need to work
10613 // out how to do that.
10614
10615 instruct addI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10616 match(Set dst (AddI src1 src2));
10617
10618 ins_cost(INSN_COST);
10619 format %{ "addw $dst, $src1, $src2" %}
10620
10621 ins_encode %{
10622 __ addw(as_Register($dst$$reg),
10623 as_Register($src1$$reg),
10624 as_Register($src2$$reg));
10625 %}
10626
10627 ins_pipe(ialu_reg_reg);
10628 %}
10629
10630 instruct addI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immIAddSub src2) %{
10631 match(Set dst (AddI src1 src2));
10632
10633 ins_cost(INSN_COST);
10634 format %{ "addw $dst, $src1, $src2" %}
10635
10636 // use opcode to indicate that this is an add not a sub
10637 opcode(0x0);
10638
10639 ins_encode(aarch64_enc_addsubw_imm(dst, src1, src2));
10640
10641 ins_pipe(ialu_reg_imm);
10642 %}
10643
10644 instruct addI_reg_imm_i2l(iRegINoSp dst, iRegL src1, immIAddSub src2) %{
10645 match(Set dst (AddI (ConvL2I src1) src2));
10646
10647 ins_cost(INSN_COST);
10648 format %{ "addw $dst, $src1, $src2" %}
10649
10650 // use opcode to indicate that this is an add not a sub
10651 opcode(0x0);
10652
10653 ins_encode(aarch64_enc_addsubw_imm(dst, src1, src2));
10654
10655 ins_pipe(ialu_reg_imm);
10656 %}
10657
10658 // Pointer Addition
10659 instruct addP_reg_reg(iRegPNoSp dst, iRegP src1, iRegL src2) %{
10660 match(Set dst (AddP src1 src2));
10661
10662 ins_cost(INSN_COST);
10663 format %{ "add $dst, $src1, $src2\t# ptr" %}
10664
10665 ins_encode %{
10666 __ add(as_Register($dst$$reg),
10667 as_Register($src1$$reg),
10668 as_Register($src2$$reg));
10669 %}
10670
10671 ins_pipe(ialu_reg_reg);
10672 %}
10673
10674 instruct addP_reg_reg_ext(iRegPNoSp dst, iRegP src1, iRegIorL2I src2) %{
10675 match(Set dst (AddP src1 (ConvI2L src2)));
10676
10677 ins_cost(1.9 * INSN_COST);
10678 format %{ "add $dst, $src1, $src2, sxtw\t# ptr" %}
10679
10680 ins_encode %{
10681 __ add(as_Register($dst$$reg),
10682 as_Register($src1$$reg),
10683 as_Register($src2$$reg), ext::sxtw);
10684 %}
10685
10686 ins_pipe(ialu_reg_reg);
10687 %}
10688
10689 instruct addP_reg_reg_lsl(iRegPNoSp dst, iRegP src1, iRegL src2, immIScale scale) %{
10690 match(Set dst (AddP src1 (LShiftL src2 scale)));
10691
10692 ins_cost(1.9 * INSN_COST);
10693 format %{ "add $dst, $src1, $src2, LShiftL $scale\t# ptr" %}
10694
10695 ins_encode %{
10696 __ lea(as_Register($dst$$reg),
10697 Address(as_Register($src1$$reg), as_Register($src2$$reg),
10698 Address::lsl($scale$$constant)));
10699 %}
10700
10701 ins_pipe(ialu_reg_reg_shift);
10702 %}
10703
10704 instruct addP_reg_reg_ext_shift(iRegPNoSp dst, iRegP src1, iRegIorL2I src2, immIScale scale) %{
10705 match(Set dst (AddP src1 (LShiftL (ConvI2L src2) scale)));
10706
10707 ins_cost(1.9 * INSN_COST);
10708 format %{ "add $dst, $src1, $src2, I2L $scale\t# ptr" %}
10709
10710 ins_encode %{
10711 __ lea(as_Register($dst$$reg),
10712 Address(as_Register($src1$$reg), as_Register($src2$$reg),
10713 Address::sxtw($scale$$constant)));
10714 %}
10715
10716 ins_pipe(ialu_reg_reg_shift);
10717 %}
10718
10719 instruct lshift_ext(iRegLNoSp dst, iRegIorL2I src, immI scale, rFlagsReg cr) %{
10720 match(Set dst (LShiftL (ConvI2L src) scale));
10721
10722 ins_cost(INSN_COST);
10723 format %{ "sbfiz $dst, $src, $scale & 63, -$scale & 63\t" %}
10724
10725 ins_encode %{
10726 __ sbfiz(as_Register($dst$$reg),
10727 as_Register($src$$reg),
10728 $scale$$constant & 63, MIN2(32, (int)((-$scale$$constant) & 63)));
10729 %}
10730
10731 ins_pipe(ialu_reg_shift);
10732 %}
10733
10734 // Pointer Immediate Addition
10735 // n.b. this needs to be more expensive than using an indirect memory
10736 // operand
10737 instruct addP_reg_imm(iRegPNoSp dst, iRegP src1, immLAddSub src2) %{
10738 match(Set dst (AddP src1 src2));
10739
10740 ins_cost(INSN_COST);
10741 format %{ "add $dst, $src1, $src2\t# ptr" %}
10742
10743 // use opcode to indicate that this is an add not a sub
10744 opcode(0x0);
10745
10746 ins_encode( aarch64_enc_addsub_imm(dst, src1, src2) );
10747
10748 ins_pipe(ialu_reg_imm);
10749 %}
10750
10751 // Long Addition
10752 instruct addL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
10753
10754 match(Set dst (AddL src1 src2));
10755
10756 ins_cost(INSN_COST);
10757 format %{ "add $dst, $src1, $src2" %}
10758
10759 ins_encode %{
10760 __ add(as_Register($dst$$reg),
10761 as_Register($src1$$reg),
10762 as_Register($src2$$reg));
10763 %}
10764
10765 ins_pipe(ialu_reg_reg);
10766 %}
10767
10768 // No constant pool entries requiredLong Immediate Addition.
10769 instruct addL_reg_imm(iRegLNoSp dst, iRegL src1, immLAddSub src2) %{
10770 match(Set dst (AddL src1 src2));
10771
10772 ins_cost(INSN_COST);
10773 format %{ "add $dst, $src1, $src2" %}
10774
10775 // use opcode to indicate that this is an add not a sub
10776 opcode(0x0);
10777
10778 ins_encode( aarch64_enc_addsub_imm(dst, src1, src2) );
10779
10780 ins_pipe(ialu_reg_imm);
10781 %}
10782
10783 // Integer Subtraction
10784 instruct subI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10785 match(Set dst (SubI src1 src2));
10786
10787 ins_cost(INSN_COST);
10788 format %{ "subw $dst, $src1, $src2" %}
10789
10790 ins_encode %{
10791 __ subw(as_Register($dst$$reg),
10792 as_Register($src1$$reg),
10793 as_Register($src2$$reg));
10794 %}
10795
10796 ins_pipe(ialu_reg_reg);
10797 %}
10798
10799 // Immediate Subtraction
10800 instruct subI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immIAddSub src2) %{
10801 match(Set dst (SubI src1 src2));
10802
10803 ins_cost(INSN_COST);
10804 format %{ "subw $dst, $src1, $src2" %}
10805
10806 // use opcode to indicate that this is a sub not an add
10807 opcode(0x1);
10808
10809 ins_encode(aarch64_enc_addsubw_imm(dst, src1, src2));
10810
10811 ins_pipe(ialu_reg_imm);
10812 %}
10813
10814 // Long Subtraction
10815 instruct subL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
10816
10817 match(Set dst (SubL src1 src2));
10818
10819 ins_cost(INSN_COST);
10820 format %{ "sub $dst, $src1, $src2" %}
10821
10822 ins_encode %{
10823 __ sub(as_Register($dst$$reg),
10824 as_Register($src1$$reg),
10825 as_Register($src2$$reg));
10826 %}
10827
10828 ins_pipe(ialu_reg_reg);
10829 %}
10830
10831 // No constant pool entries requiredLong Immediate Subtraction.
10832 instruct subL_reg_imm(iRegLNoSp dst, iRegL src1, immLAddSub src2) %{
10833 match(Set dst (SubL src1 src2));
10834
10835 ins_cost(INSN_COST);
10836 format %{ "sub$dst, $src1, $src2" %}
10837
10838 // use opcode to indicate that this is a sub not an add
10839 opcode(0x1);
10840
10841 ins_encode( aarch64_enc_addsub_imm(dst, src1, src2) );
10842
10843 ins_pipe(ialu_reg_imm);
10844 %}
10845
10846 // Integer Negation (special case for sub)
10847
10848 instruct negI_reg(iRegINoSp dst, iRegIorL2I src, immI0 zero, rFlagsReg cr) %{
10849 match(Set dst (SubI zero src));
10850
10851 ins_cost(INSN_COST);
10852 format %{ "negw $dst, $src\t# int" %}
10853
10854 ins_encode %{
10855 __ negw(as_Register($dst$$reg),
10856 as_Register($src$$reg));
10857 %}
10858
10859 ins_pipe(ialu_reg);
10860 %}
10861
10862 // Long Negation
10863
10864 instruct negL_reg(iRegLNoSp dst, iRegL src, immL0 zero, rFlagsReg cr) %{
10865 match(Set dst (SubL zero src));
10866
10867 ins_cost(INSN_COST);
10868 format %{ "neg $dst, $src\t# long" %}
10869
10870 ins_encode %{
10871 __ neg(as_Register($dst$$reg),
10872 as_Register($src$$reg));
10873 %}
10874
10875 ins_pipe(ialu_reg);
10876 %}
10877
10878 // Integer Multiply
10879
10880 instruct mulI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10881 match(Set dst (MulI src1 src2));
10882
10883 ins_cost(INSN_COST * 3);
10884 format %{ "mulw $dst, $src1, $src2" %}
10885
10886 ins_encode %{
10887 __ mulw(as_Register($dst$$reg),
10888 as_Register($src1$$reg),
10889 as_Register($src2$$reg));
10890 %}
10891
10892 ins_pipe(imul_reg_reg);
10893 %}
10894
10895 instruct smulI(iRegLNoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10896 match(Set dst (MulL (ConvI2L src1) (ConvI2L src2)));
10897
10898 ins_cost(INSN_COST * 3);
10899 format %{ "smull $dst, $src1, $src2" %}
10900
10901 ins_encode %{
10902 __ smull(as_Register($dst$$reg),
10903 as_Register($src1$$reg),
10904 as_Register($src2$$reg));
10905 %}
10906
10907 ins_pipe(imul_reg_reg);
10908 %}
10909
10910 // Long Multiply
10911
10912 instruct mulL(iRegLNoSp dst, iRegL src1, iRegL src2) %{
10913 match(Set dst (MulL src1 src2));
10914
10915 ins_cost(INSN_COST * 5);
10916 format %{ "mul $dst, $src1, $src2" %}
10917
10918 ins_encode %{
10919 __ mul(as_Register($dst$$reg),
10920 as_Register($src1$$reg),
10921 as_Register($src2$$reg));
10922 %}
10923
10924 ins_pipe(lmul_reg_reg);
10925 %}
10926
10927 instruct mulHiL_rReg(iRegLNoSp dst, iRegL src1, iRegL src2, rFlagsReg cr)
10928 %{
10929 match(Set dst (MulHiL src1 src2));
10930
10931 ins_cost(INSN_COST * 7);
10932 format %{ "smulh $dst, $src1, $src2, \t# mulhi" %}
10933
10934 ins_encode %{
10935 __ smulh(as_Register($dst$$reg),
10936 as_Register($src1$$reg),
10937 as_Register($src2$$reg));
10938 %}
10939
10940 ins_pipe(lmul_reg_reg);
10941 %}
10942
10943 // Combined Integer Multiply & Add/Sub
10944
10945 instruct maddI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegIorL2I src3) %{
10946 match(Set dst (AddI src3 (MulI src1 src2)));
10947
10948 ins_cost(INSN_COST * 3);
10949 format %{ "madd $dst, $src1, $src2, $src3" %}
10950
10951 ins_encode %{
10952 __ maddw(as_Register($dst$$reg),
10953 as_Register($src1$$reg),
10954 as_Register($src2$$reg),
10955 as_Register($src3$$reg));
10956 %}
10957
10958 ins_pipe(imac_reg_reg);
10959 %}
10960
10961 instruct msubI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegIorL2I src3) %{
10962 match(Set dst (SubI src3 (MulI src1 src2)));
10963
10964 ins_cost(INSN_COST * 3);
10965 format %{ "msub $dst, $src1, $src2, $src3" %}
10966
10967 ins_encode %{
10968 __ msubw(as_Register($dst$$reg),
10969 as_Register($src1$$reg),
10970 as_Register($src2$$reg),
10971 as_Register($src3$$reg));
10972 %}
10973
10974 ins_pipe(imac_reg_reg);
10975 %}
10976
10977 // Combined Integer Multiply & Neg
10978
10979 instruct mnegI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI0 zero) %{
10980 match(Set dst (MulI (SubI zero src1) src2));
10981 match(Set dst (MulI src1 (SubI zero src2)));
10982
10983 ins_cost(INSN_COST * 3);
10984 format %{ "mneg $dst, $src1, $src2" %}
10985
10986 ins_encode %{
10987 __ mnegw(as_Register($dst$$reg),
10988 as_Register($src1$$reg),
10989 as_Register($src2$$reg));
10990 %}
10991
10992 ins_pipe(imac_reg_reg);
10993 %}
10994
10995 // Combined Long Multiply & Add/Sub
10996
10997 instruct maddL(iRegLNoSp dst, iRegL src1, iRegL src2, iRegL src3) %{
10998 match(Set dst (AddL src3 (MulL src1 src2)));
10999
11000 ins_cost(INSN_COST * 5);
11001 format %{ "madd $dst, $src1, $src2, $src3" %}
11002
11003 ins_encode %{
11004 __ madd(as_Register($dst$$reg),
11005 as_Register($src1$$reg),
11006 as_Register($src2$$reg),
11007 as_Register($src3$$reg));
11008 %}
11009
11010 ins_pipe(lmac_reg_reg);
11011 %}
11012
11013 instruct msubL(iRegLNoSp dst, iRegL src1, iRegL src2, iRegL src3) %{
11014 match(Set dst (SubL src3 (MulL src1 src2)));
11015
11016 ins_cost(INSN_COST * 5);
11017 format %{ "msub $dst, $src1, $src2, $src3" %}
11018
11019 ins_encode %{
11020 __ msub(as_Register($dst$$reg),
11021 as_Register($src1$$reg),
11022 as_Register($src2$$reg),
11023 as_Register($src3$$reg));
11024 %}
11025
11026 ins_pipe(lmac_reg_reg);
11027 %}
11028
11029 // Combined Long Multiply & Neg
11030
11031 instruct mnegL(iRegLNoSp dst, iRegL src1, iRegL src2, immL0 zero) %{
11032 match(Set dst (MulL (SubL zero src1) src2));
11033 match(Set dst (MulL src1 (SubL zero src2)));
11034
11035 ins_cost(INSN_COST * 5);
11036 format %{ "mneg $dst, $src1, $src2" %}
11037
11038 ins_encode %{
11039 __ mneg(as_Register($dst$$reg),
11040 as_Register($src1$$reg),
11041 as_Register($src2$$reg));
11042 %}
11043
11044 ins_pipe(lmac_reg_reg);
11045 %}
11046
11047 // Combine Integer Signed Multiply & Add/Sub/Neg Long
11048
11049 instruct smaddL(iRegLNoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegLNoSp src3) %{
11050 match(Set dst (AddL src3 (MulL (ConvI2L src1) (ConvI2L src2))));
11051
11052 ins_cost(INSN_COST * 3);
11053 format %{ "smaddl $dst, $src1, $src2, $src3" %}
11054
11055 ins_encode %{
11056 __ smaddl(as_Register($dst$$reg),
11057 as_Register($src1$$reg),
11058 as_Register($src2$$reg),
11059 as_Register($src3$$reg));
11060 %}
11061
11062 ins_pipe(imac_reg_reg);
11063 %}
11064
11065 instruct smsubL(iRegLNoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegLNoSp src3) %{
11066 match(Set dst (SubL src3 (MulL (ConvI2L src1) (ConvI2L src2))));
11067
11068 ins_cost(INSN_COST * 3);
11069 format %{ "smsubl $dst, $src1, $src2, $src3" %}
11070
11071 ins_encode %{
11072 __ smsubl(as_Register($dst$$reg),
11073 as_Register($src1$$reg),
11074 as_Register($src2$$reg),
11075 as_Register($src3$$reg));
11076 %}
11077
11078 ins_pipe(imac_reg_reg);
11079 %}
11080
11081 instruct smnegL(iRegLNoSp dst, iRegIorL2I src1, iRegIorL2I src2, immL0 zero) %{
11082 match(Set dst (MulL (SubL zero (ConvI2L src1)) (ConvI2L src2)));
11083 match(Set dst (MulL (ConvI2L src1) (SubL zero (ConvI2L src2))));
11084
11085 ins_cost(INSN_COST * 3);
11086 format %{ "smnegl $dst, $src1, $src2" %}
11087
11088 ins_encode %{
11089 __ smnegl(as_Register($dst$$reg),
11090 as_Register($src1$$reg),
11091 as_Register($src2$$reg));
11092 %}
11093
11094 ins_pipe(imac_reg_reg);
11095 %}
11096
11097 // Combined Multiply-Add Shorts into Integer (dst = src1 * src2 + src3 * src4)
11098
11099 instruct muladdS2I(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegIorL2I src3, iRegIorL2I src4) %{
11100 match(Set dst (MulAddS2I (Binary src1 src2) (Binary src3 src4)));
11101
11102 ins_cost(INSN_COST * 5);
11103 format %{ "mulw rscratch1, $src1, $src2\n\t"
11104 "maddw $dst, $src3, $src4, rscratch1" %}
11105
11106 ins_encode %{
11107 __ mulw(rscratch1, as_Register($src1$$reg), as_Register($src2$$reg));
11108 __ maddw(as_Register($dst$$reg), as_Register($src3$$reg), as_Register($src4$$reg), rscratch1); %}
11109
11110 ins_pipe(imac_reg_reg);
11111 %}
11112
11113 // Integer Divide
11114
11115 instruct divI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
11116 match(Set dst (DivI src1 src2));
11117
11118 ins_cost(INSN_COST * 19);
11119 format %{ "sdivw $dst, $src1, $src2" %}
11120
11121 ins_encode(aarch64_enc_divw(dst, src1, src2));
11122 ins_pipe(idiv_reg_reg);
11123 %}
11124
11125 // Long Divide
11126
11127 instruct divL(iRegLNoSp dst, iRegL src1, iRegL src2) %{
11128 match(Set dst (DivL src1 src2));
11129
11130 ins_cost(INSN_COST * 35);
11131 format %{ "sdiv $dst, $src1, $src2" %}
11132
11133 ins_encode(aarch64_enc_div(dst, src1, src2));
11134 ins_pipe(ldiv_reg_reg);
11135 %}
11136
11137 // Integer Remainder
11138
11139 instruct modI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
11140 match(Set dst (ModI src1 src2));
11141
11142 ins_cost(INSN_COST * 22);
11143 format %{ "sdivw rscratch1, $src1, $src2\n\t"
11144 "msubw($dst, rscratch1, $src2, $src1" %}
11145
11146 ins_encode(aarch64_enc_modw(dst, src1, src2));
11147 ins_pipe(idiv_reg_reg);
11148 %}
11149
11150 // Long Remainder
11151
11152 instruct modL(iRegLNoSp dst, iRegL src1, iRegL src2) %{
11153 match(Set dst (ModL src1 src2));
11154
11155 ins_cost(INSN_COST * 38);
11156 format %{ "sdiv rscratch1, $src1, $src2\n"
11157 "msub($dst, rscratch1, $src2, $src1" %}
11158
11159 ins_encode(aarch64_enc_mod(dst, src1, src2));
11160 ins_pipe(ldiv_reg_reg);
11161 %}
11162
11163 // Integer Shifts
11164
11165 // Shift Left Register
11166 instruct lShiftI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
11167 match(Set dst (LShiftI src1 src2));
11168
11169 ins_cost(INSN_COST * 2);
11170 format %{ "lslvw $dst, $src1, $src2" %}
11171
11172 ins_encode %{
11173 __ lslvw(as_Register($dst$$reg),
11174 as_Register($src1$$reg),
11175 as_Register($src2$$reg));
11176 %}
11177
11178 ins_pipe(ialu_reg_reg_vshift);
11179 %}
11180
11181 // Shift Left Immediate
11182 instruct lShiftI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immI src2) %{
11183 match(Set dst (LShiftI src1 src2));
11184
11185 ins_cost(INSN_COST);
11186 format %{ "lslw $dst, $src1, ($src2 & 0x1f)" %}
11187
11188 ins_encode %{
11189 __ lslw(as_Register($dst$$reg),
11190 as_Register($src1$$reg),
11191 $src2$$constant & 0x1f);
11192 %}
11193
11194 ins_pipe(ialu_reg_shift);
11195 %}
11196
11197 // Shift Right Logical Register
11198 instruct urShiftI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
11199 match(Set dst (URShiftI src1 src2));
11200
11201 ins_cost(INSN_COST * 2);
11202 format %{ "lsrvw $dst, $src1, $src2" %}
11203
11204 ins_encode %{
11205 __ lsrvw(as_Register($dst$$reg),
11206 as_Register($src1$$reg),
11207 as_Register($src2$$reg));
11208 %}
11209
11210 ins_pipe(ialu_reg_reg_vshift);
11211 %}
11212
11213 // Shift Right Logical Immediate
11214 instruct urShiftI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immI src2) %{
11215 match(Set dst (URShiftI src1 src2));
11216
11217 ins_cost(INSN_COST);
11218 format %{ "lsrw $dst, $src1, ($src2 & 0x1f)" %}
11219
11220 ins_encode %{
11221 __ lsrw(as_Register($dst$$reg),
11222 as_Register($src1$$reg),
11223 $src2$$constant & 0x1f);
11224 %}
11225
11226 ins_pipe(ialu_reg_shift);
11227 %}
11228
11229 // Shift Right Arithmetic Register
11230 instruct rShiftI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
11231 match(Set dst (RShiftI src1 src2));
11232
11233 ins_cost(INSN_COST * 2);
11234 format %{ "asrvw $dst, $src1, $src2" %}
11235
11236 ins_encode %{
11237 __ asrvw(as_Register($dst$$reg),
11238 as_Register($src1$$reg),
11239 as_Register($src2$$reg));
11240 %}
11241
11242 ins_pipe(ialu_reg_reg_vshift);
11243 %}
11244
11245 // Shift Right Arithmetic Immediate
11246 instruct rShiftI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immI src2) %{
11247 match(Set dst (RShiftI src1 src2));
11248
11249 ins_cost(INSN_COST);
11250 format %{ "asrw $dst, $src1, ($src2 & 0x1f)" %}
11251
11252 ins_encode %{
11253 __ asrw(as_Register($dst$$reg),
11254 as_Register($src1$$reg),
11255 $src2$$constant & 0x1f);
11256 %}
11257
11258 ins_pipe(ialu_reg_shift);
11259 %}
11260
11261 // Combined Int Mask and Right Shift (using UBFM)
11262 // TODO
11263
11264 // Long Shifts
11265
11266 // Shift Left Register
11267 instruct lShiftL_reg_reg(iRegLNoSp dst, iRegL src1, iRegIorL2I src2) %{
11268 match(Set dst (LShiftL src1 src2));
11269
11270 ins_cost(INSN_COST * 2);
11271 format %{ "lslv $dst, $src1, $src2" %}
11272
11273 ins_encode %{
11274 __ lslv(as_Register($dst$$reg),
11275 as_Register($src1$$reg),
11276 as_Register($src2$$reg));
11277 %}
11278
11279 ins_pipe(ialu_reg_reg_vshift);
11280 %}
11281
11282 // Shift Left Immediate
11283 instruct lShiftL_reg_imm(iRegLNoSp dst, iRegL src1, immI src2) %{
11284 match(Set dst (LShiftL src1 src2));
11285
11286 ins_cost(INSN_COST);
11287 format %{ "lsl $dst, $src1, ($src2 & 0x3f)" %}
11288
11289 ins_encode %{
11290 __ lsl(as_Register($dst$$reg),
11291 as_Register($src1$$reg),
11292 $src2$$constant & 0x3f);
11293 %}
11294
11295 ins_pipe(ialu_reg_shift);
11296 %}
11297
11298 // Shift Right Logical Register
11299 instruct urShiftL_reg_reg(iRegLNoSp dst, iRegL src1, iRegIorL2I src2) %{
11300 match(Set dst (URShiftL src1 src2));
11301
11302 ins_cost(INSN_COST * 2);
11303 format %{ "lsrv $dst, $src1, $src2" %}
11304
11305 ins_encode %{
11306 __ lsrv(as_Register($dst$$reg),
11307 as_Register($src1$$reg),
11308 as_Register($src2$$reg));
11309 %}
11310
11311 ins_pipe(ialu_reg_reg_vshift);
11312 %}
11313
11314 // Shift Right Logical Immediate
11315 instruct urShiftL_reg_imm(iRegLNoSp dst, iRegL src1, immI src2) %{
11316 match(Set dst (URShiftL src1 src2));
11317
11318 ins_cost(INSN_COST);
11319 format %{ "lsr $dst, $src1, ($src2 & 0x3f)" %}
11320
11321 ins_encode %{
11322 __ lsr(as_Register($dst$$reg),
11323 as_Register($src1$$reg),
11324 $src2$$constant & 0x3f);
11325 %}
11326
11327 ins_pipe(ialu_reg_shift);
11328 %}
11329
11330 // A special-case pattern for card table stores.
11331 instruct urShiftP_reg_imm(iRegLNoSp dst, iRegP src1, immI src2) %{
11332 match(Set dst (URShiftL (CastP2X src1) src2));
11333
11334 ins_cost(INSN_COST);
11335 format %{ "lsr $dst, p2x($src1), ($src2 & 0x3f)" %}
11336
11337 ins_encode %{
11338 __ lsr(as_Register($dst$$reg),
11339 as_Register($src1$$reg),
11340 $src2$$constant & 0x3f);
11341 %}
11342
11343 ins_pipe(ialu_reg_shift);
11344 %}
11345
11346 // Shift Right Arithmetic Register
11347 instruct rShiftL_reg_reg(iRegLNoSp dst, iRegL src1, iRegIorL2I src2) %{
11348 match(Set dst (RShiftL src1 src2));
11349
11350 ins_cost(INSN_COST * 2);
11351 format %{ "asrv $dst, $src1, $src2" %}
11352
11353 ins_encode %{
11354 __ asrv(as_Register($dst$$reg),
11355 as_Register($src1$$reg),
11356 as_Register($src2$$reg));
11357 %}
11358
11359 ins_pipe(ialu_reg_reg_vshift);
11360 %}
11361
11362 // Shift Right Arithmetic Immediate
11363 instruct rShiftL_reg_imm(iRegLNoSp dst, iRegL src1, immI src2) %{
11364 match(Set dst (RShiftL src1 src2));
11365
11366 ins_cost(INSN_COST);
11367 format %{ "asr $dst, $src1, ($src2 & 0x3f)" %}
11368
11369 ins_encode %{
11370 __ asr(as_Register($dst$$reg),
11371 as_Register($src1$$reg),
11372 $src2$$constant & 0x3f);
11373 %}
11374
11375 ins_pipe(ialu_reg_shift);
11376 %}
11377
11378 // BEGIN This section of the file is automatically generated. Do not edit --------------
11379 // This section is generated from aarch64_ad.m4
11380
11381 // This pattern is automatically generated from aarch64_ad.m4
11382 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11383 instruct regL_not_reg(iRegLNoSp dst,
11384 iRegL src1, immL_M1 m1,
11385 rFlagsReg cr) %{
11386 match(Set dst (XorL src1 m1));
11387 ins_cost(INSN_COST);
11388 format %{ "eon $dst, $src1, zr" %}
11389
11390 ins_encode %{
11391 __ eon(as_Register($dst$$reg),
11392 as_Register($src1$$reg),
11393 zr,
11394 Assembler::LSL, 0);
11395 %}
11396
11397 ins_pipe(ialu_reg);
11398 %}
11399
11400 // This pattern is automatically generated from aarch64_ad.m4
11401 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11402 instruct regI_not_reg(iRegINoSp dst,
11403 iRegIorL2I src1, immI_M1 m1,
11404 rFlagsReg cr) %{
11405 match(Set dst (XorI src1 m1));
11406 ins_cost(INSN_COST);
11407 format %{ "eonw $dst, $src1, zr" %}
11408
11409 ins_encode %{
11410 __ eonw(as_Register($dst$$reg),
11411 as_Register($src1$$reg),
11412 zr,
11413 Assembler::LSL, 0);
11414 %}
11415
11416 ins_pipe(ialu_reg);
11417 %}
11418
11419 // This pattern is automatically generated from aarch64_ad.m4
11420 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11421 instruct NegI_reg_URShift_reg(iRegINoSp dst,
11422 immI0 zero, iRegIorL2I src1, immI src2) %{
11423 match(Set dst (SubI zero (URShiftI src1 src2)));
11424
11425 ins_cost(1.9 * INSN_COST);
11426 format %{ "negw $dst, $src1, LSR $src2" %}
11427
11428 ins_encode %{
11429 __ negw(as_Register($dst$$reg), as_Register($src1$$reg),
11430 Assembler::LSR, $src2$$constant & 0x1f);
11431 %}
11432
11433 ins_pipe(ialu_reg_shift);
11434 %}
11435
11436 // This pattern is automatically generated from aarch64_ad.m4
11437 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11438 instruct NegI_reg_RShift_reg(iRegINoSp dst,
11439 immI0 zero, iRegIorL2I src1, immI src2) %{
11440 match(Set dst (SubI zero (RShiftI src1 src2)));
11441
11442 ins_cost(1.9 * INSN_COST);
11443 format %{ "negw $dst, $src1, ASR $src2" %}
11444
11445 ins_encode %{
11446 __ negw(as_Register($dst$$reg), as_Register($src1$$reg),
11447 Assembler::ASR, $src2$$constant & 0x1f);
11448 %}
11449
11450 ins_pipe(ialu_reg_shift);
11451 %}
11452
11453 // This pattern is automatically generated from aarch64_ad.m4
11454 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11455 instruct NegI_reg_LShift_reg(iRegINoSp dst,
11456 immI0 zero, iRegIorL2I src1, immI src2) %{
11457 match(Set dst (SubI zero (LShiftI src1 src2)));
11458
11459 ins_cost(1.9 * INSN_COST);
11460 format %{ "negw $dst, $src1, LSL $src2" %}
11461
11462 ins_encode %{
11463 __ negw(as_Register($dst$$reg), as_Register($src1$$reg),
11464 Assembler::LSL, $src2$$constant & 0x1f);
11465 %}
11466
11467 ins_pipe(ialu_reg_shift);
11468 %}
11469
11470 // This pattern is automatically generated from aarch64_ad.m4
11471 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11472 instruct NegL_reg_URShift_reg(iRegLNoSp dst,
11473 immL0 zero, iRegL src1, immI src2) %{
11474 match(Set dst (SubL zero (URShiftL src1 src2)));
11475
11476 ins_cost(1.9 * INSN_COST);
11477 format %{ "neg $dst, $src1, LSR $src2" %}
11478
11479 ins_encode %{
11480 __ neg(as_Register($dst$$reg), as_Register($src1$$reg),
11481 Assembler::LSR, $src2$$constant & 0x3f);
11482 %}
11483
11484 ins_pipe(ialu_reg_shift);
11485 %}
11486
11487 // This pattern is automatically generated from aarch64_ad.m4
11488 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11489 instruct NegL_reg_RShift_reg(iRegLNoSp dst,
11490 immL0 zero, iRegL src1, immI src2) %{
11491 match(Set dst (SubL zero (RShiftL src1 src2)));
11492
11493 ins_cost(1.9 * INSN_COST);
11494 format %{ "neg $dst, $src1, ASR $src2" %}
11495
11496 ins_encode %{
11497 __ neg(as_Register($dst$$reg), as_Register($src1$$reg),
11498 Assembler::ASR, $src2$$constant & 0x3f);
11499 %}
11500
11501 ins_pipe(ialu_reg_shift);
11502 %}
11503
11504 // This pattern is automatically generated from aarch64_ad.m4
11505 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11506 instruct NegL_reg_LShift_reg(iRegLNoSp dst,
11507 immL0 zero, iRegL src1, immI src2) %{
11508 match(Set dst (SubL zero (LShiftL src1 src2)));
11509
11510 ins_cost(1.9 * INSN_COST);
11511 format %{ "neg $dst, $src1, LSL $src2" %}
11512
11513 ins_encode %{
11514 __ neg(as_Register($dst$$reg), as_Register($src1$$reg),
11515 Assembler::LSL, $src2$$constant & 0x3f);
11516 %}
11517
11518 ins_pipe(ialu_reg_shift);
11519 %}
11520
11521 // This pattern is automatically generated from aarch64_ad.m4
11522 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11523 instruct AndI_reg_not_reg(iRegINoSp dst,
11524 iRegIorL2I src1, iRegIorL2I src2, immI_M1 m1) %{
11525 match(Set dst (AndI src1 (XorI src2 m1)));
11526 ins_cost(INSN_COST);
11527 format %{ "bicw $dst, $src1, $src2" %}
11528
11529 ins_encode %{
11530 __ bicw(as_Register($dst$$reg),
11531 as_Register($src1$$reg),
11532 as_Register($src2$$reg),
11533 Assembler::LSL, 0);
11534 %}
11535
11536 ins_pipe(ialu_reg_reg);
11537 %}
11538
11539 // This pattern is automatically generated from aarch64_ad.m4
11540 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11541 instruct AndL_reg_not_reg(iRegLNoSp dst,
11542 iRegL src1, iRegL src2, immL_M1 m1) %{
11543 match(Set dst (AndL src1 (XorL src2 m1)));
11544 ins_cost(INSN_COST);
11545 format %{ "bic $dst, $src1, $src2" %}
11546
11547 ins_encode %{
11548 __ bic(as_Register($dst$$reg),
11549 as_Register($src1$$reg),
11550 as_Register($src2$$reg),
11551 Assembler::LSL, 0);
11552 %}
11553
11554 ins_pipe(ialu_reg_reg);
11555 %}
11556
11557 // This pattern is automatically generated from aarch64_ad.m4
11558 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11559 instruct OrI_reg_not_reg(iRegINoSp dst,
11560 iRegIorL2I src1, iRegIorL2I src2, immI_M1 m1) %{
11561 match(Set dst (OrI src1 (XorI src2 m1)));
11562 ins_cost(INSN_COST);
11563 format %{ "ornw $dst, $src1, $src2" %}
11564
11565 ins_encode %{
11566 __ ornw(as_Register($dst$$reg),
11567 as_Register($src1$$reg),
11568 as_Register($src2$$reg),
11569 Assembler::LSL, 0);
11570 %}
11571
11572 ins_pipe(ialu_reg_reg);
11573 %}
11574
11575 // This pattern is automatically generated from aarch64_ad.m4
11576 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11577 instruct OrL_reg_not_reg(iRegLNoSp dst,
11578 iRegL src1, iRegL src2, immL_M1 m1) %{
11579 match(Set dst (OrL src1 (XorL src2 m1)));
11580 ins_cost(INSN_COST);
11581 format %{ "orn $dst, $src1, $src2" %}
11582
11583 ins_encode %{
11584 __ orn(as_Register($dst$$reg),
11585 as_Register($src1$$reg),
11586 as_Register($src2$$reg),
11587 Assembler::LSL, 0);
11588 %}
11589
11590 ins_pipe(ialu_reg_reg);
11591 %}
11592
11593 // This pattern is automatically generated from aarch64_ad.m4
11594 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11595 instruct XorI_reg_not_reg(iRegINoSp dst,
11596 iRegIorL2I src1, iRegIorL2I src2, immI_M1 m1) %{
11597 match(Set dst (XorI m1 (XorI src2 src1)));
11598 ins_cost(INSN_COST);
11599 format %{ "eonw $dst, $src1, $src2" %}
11600
11601 ins_encode %{
11602 __ eonw(as_Register($dst$$reg),
11603 as_Register($src1$$reg),
11604 as_Register($src2$$reg),
11605 Assembler::LSL, 0);
11606 %}
11607
11608 ins_pipe(ialu_reg_reg);
11609 %}
11610
11611 // This pattern is automatically generated from aarch64_ad.m4
11612 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11613 instruct XorL_reg_not_reg(iRegLNoSp dst,
11614 iRegL src1, iRegL src2, immL_M1 m1) %{
11615 match(Set dst (XorL m1 (XorL src2 src1)));
11616 ins_cost(INSN_COST);
11617 format %{ "eon $dst, $src1, $src2" %}
11618
11619 ins_encode %{
11620 __ eon(as_Register($dst$$reg),
11621 as_Register($src1$$reg),
11622 as_Register($src2$$reg),
11623 Assembler::LSL, 0);
11624 %}
11625
11626 ins_pipe(ialu_reg_reg);
11627 %}
11628
11629 // This pattern is automatically generated from aarch64_ad.m4
11630 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11631 // val & (-1 ^ (val >>> shift)) ==> bicw
11632 instruct AndI_reg_URShift_not_reg(iRegINoSp dst,
11633 iRegIorL2I src1, iRegIorL2I src2,
11634 immI src3, immI_M1 src4) %{
11635 match(Set dst (AndI src1 (XorI(URShiftI src2 src3) src4)));
11636 ins_cost(1.9 * INSN_COST);
11637 format %{ "bicw $dst, $src1, $src2, LSR $src3" %}
11638
11639 ins_encode %{
11640 __ bicw(as_Register($dst$$reg),
11641 as_Register($src1$$reg),
11642 as_Register($src2$$reg),
11643 Assembler::LSR,
11644 $src3$$constant & 0x1f);
11645 %}
11646
11647 ins_pipe(ialu_reg_reg_shift);
11648 %}
11649
11650 // This pattern is automatically generated from aarch64_ad.m4
11651 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11652 // val & (-1 ^ (val >>> shift)) ==> bic
11653 instruct AndL_reg_URShift_not_reg(iRegLNoSp dst,
11654 iRegL src1, iRegL src2,
11655 immI src3, immL_M1 src4) %{
11656 match(Set dst (AndL src1 (XorL(URShiftL src2 src3) src4)));
11657 ins_cost(1.9 * INSN_COST);
11658 format %{ "bic $dst, $src1, $src2, LSR $src3" %}
11659
11660 ins_encode %{
11661 __ bic(as_Register($dst$$reg),
11662 as_Register($src1$$reg),
11663 as_Register($src2$$reg),
11664 Assembler::LSR,
11665 $src3$$constant & 0x3f);
11666 %}
11667
11668 ins_pipe(ialu_reg_reg_shift);
11669 %}
11670
11671 // This pattern is automatically generated from aarch64_ad.m4
11672 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11673 // val & (-1 ^ (val >> shift)) ==> bicw
11674 instruct AndI_reg_RShift_not_reg(iRegINoSp dst,
11675 iRegIorL2I src1, iRegIorL2I src2,
11676 immI src3, immI_M1 src4) %{
11677 match(Set dst (AndI src1 (XorI(RShiftI src2 src3) src4)));
11678 ins_cost(1.9 * INSN_COST);
11679 format %{ "bicw $dst, $src1, $src2, ASR $src3" %}
11680
11681 ins_encode %{
11682 __ bicw(as_Register($dst$$reg),
11683 as_Register($src1$$reg),
11684 as_Register($src2$$reg),
11685 Assembler::ASR,
11686 $src3$$constant & 0x1f);
11687 %}
11688
11689 ins_pipe(ialu_reg_reg_shift);
11690 %}
11691
11692 // This pattern is automatically generated from aarch64_ad.m4
11693 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11694 // val & (-1 ^ (val >> shift)) ==> bic
11695 instruct AndL_reg_RShift_not_reg(iRegLNoSp dst,
11696 iRegL src1, iRegL src2,
11697 immI src3, immL_M1 src4) %{
11698 match(Set dst (AndL src1 (XorL(RShiftL src2 src3) src4)));
11699 ins_cost(1.9 * INSN_COST);
11700 format %{ "bic $dst, $src1, $src2, ASR $src3" %}
11701
11702 ins_encode %{
11703 __ bic(as_Register($dst$$reg),
11704 as_Register($src1$$reg),
11705 as_Register($src2$$reg),
11706 Assembler::ASR,
11707 $src3$$constant & 0x3f);
11708 %}
11709
11710 ins_pipe(ialu_reg_reg_shift);
11711 %}
11712
11713 // This pattern is automatically generated from aarch64_ad.m4
11714 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11715 // val & (-1 ^ (val ror shift)) ==> bicw
11716 instruct AndI_reg_RotateRight_not_reg(iRegINoSp dst,
11717 iRegIorL2I src1, iRegIorL2I src2,
11718 immI src3, immI_M1 src4) %{
11719 match(Set dst (AndI src1 (XorI(RotateRight src2 src3) src4)));
11720 ins_cost(1.9 * INSN_COST);
11721 format %{ "bicw $dst, $src1, $src2, ROR $src3" %}
11722
11723 ins_encode %{
11724 __ bicw(as_Register($dst$$reg),
11725 as_Register($src1$$reg),
11726 as_Register($src2$$reg),
11727 Assembler::ROR,
11728 $src3$$constant & 0x1f);
11729 %}
11730
11731 ins_pipe(ialu_reg_reg_shift);
11732 %}
11733
11734 // This pattern is automatically generated from aarch64_ad.m4
11735 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11736 // val & (-1 ^ (val ror shift)) ==> bic
11737 instruct AndL_reg_RotateRight_not_reg(iRegLNoSp dst,
11738 iRegL src1, iRegL src2,
11739 immI src3, immL_M1 src4) %{
11740 match(Set dst (AndL src1 (XorL(RotateRight src2 src3) src4)));
11741 ins_cost(1.9 * INSN_COST);
11742 format %{ "bic $dst, $src1, $src2, ROR $src3" %}
11743
11744 ins_encode %{
11745 __ bic(as_Register($dst$$reg),
11746 as_Register($src1$$reg),
11747 as_Register($src2$$reg),
11748 Assembler::ROR,
11749 $src3$$constant & 0x3f);
11750 %}
11751
11752 ins_pipe(ialu_reg_reg_shift);
11753 %}
11754
11755 // This pattern is automatically generated from aarch64_ad.m4
11756 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11757 // val & (-1 ^ (val << shift)) ==> bicw
11758 instruct AndI_reg_LShift_not_reg(iRegINoSp dst,
11759 iRegIorL2I src1, iRegIorL2I src2,
11760 immI src3, immI_M1 src4) %{
11761 match(Set dst (AndI src1 (XorI(LShiftI src2 src3) src4)));
11762 ins_cost(1.9 * INSN_COST);
11763 format %{ "bicw $dst, $src1, $src2, LSL $src3" %}
11764
11765 ins_encode %{
11766 __ bicw(as_Register($dst$$reg),
11767 as_Register($src1$$reg),
11768 as_Register($src2$$reg),
11769 Assembler::LSL,
11770 $src3$$constant & 0x1f);
11771 %}
11772
11773 ins_pipe(ialu_reg_reg_shift);
11774 %}
11775
11776 // This pattern is automatically generated from aarch64_ad.m4
11777 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11778 // val & (-1 ^ (val << shift)) ==> bic
11779 instruct AndL_reg_LShift_not_reg(iRegLNoSp dst,
11780 iRegL src1, iRegL src2,
11781 immI src3, immL_M1 src4) %{
11782 match(Set dst (AndL src1 (XorL(LShiftL src2 src3) src4)));
11783 ins_cost(1.9 * INSN_COST);
11784 format %{ "bic $dst, $src1, $src2, LSL $src3" %}
11785
11786 ins_encode %{
11787 __ bic(as_Register($dst$$reg),
11788 as_Register($src1$$reg),
11789 as_Register($src2$$reg),
11790 Assembler::LSL,
11791 $src3$$constant & 0x3f);
11792 %}
11793
11794 ins_pipe(ialu_reg_reg_shift);
11795 %}
11796
11797 // This pattern is automatically generated from aarch64_ad.m4
11798 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11799 // val ^ (-1 ^ (val >>> shift)) ==> eonw
11800 instruct XorI_reg_URShift_not_reg(iRegINoSp dst,
11801 iRegIorL2I src1, iRegIorL2I src2,
11802 immI src3, immI_M1 src4) %{
11803 match(Set dst (XorI src4 (XorI(URShiftI src2 src3) src1)));
11804 ins_cost(1.9 * INSN_COST);
11805 format %{ "eonw $dst, $src1, $src2, LSR $src3" %}
11806
11807 ins_encode %{
11808 __ eonw(as_Register($dst$$reg),
11809 as_Register($src1$$reg),
11810 as_Register($src2$$reg),
11811 Assembler::LSR,
11812 $src3$$constant & 0x1f);
11813 %}
11814
11815 ins_pipe(ialu_reg_reg_shift);
11816 %}
11817
11818 // This pattern is automatically generated from aarch64_ad.m4
11819 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11820 // val ^ (-1 ^ (val >>> shift)) ==> eon
11821 instruct XorL_reg_URShift_not_reg(iRegLNoSp dst,
11822 iRegL src1, iRegL src2,
11823 immI src3, immL_M1 src4) %{
11824 match(Set dst (XorL src4 (XorL(URShiftL src2 src3) src1)));
11825 ins_cost(1.9 * INSN_COST);
11826 format %{ "eon $dst, $src1, $src2, LSR $src3" %}
11827
11828 ins_encode %{
11829 __ eon(as_Register($dst$$reg),
11830 as_Register($src1$$reg),
11831 as_Register($src2$$reg),
11832 Assembler::LSR,
11833 $src3$$constant & 0x3f);
11834 %}
11835
11836 ins_pipe(ialu_reg_reg_shift);
11837 %}
11838
11839 // This pattern is automatically generated from aarch64_ad.m4
11840 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11841 // val ^ (-1 ^ (val >> shift)) ==> eonw
11842 instruct XorI_reg_RShift_not_reg(iRegINoSp dst,
11843 iRegIorL2I src1, iRegIorL2I src2,
11844 immI src3, immI_M1 src4) %{
11845 match(Set dst (XorI src4 (XorI(RShiftI src2 src3) src1)));
11846 ins_cost(1.9 * INSN_COST);
11847 format %{ "eonw $dst, $src1, $src2, ASR $src3" %}
11848
11849 ins_encode %{
11850 __ eonw(as_Register($dst$$reg),
11851 as_Register($src1$$reg),
11852 as_Register($src2$$reg),
11853 Assembler::ASR,
11854 $src3$$constant & 0x1f);
11855 %}
11856
11857 ins_pipe(ialu_reg_reg_shift);
11858 %}
11859
11860 // This pattern is automatically generated from aarch64_ad.m4
11861 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11862 // val ^ (-1 ^ (val >> shift)) ==> eon
11863 instruct XorL_reg_RShift_not_reg(iRegLNoSp dst,
11864 iRegL src1, iRegL src2,
11865 immI src3, immL_M1 src4) %{
11866 match(Set dst (XorL src4 (XorL(RShiftL src2 src3) src1)));
11867 ins_cost(1.9 * INSN_COST);
11868 format %{ "eon $dst, $src1, $src2, ASR $src3" %}
11869
11870 ins_encode %{
11871 __ eon(as_Register($dst$$reg),
11872 as_Register($src1$$reg),
11873 as_Register($src2$$reg),
11874 Assembler::ASR,
11875 $src3$$constant & 0x3f);
11876 %}
11877
11878 ins_pipe(ialu_reg_reg_shift);
11879 %}
11880
11881 // This pattern is automatically generated from aarch64_ad.m4
11882 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11883 // val ^ (-1 ^ (val ror shift)) ==> eonw
11884 instruct XorI_reg_RotateRight_not_reg(iRegINoSp dst,
11885 iRegIorL2I src1, iRegIorL2I src2,
11886 immI src3, immI_M1 src4) %{
11887 match(Set dst (XorI src4 (XorI(RotateRight src2 src3) src1)));
11888 ins_cost(1.9 * INSN_COST);
11889 format %{ "eonw $dst, $src1, $src2, ROR $src3" %}
11890
11891 ins_encode %{
11892 __ eonw(as_Register($dst$$reg),
11893 as_Register($src1$$reg),
11894 as_Register($src2$$reg),
11895 Assembler::ROR,
11896 $src3$$constant & 0x1f);
11897 %}
11898
11899 ins_pipe(ialu_reg_reg_shift);
11900 %}
11901
11902 // This pattern is automatically generated from aarch64_ad.m4
11903 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11904 // val ^ (-1 ^ (val ror shift)) ==> eon
11905 instruct XorL_reg_RotateRight_not_reg(iRegLNoSp dst,
11906 iRegL src1, iRegL src2,
11907 immI src3, immL_M1 src4) %{
11908 match(Set dst (XorL src4 (XorL(RotateRight src2 src3) src1)));
11909 ins_cost(1.9 * INSN_COST);
11910 format %{ "eon $dst, $src1, $src2, ROR $src3" %}
11911
11912 ins_encode %{
11913 __ eon(as_Register($dst$$reg),
11914 as_Register($src1$$reg),
11915 as_Register($src2$$reg),
11916 Assembler::ROR,
11917 $src3$$constant & 0x3f);
11918 %}
11919
11920 ins_pipe(ialu_reg_reg_shift);
11921 %}
11922
11923 // This pattern is automatically generated from aarch64_ad.m4
11924 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11925 // val ^ (-1 ^ (val << shift)) ==> eonw
11926 instruct XorI_reg_LShift_not_reg(iRegINoSp dst,
11927 iRegIorL2I src1, iRegIorL2I src2,
11928 immI src3, immI_M1 src4) %{
11929 match(Set dst (XorI src4 (XorI(LShiftI src2 src3) src1)));
11930 ins_cost(1.9 * INSN_COST);
11931 format %{ "eonw $dst, $src1, $src2, LSL $src3" %}
11932
11933 ins_encode %{
11934 __ eonw(as_Register($dst$$reg),
11935 as_Register($src1$$reg),
11936 as_Register($src2$$reg),
11937 Assembler::LSL,
11938 $src3$$constant & 0x1f);
11939 %}
11940
11941 ins_pipe(ialu_reg_reg_shift);
11942 %}
11943
11944 // This pattern is automatically generated from aarch64_ad.m4
11945 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11946 // val ^ (-1 ^ (val << shift)) ==> eon
11947 instruct XorL_reg_LShift_not_reg(iRegLNoSp dst,
11948 iRegL src1, iRegL src2,
11949 immI src3, immL_M1 src4) %{
11950 match(Set dst (XorL src4 (XorL(LShiftL src2 src3) src1)));
11951 ins_cost(1.9 * INSN_COST);
11952 format %{ "eon $dst, $src1, $src2, LSL $src3" %}
11953
11954 ins_encode %{
11955 __ eon(as_Register($dst$$reg),
11956 as_Register($src1$$reg),
11957 as_Register($src2$$reg),
11958 Assembler::LSL,
11959 $src3$$constant & 0x3f);
11960 %}
11961
11962 ins_pipe(ialu_reg_reg_shift);
11963 %}
11964
11965 // This pattern is automatically generated from aarch64_ad.m4
11966 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11967 // val | (-1 ^ (val >>> shift)) ==> ornw
11968 instruct OrI_reg_URShift_not_reg(iRegINoSp dst,
11969 iRegIorL2I src1, iRegIorL2I src2,
11970 immI src3, immI_M1 src4) %{
11971 match(Set dst (OrI src1 (XorI(URShiftI src2 src3) src4)));
11972 ins_cost(1.9 * INSN_COST);
11973 format %{ "ornw $dst, $src1, $src2, LSR $src3" %}
11974
11975 ins_encode %{
11976 __ ornw(as_Register($dst$$reg),
11977 as_Register($src1$$reg),
11978 as_Register($src2$$reg),
11979 Assembler::LSR,
11980 $src3$$constant & 0x1f);
11981 %}
11982
11983 ins_pipe(ialu_reg_reg_shift);
11984 %}
11985
11986 // This pattern is automatically generated from aarch64_ad.m4
11987 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11988 // val | (-1 ^ (val >>> shift)) ==> orn
11989 instruct OrL_reg_URShift_not_reg(iRegLNoSp dst,
11990 iRegL src1, iRegL src2,
11991 immI src3, immL_M1 src4) %{
11992 match(Set dst (OrL src1 (XorL(URShiftL src2 src3) src4)));
11993 ins_cost(1.9 * INSN_COST);
11994 format %{ "orn $dst, $src1, $src2, LSR $src3" %}
11995
11996 ins_encode %{
11997 __ orn(as_Register($dst$$reg),
11998 as_Register($src1$$reg),
11999 as_Register($src2$$reg),
12000 Assembler::LSR,
12001 $src3$$constant & 0x3f);
12002 %}
12003
12004 ins_pipe(ialu_reg_reg_shift);
12005 %}
12006
12007 // This pattern is automatically generated from aarch64_ad.m4
12008 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12009 // val | (-1 ^ (val >> shift)) ==> ornw
12010 instruct OrI_reg_RShift_not_reg(iRegINoSp dst,
12011 iRegIorL2I src1, iRegIorL2I src2,
12012 immI src3, immI_M1 src4) %{
12013 match(Set dst (OrI src1 (XorI(RShiftI src2 src3) src4)));
12014 ins_cost(1.9 * INSN_COST);
12015 format %{ "ornw $dst, $src1, $src2, ASR $src3" %}
12016
12017 ins_encode %{
12018 __ ornw(as_Register($dst$$reg),
12019 as_Register($src1$$reg),
12020 as_Register($src2$$reg),
12021 Assembler::ASR,
12022 $src3$$constant & 0x1f);
12023 %}
12024
12025 ins_pipe(ialu_reg_reg_shift);
12026 %}
12027
12028 // This pattern is automatically generated from aarch64_ad.m4
12029 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12030 // val | (-1 ^ (val >> shift)) ==> orn
12031 instruct OrL_reg_RShift_not_reg(iRegLNoSp dst,
12032 iRegL src1, iRegL src2,
12033 immI src3, immL_M1 src4) %{
12034 match(Set dst (OrL src1 (XorL(RShiftL src2 src3) src4)));
12035 ins_cost(1.9 * INSN_COST);
12036 format %{ "orn $dst, $src1, $src2, ASR $src3" %}
12037
12038 ins_encode %{
12039 __ orn(as_Register($dst$$reg),
12040 as_Register($src1$$reg),
12041 as_Register($src2$$reg),
12042 Assembler::ASR,
12043 $src3$$constant & 0x3f);
12044 %}
12045
12046 ins_pipe(ialu_reg_reg_shift);
12047 %}
12048
12049 // This pattern is automatically generated from aarch64_ad.m4
12050 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12051 // val | (-1 ^ (val ror shift)) ==> ornw
12052 instruct OrI_reg_RotateRight_not_reg(iRegINoSp dst,
12053 iRegIorL2I src1, iRegIorL2I src2,
12054 immI src3, immI_M1 src4) %{
12055 match(Set dst (OrI src1 (XorI(RotateRight src2 src3) src4)));
12056 ins_cost(1.9 * INSN_COST);
12057 format %{ "ornw $dst, $src1, $src2, ROR $src3" %}
12058
12059 ins_encode %{
12060 __ ornw(as_Register($dst$$reg),
12061 as_Register($src1$$reg),
12062 as_Register($src2$$reg),
12063 Assembler::ROR,
12064 $src3$$constant & 0x1f);
12065 %}
12066
12067 ins_pipe(ialu_reg_reg_shift);
12068 %}
12069
12070 // This pattern is automatically generated from aarch64_ad.m4
12071 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12072 // val | (-1 ^ (val ror shift)) ==> orn
12073 instruct OrL_reg_RotateRight_not_reg(iRegLNoSp dst,
12074 iRegL src1, iRegL src2,
12075 immI src3, immL_M1 src4) %{
12076 match(Set dst (OrL src1 (XorL(RotateRight src2 src3) src4)));
12077 ins_cost(1.9 * INSN_COST);
12078 format %{ "orn $dst, $src1, $src2, ROR $src3" %}
12079
12080 ins_encode %{
12081 __ orn(as_Register($dst$$reg),
12082 as_Register($src1$$reg),
12083 as_Register($src2$$reg),
12084 Assembler::ROR,
12085 $src3$$constant & 0x3f);
12086 %}
12087
12088 ins_pipe(ialu_reg_reg_shift);
12089 %}
12090
12091 // This pattern is automatically generated from aarch64_ad.m4
12092 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12093 // val | (-1 ^ (val << shift)) ==> ornw
12094 instruct OrI_reg_LShift_not_reg(iRegINoSp dst,
12095 iRegIorL2I src1, iRegIorL2I src2,
12096 immI src3, immI_M1 src4) %{
12097 match(Set dst (OrI src1 (XorI(LShiftI src2 src3) src4)));
12098 ins_cost(1.9 * INSN_COST);
12099 format %{ "ornw $dst, $src1, $src2, LSL $src3" %}
12100
12101 ins_encode %{
12102 __ ornw(as_Register($dst$$reg),
12103 as_Register($src1$$reg),
12104 as_Register($src2$$reg),
12105 Assembler::LSL,
12106 $src3$$constant & 0x1f);
12107 %}
12108
12109 ins_pipe(ialu_reg_reg_shift);
12110 %}
12111
12112 // This pattern is automatically generated from aarch64_ad.m4
12113 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12114 // val | (-1 ^ (val << shift)) ==> orn
12115 instruct OrL_reg_LShift_not_reg(iRegLNoSp dst,
12116 iRegL src1, iRegL src2,
12117 immI src3, immL_M1 src4) %{
12118 match(Set dst (OrL src1 (XorL(LShiftL src2 src3) src4)));
12119 ins_cost(1.9 * INSN_COST);
12120 format %{ "orn $dst, $src1, $src2, LSL $src3" %}
12121
12122 ins_encode %{
12123 __ orn(as_Register($dst$$reg),
12124 as_Register($src1$$reg),
12125 as_Register($src2$$reg),
12126 Assembler::LSL,
12127 $src3$$constant & 0x3f);
12128 %}
12129
12130 ins_pipe(ialu_reg_reg_shift);
12131 %}
12132
12133 // This pattern is automatically generated from aarch64_ad.m4
12134 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12135 instruct AndI_reg_URShift_reg(iRegINoSp dst,
12136 iRegIorL2I src1, iRegIorL2I src2,
12137 immI src3) %{
12138 match(Set dst (AndI src1 (URShiftI src2 src3)));
12139
12140 ins_cost(1.9 * INSN_COST);
12141 format %{ "andw $dst, $src1, $src2, LSR $src3" %}
12142
12143 ins_encode %{
12144 __ andw(as_Register($dst$$reg),
12145 as_Register($src1$$reg),
12146 as_Register($src2$$reg),
12147 Assembler::LSR,
12148 $src3$$constant & 0x1f);
12149 %}
12150
12151 ins_pipe(ialu_reg_reg_shift);
12152 %}
12153
12154 // This pattern is automatically generated from aarch64_ad.m4
12155 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12156 instruct AndL_reg_URShift_reg(iRegLNoSp dst,
12157 iRegL src1, iRegL src2,
12158 immI src3) %{
12159 match(Set dst (AndL src1 (URShiftL src2 src3)));
12160
12161 ins_cost(1.9 * INSN_COST);
12162 format %{ "andr $dst, $src1, $src2, LSR $src3" %}
12163
12164 ins_encode %{
12165 __ andr(as_Register($dst$$reg),
12166 as_Register($src1$$reg),
12167 as_Register($src2$$reg),
12168 Assembler::LSR,
12169 $src3$$constant & 0x3f);
12170 %}
12171
12172 ins_pipe(ialu_reg_reg_shift);
12173 %}
12174
12175 // This pattern is automatically generated from aarch64_ad.m4
12176 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12177 instruct AndI_reg_RShift_reg(iRegINoSp dst,
12178 iRegIorL2I src1, iRegIorL2I src2,
12179 immI src3) %{
12180 match(Set dst (AndI src1 (RShiftI src2 src3)));
12181
12182 ins_cost(1.9 * INSN_COST);
12183 format %{ "andw $dst, $src1, $src2, ASR $src3" %}
12184
12185 ins_encode %{
12186 __ andw(as_Register($dst$$reg),
12187 as_Register($src1$$reg),
12188 as_Register($src2$$reg),
12189 Assembler::ASR,
12190 $src3$$constant & 0x1f);
12191 %}
12192
12193 ins_pipe(ialu_reg_reg_shift);
12194 %}
12195
12196 // This pattern is automatically generated from aarch64_ad.m4
12197 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12198 instruct AndL_reg_RShift_reg(iRegLNoSp dst,
12199 iRegL src1, iRegL src2,
12200 immI src3) %{
12201 match(Set dst (AndL src1 (RShiftL src2 src3)));
12202
12203 ins_cost(1.9 * INSN_COST);
12204 format %{ "andr $dst, $src1, $src2, ASR $src3" %}
12205
12206 ins_encode %{
12207 __ andr(as_Register($dst$$reg),
12208 as_Register($src1$$reg),
12209 as_Register($src2$$reg),
12210 Assembler::ASR,
12211 $src3$$constant & 0x3f);
12212 %}
12213
12214 ins_pipe(ialu_reg_reg_shift);
12215 %}
12216
12217 // This pattern is automatically generated from aarch64_ad.m4
12218 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12219 instruct AndI_reg_LShift_reg(iRegINoSp dst,
12220 iRegIorL2I src1, iRegIorL2I src2,
12221 immI src3) %{
12222 match(Set dst (AndI src1 (LShiftI src2 src3)));
12223
12224 ins_cost(1.9 * INSN_COST);
12225 format %{ "andw $dst, $src1, $src2, LSL $src3" %}
12226
12227 ins_encode %{
12228 __ andw(as_Register($dst$$reg),
12229 as_Register($src1$$reg),
12230 as_Register($src2$$reg),
12231 Assembler::LSL,
12232 $src3$$constant & 0x1f);
12233 %}
12234
12235 ins_pipe(ialu_reg_reg_shift);
12236 %}
12237
12238 // This pattern is automatically generated from aarch64_ad.m4
12239 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12240 instruct AndL_reg_LShift_reg(iRegLNoSp dst,
12241 iRegL src1, iRegL src2,
12242 immI src3) %{
12243 match(Set dst (AndL src1 (LShiftL src2 src3)));
12244
12245 ins_cost(1.9 * INSN_COST);
12246 format %{ "andr $dst, $src1, $src2, LSL $src3" %}
12247
12248 ins_encode %{
12249 __ andr(as_Register($dst$$reg),
12250 as_Register($src1$$reg),
12251 as_Register($src2$$reg),
12252 Assembler::LSL,
12253 $src3$$constant & 0x3f);
12254 %}
12255
12256 ins_pipe(ialu_reg_reg_shift);
12257 %}
12258
12259 // This pattern is automatically generated from aarch64_ad.m4
12260 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12261 instruct AndI_reg_RotateRight_reg(iRegINoSp dst,
12262 iRegIorL2I src1, iRegIorL2I src2,
12263 immI src3) %{
12264 match(Set dst (AndI src1 (RotateRight src2 src3)));
12265
12266 ins_cost(1.9 * INSN_COST);
12267 format %{ "andw $dst, $src1, $src2, ROR $src3" %}
12268
12269 ins_encode %{
12270 __ andw(as_Register($dst$$reg),
12271 as_Register($src1$$reg),
12272 as_Register($src2$$reg),
12273 Assembler::ROR,
12274 $src3$$constant & 0x1f);
12275 %}
12276
12277 ins_pipe(ialu_reg_reg_shift);
12278 %}
12279
12280 // This pattern is automatically generated from aarch64_ad.m4
12281 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12282 instruct AndL_reg_RotateRight_reg(iRegLNoSp dst,
12283 iRegL src1, iRegL src2,
12284 immI src3) %{
12285 match(Set dst (AndL src1 (RotateRight src2 src3)));
12286
12287 ins_cost(1.9 * INSN_COST);
12288 format %{ "andr $dst, $src1, $src2, ROR $src3" %}
12289
12290 ins_encode %{
12291 __ andr(as_Register($dst$$reg),
12292 as_Register($src1$$reg),
12293 as_Register($src2$$reg),
12294 Assembler::ROR,
12295 $src3$$constant & 0x3f);
12296 %}
12297
12298 ins_pipe(ialu_reg_reg_shift);
12299 %}
12300
12301 // This pattern is automatically generated from aarch64_ad.m4
12302 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12303 instruct XorI_reg_URShift_reg(iRegINoSp dst,
12304 iRegIorL2I src1, iRegIorL2I src2,
12305 immI src3) %{
12306 match(Set dst (XorI src1 (URShiftI src2 src3)));
12307
12308 ins_cost(1.9 * INSN_COST);
12309 format %{ "eorw $dst, $src1, $src2, LSR $src3" %}
12310
12311 ins_encode %{
12312 __ eorw(as_Register($dst$$reg),
12313 as_Register($src1$$reg),
12314 as_Register($src2$$reg),
12315 Assembler::LSR,
12316 $src3$$constant & 0x1f);
12317 %}
12318
12319 ins_pipe(ialu_reg_reg_shift);
12320 %}
12321
12322 // This pattern is automatically generated from aarch64_ad.m4
12323 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12324 instruct XorL_reg_URShift_reg(iRegLNoSp dst,
12325 iRegL src1, iRegL src2,
12326 immI src3) %{
12327 match(Set dst (XorL src1 (URShiftL src2 src3)));
12328
12329 ins_cost(1.9 * INSN_COST);
12330 format %{ "eor $dst, $src1, $src2, LSR $src3" %}
12331
12332 ins_encode %{
12333 __ eor(as_Register($dst$$reg),
12334 as_Register($src1$$reg),
12335 as_Register($src2$$reg),
12336 Assembler::LSR,
12337 $src3$$constant & 0x3f);
12338 %}
12339
12340 ins_pipe(ialu_reg_reg_shift);
12341 %}
12342
12343 // This pattern is automatically generated from aarch64_ad.m4
12344 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12345 instruct XorI_reg_RShift_reg(iRegINoSp dst,
12346 iRegIorL2I src1, iRegIorL2I src2,
12347 immI src3) %{
12348 match(Set dst (XorI src1 (RShiftI src2 src3)));
12349
12350 ins_cost(1.9 * INSN_COST);
12351 format %{ "eorw $dst, $src1, $src2, ASR $src3" %}
12352
12353 ins_encode %{
12354 __ eorw(as_Register($dst$$reg),
12355 as_Register($src1$$reg),
12356 as_Register($src2$$reg),
12357 Assembler::ASR,
12358 $src3$$constant & 0x1f);
12359 %}
12360
12361 ins_pipe(ialu_reg_reg_shift);
12362 %}
12363
12364 // This pattern is automatically generated from aarch64_ad.m4
12365 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12366 instruct XorL_reg_RShift_reg(iRegLNoSp dst,
12367 iRegL src1, iRegL src2,
12368 immI src3) %{
12369 match(Set dst (XorL src1 (RShiftL src2 src3)));
12370
12371 ins_cost(1.9 * INSN_COST);
12372 format %{ "eor $dst, $src1, $src2, ASR $src3" %}
12373
12374 ins_encode %{
12375 __ eor(as_Register($dst$$reg),
12376 as_Register($src1$$reg),
12377 as_Register($src2$$reg),
12378 Assembler::ASR,
12379 $src3$$constant & 0x3f);
12380 %}
12381
12382 ins_pipe(ialu_reg_reg_shift);
12383 %}
12384
12385 // This pattern is automatically generated from aarch64_ad.m4
12386 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12387 instruct XorI_reg_LShift_reg(iRegINoSp dst,
12388 iRegIorL2I src1, iRegIorL2I src2,
12389 immI src3) %{
12390 match(Set dst (XorI src1 (LShiftI src2 src3)));
12391
12392 ins_cost(1.9 * INSN_COST);
12393 format %{ "eorw $dst, $src1, $src2, LSL $src3" %}
12394
12395 ins_encode %{
12396 __ eorw(as_Register($dst$$reg),
12397 as_Register($src1$$reg),
12398 as_Register($src2$$reg),
12399 Assembler::LSL,
12400 $src3$$constant & 0x1f);
12401 %}
12402
12403 ins_pipe(ialu_reg_reg_shift);
12404 %}
12405
12406 // This pattern is automatically generated from aarch64_ad.m4
12407 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12408 instruct XorL_reg_LShift_reg(iRegLNoSp dst,
12409 iRegL src1, iRegL src2,
12410 immI src3) %{
12411 match(Set dst (XorL src1 (LShiftL src2 src3)));
12412
12413 ins_cost(1.9 * INSN_COST);
12414 format %{ "eor $dst, $src1, $src2, LSL $src3" %}
12415
12416 ins_encode %{
12417 __ eor(as_Register($dst$$reg),
12418 as_Register($src1$$reg),
12419 as_Register($src2$$reg),
12420 Assembler::LSL,
12421 $src3$$constant & 0x3f);
12422 %}
12423
12424 ins_pipe(ialu_reg_reg_shift);
12425 %}
12426
12427 // This pattern is automatically generated from aarch64_ad.m4
12428 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12429 instruct XorI_reg_RotateRight_reg(iRegINoSp dst,
12430 iRegIorL2I src1, iRegIorL2I src2,
12431 immI src3) %{
12432 match(Set dst (XorI src1 (RotateRight src2 src3)));
12433
12434 ins_cost(1.9 * INSN_COST);
12435 format %{ "eorw $dst, $src1, $src2, ROR $src3" %}
12436
12437 ins_encode %{
12438 __ eorw(as_Register($dst$$reg),
12439 as_Register($src1$$reg),
12440 as_Register($src2$$reg),
12441 Assembler::ROR,
12442 $src3$$constant & 0x1f);
12443 %}
12444
12445 ins_pipe(ialu_reg_reg_shift);
12446 %}
12447
12448 // This pattern is automatically generated from aarch64_ad.m4
12449 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12450 instruct XorL_reg_RotateRight_reg(iRegLNoSp dst,
12451 iRegL src1, iRegL src2,
12452 immI src3) %{
12453 match(Set dst (XorL src1 (RotateRight src2 src3)));
12454
12455 ins_cost(1.9 * INSN_COST);
12456 format %{ "eor $dst, $src1, $src2, ROR $src3" %}
12457
12458 ins_encode %{
12459 __ eor(as_Register($dst$$reg),
12460 as_Register($src1$$reg),
12461 as_Register($src2$$reg),
12462 Assembler::ROR,
12463 $src3$$constant & 0x3f);
12464 %}
12465
12466 ins_pipe(ialu_reg_reg_shift);
12467 %}
12468
12469 // This pattern is automatically generated from aarch64_ad.m4
12470 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12471 instruct OrI_reg_URShift_reg(iRegINoSp dst,
12472 iRegIorL2I src1, iRegIorL2I src2,
12473 immI src3) %{
12474 match(Set dst (OrI src1 (URShiftI src2 src3)));
12475
12476 ins_cost(1.9 * INSN_COST);
12477 format %{ "orrw $dst, $src1, $src2, LSR $src3" %}
12478
12479 ins_encode %{
12480 __ orrw(as_Register($dst$$reg),
12481 as_Register($src1$$reg),
12482 as_Register($src2$$reg),
12483 Assembler::LSR,
12484 $src3$$constant & 0x1f);
12485 %}
12486
12487 ins_pipe(ialu_reg_reg_shift);
12488 %}
12489
12490 // This pattern is automatically generated from aarch64_ad.m4
12491 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12492 instruct OrL_reg_URShift_reg(iRegLNoSp dst,
12493 iRegL src1, iRegL src2,
12494 immI src3) %{
12495 match(Set dst (OrL src1 (URShiftL src2 src3)));
12496
12497 ins_cost(1.9 * INSN_COST);
12498 format %{ "orr $dst, $src1, $src2, LSR $src3" %}
12499
12500 ins_encode %{
12501 __ orr(as_Register($dst$$reg),
12502 as_Register($src1$$reg),
12503 as_Register($src2$$reg),
12504 Assembler::LSR,
12505 $src3$$constant & 0x3f);
12506 %}
12507
12508 ins_pipe(ialu_reg_reg_shift);
12509 %}
12510
12511 // This pattern is automatically generated from aarch64_ad.m4
12512 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12513 instruct OrI_reg_RShift_reg(iRegINoSp dst,
12514 iRegIorL2I src1, iRegIorL2I src2,
12515 immI src3) %{
12516 match(Set dst (OrI src1 (RShiftI src2 src3)));
12517
12518 ins_cost(1.9 * INSN_COST);
12519 format %{ "orrw $dst, $src1, $src2, ASR $src3" %}
12520
12521 ins_encode %{
12522 __ orrw(as_Register($dst$$reg),
12523 as_Register($src1$$reg),
12524 as_Register($src2$$reg),
12525 Assembler::ASR,
12526 $src3$$constant & 0x1f);
12527 %}
12528
12529 ins_pipe(ialu_reg_reg_shift);
12530 %}
12531
12532 // This pattern is automatically generated from aarch64_ad.m4
12533 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12534 instruct OrL_reg_RShift_reg(iRegLNoSp dst,
12535 iRegL src1, iRegL src2,
12536 immI src3) %{
12537 match(Set dst (OrL src1 (RShiftL src2 src3)));
12538
12539 ins_cost(1.9 * INSN_COST);
12540 format %{ "orr $dst, $src1, $src2, ASR $src3" %}
12541
12542 ins_encode %{
12543 __ orr(as_Register($dst$$reg),
12544 as_Register($src1$$reg),
12545 as_Register($src2$$reg),
12546 Assembler::ASR,
12547 $src3$$constant & 0x3f);
12548 %}
12549
12550 ins_pipe(ialu_reg_reg_shift);
12551 %}
12552
12553 // This pattern is automatically generated from aarch64_ad.m4
12554 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12555 instruct OrI_reg_LShift_reg(iRegINoSp dst,
12556 iRegIorL2I src1, iRegIorL2I src2,
12557 immI src3) %{
12558 match(Set dst (OrI src1 (LShiftI src2 src3)));
12559
12560 ins_cost(1.9 * INSN_COST);
12561 format %{ "orrw $dst, $src1, $src2, LSL $src3" %}
12562
12563 ins_encode %{
12564 __ orrw(as_Register($dst$$reg),
12565 as_Register($src1$$reg),
12566 as_Register($src2$$reg),
12567 Assembler::LSL,
12568 $src3$$constant & 0x1f);
12569 %}
12570
12571 ins_pipe(ialu_reg_reg_shift);
12572 %}
12573
12574 // This pattern is automatically generated from aarch64_ad.m4
12575 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12576 instruct OrL_reg_LShift_reg(iRegLNoSp dst,
12577 iRegL src1, iRegL src2,
12578 immI src3) %{
12579 match(Set dst (OrL src1 (LShiftL src2 src3)));
12580
12581 ins_cost(1.9 * INSN_COST);
12582 format %{ "orr $dst, $src1, $src2, LSL $src3" %}
12583
12584 ins_encode %{
12585 __ orr(as_Register($dst$$reg),
12586 as_Register($src1$$reg),
12587 as_Register($src2$$reg),
12588 Assembler::LSL,
12589 $src3$$constant & 0x3f);
12590 %}
12591
12592 ins_pipe(ialu_reg_reg_shift);
12593 %}
12594
12595 // This pattern is automatically generated from aarch64_ad.m4
12596 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12597 instruct OrI_reg_RotateRight_reg(iRegINoSp dst,
12598 iRegIorL2I src1, iRegIorL2I src2,
12599 immI src3) %{
12600 match(Set dst (OrI src1 (RotateRight src2 src3)));
12601
12602 ins_cost(1.9 * INSN_COST);
12603 format %{ "orrw $dst, $src1, $src2, ROR $src3" %}
12604
12605 ins_encode %{
12606 __ orrw(as_Register($dst$$reg),
12607 as_Register($src1$$reg),
12608 as_Register($src2$$reg),
12609 Assembler::ROR,
12610 $src3$$constant & 0x1f);
12611 %}
12612
12613 ins_pipe(ialu_reg_reg_shift);
12614 %}
12615
12616 // This pattern is automatically generated from aarch64_ad.m4
12617 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12618 instruct OrL_reg_RotateRight_reg(iRegLNoSp dst,
12619 iRegL src1, iRegL src2,
12620 immI src3) %{
12621 match(Set dst (OrL src1 (RotateRight src2 src3)));
12622
12623 ins_cost(1.9 * INSN_COST);
12624 format %{ "orr $dst, $src1, $src2, ROR $src3" %}
12625
12626 ins_encode %{
12627 __ orr(as_Register($dst$$reg),
12628 as_Register($src1$$reg),
12629 as_Register($src2$$reg),
12630 Assembler::ROR,
12631 $src3$$constant & 0x3f);
12632 %}
12633
12634 ins_pipe(ialu_reg_reg_shift);
12635 %}
12636
12637 // This pattern is automatically generated from aarch64_ad.m4
12638 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12639 instruct AddI_reg_URShift_reg(iRegINoSp dst,
12640 iRegIorL2I src1, iRegIorL2I src2,
12641 immI src3) %{
12642 match(Set dst (AddI src1 (URShiftI src2 src3)));
12643
12644 ins_cost(1.9 * INSN_COST);
12645 format %{ "addw $dst, $src1, $src2, LSR $src3" %}
12646
12647 ins_encode %{
12648 __ addw(as_Register($dst$$reg),
12649 as_Register($src1$$reg),
12650 as_Register($src2$$reg),
12651 Assembler::LSR,
12652 $src3$$constant & 0x1f);
12653 %}
12654
12655 ins_pipe(ialu_reg_reg_shift);
12656 %}
12657
12658 // This pattern is automatically generated from aarch64_ad.m4
12659 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12660 instruct AddL_reg_URShift_reg(iRegLNoSp dst,
12661 iRegL src1, iRegL src2,
12662 immI src3) %{
12663 match(Set dst (AddL src1 (URShiftL src2 src3)));
12664
12665 ins_cost(1.9 * INSN_COST);
12666 format %{ "add $dst, $src1, $src2, LSR $src3" %}
12667
12668 ins_encode %{
12669 __ add(as_Register($dst$$reg),
12670 as_Register($src1$$reg),
12671 as_Register($src2$$reg),
12672 Assembler::LSR,
12673 $src3$$constant & 0x3f);
12674 %}
12675
12676 ins_pipe(ialu_reg_reg_shift);
12677 %}
12678
12679 // This pattern is automatically generated from aarch64_ad.m4
12680 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12681 instruct AddI_reg_RShift_reg(iRegINoSp dst,
12682 iRegIorL2I src1, iRegIorL2I src2,
12683 immI src3) %{
12684 match(Set dst (AddI src1 (RShiftI src2 src3)));
12685
12686 ins_cost(1.9 * INSN_COST);
12687 format %{ "addw $dst, $src1, $src2, ASR $src3" %}
12688
12689 ins_encode %{
12690 __ addw(as_Register($dst$$reg),
12691 as_Register($src1$$reg),
12692 as_Register($src2$$reg),
12693 Assembler::ASR,
12694 $src3$$constant & 0x1f);
12695 %}
12696
12697 ins_pipe(ialu_reg_reg_shift);
12698 %}
12699
12700 // This pattern is automatically generated from aarch64_ad.m4
12701 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12702 instruct AddL_reg_RShift_reg(iRegLNoSp dst,
12703 iRegL src1, iRegL src2,
12704 immI src3) %{
12705 match(Set dst (AddL src1 (RShiftL src2 src3)));
12706
12707 ins_cost(1.9 * INSN_COST);
12708 format %{ "add $dst, $src1, $src2, ASR $src3" %}
12709
12710 ins_encode %{
12711 __ add(as_Register($dst$$reg),
12712 as_Register($src1$$reg),
12713 as_Register($src2$$reg),
12714 Assembler::ASR,
12715 $src3$$constant & 0x3f);
12716 %}
12717
12718 ins_pipe(ialu_reg_reg_shift);
12719 %}
12720
12721 // This pattern is automatically generated from aarch64_ad.m4
12722 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12723 instruct AddI_reg_LShift_reg(iRegINoSp dst,
12724 iRegIorL2I src1, iRegIorL2I src2,
12725 immI src3) %{
12726 match(Set dst (AddI src1 (LShiftI src2 src3)));
12727
12728 ins_cost(1.9 * INSN_COST);
12729 format %{ "addw $dst, $src1, $src2, LSL $src3" %}
12730
12731 ins_encode %{
12732 __ addw(as_Register($dst$$reg),
12733 as_Register($src1$$reg),
12734 as_Register($src2$$reg),
12735 Assembler::LSL,
12736 $src3$$constant & 0x1f);
12737 %}
12738
12739 ins_pipe(ialu_reg_reg_shift);
12740 %}
12741
12742 // This pattern is automatically generated from aarch64_ad.m4
12743 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12744 instruct AddL_reg_LShift_reg(iRegLNoSp dst,
12745 iRegL src1, iRegL src2,
12746 immI src3) %{
12747 match(Set dst (AddL src1 (LShiftL src2 src3)));
12748
12749 ins_cost(1.9 * INSN_COST);
12750 format %{ "add $dst, $src1, $src2, LSL $src3" %}
12751
12752 ins_encode %{
12753 __ add(as_Register($dst$$reg),
12754 as_Register($src1$$reg),
12755 as_Register($src2$$reg),
12756 Assembler::LSL,
12757 $src3$$constant & 0x3f);
12758 %}
12759
12760 ins_pipe(ialu_reg_reg_shift);
12761 %}
12762
12763 // This pattern is automatically generated from aarch64_ad.m4
12764 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12765 instruct SubI_reg_URShift_reg(iRegINoSp dst,
12766 iRegIorL2I src1, iRegIorL2I src2,
12767 immI src3) %{
12768 match(Set dst (SubI src1 (URShiftI src2 src3)));
12769
12770 ins_cost(1.9 * INSN_COST);
12771 format %{ "subw $dst, $src1, $src2, LSR $src3" %}
12772
12773 ins_encode %{
12774 __ subw(as_Register($dst$$reg),
12775 as_Register($src1$$reg),
12776 as_Register($src2$$reg),
12777 Assembler::LSR,
12778 $src3$$constant & 0x1f);
12779 %}
12780
12781 ins_pipe(ialu_reg_reg_shift);
12782 %}
12783
12784 // This pattern is automatically generated from aarch64_ad.m4
12785 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12786 instruct SubL_reg_URShift_reg(iRegLNoSp dst,
12787 iRegL src1, iRegL src2,
12788 immI src3) %{
12789 match(Set dst (SubL src1 (URShiftL src2 src3)));
12790
12791 ins_cost(1.9 * INSN_COST);
12792 format %{ "sub $dst, $src1, $src2, LSR $src3" %}
12793
12794 ins_encode %{
12795 __ sub(as_Register($dst$$reg),
12796 as_Register($src1$$reg),
12797 as_Register($src2$$reg),
12798 Assembler::LSR,
12799 $src3$$constant & 0x3f);
12800 %}
12801
12802 ins_pipe(ialu_reg_reg_shift);
12803 %}
12804
12805 // This pattern is automatically generated from aarch64_ad.m4
12806 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12807 instruct SubI_reg_RShift_reg(iRegINoSp dst,
12808 iRegIorL2I src1, iRegIorL2I src2,
12809 immI src3) %{
12810 match(Set dst (SubI src1 (RShiftI src2 src3)));
12811
12812 ins_cost(1.9 * INSN_COST);
12813 format %{ "subw $dst, $src1, $src2, ASR $src3" %}
12814
12815 ins_encode %{
12816 __ subw(as_Register($dst$$reg),
12817 as_Register($src1$$reg),
12818 as_Register($src2$$reg),
12819 Assembler::ASR,
12820 $src3$$constant & 0x1f);
12821 %}
12822
12823 ins_pipe(ialu_reg_reg_shift);
12824 %}
12825
12826 // This pattern is automatically generated from aarch64_ad.m4
12827 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12828 instruct SubL_reg_RShift_reg(iRegLNoSp dst,
12829 iRegL src1, iRegL src2,
12830 immI src3) %{
12831 match(Set dst (SubL src1 (RShiftL src2 src3)));
12832
12833 ins_cost(1.9 * INSN_COST);
12834 format %{ "sub $dst, $src1, $src2, ASR $src3" %}
12835
12836 ins_encode %{
12837 __ sub(as_Register($dst$$reg),
12838 as_Register($src1$$reg),
12839 as_Register($src2$$reg),
12840 Assembler::ASR,
12841 $src3$$constant & 0x3f);
12842 %}
12843
12844 ins_pipe(ialu_reg_reg_shift);
12845 %}
12846
12847 // This pattern is automatically generated from aarch64_ad.m4
12848 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12849 instruct SubI_reg_LShift_reg(iRegINoSp dst,
12850 iRegIorL2I src1, iRegIorL2I src2,
12851 immI src3) %{
12852 match(Set dst (SubI src1 (LShiftI src2 src3)));
12853
12854 ins_cost(1.9 * INSN_COST);
12855 format %{ "subw $dst, $src1, $src2, LSL $src3" %}
12856
12857 ins_encode %{
12858 __ subw(as_Register($dst$$reg),
12859 as_Register($src1$$reg),
12860 as_Register($src2$$reg),
12861 Assembler::LSL,
12862 $src3$$constant & 0x1f);
12863 %}
12864
12865 ins_pipe(ialu_reg_reg_shift);
12866 %}
12867
12868 // This pattern is automatically generated from aarch64_ad.m4
12869 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12870 instruct SubL_reg_LShift_reg(iRegLNoSp dst,
12871 iRegL src1, iRegL src2,
12872 immI src3) %{
12873 match(Set dst (SubL src1 (LShiftL src2 src3)));
12874
12875 ins_cost(1.9 * INSN_COST);
12876 format %{ "sub $dst, $src1, $src2, LSL $src3" %}
12877
12878 ins_encode %{
12879 __ sub(as_Register($dst$$reg),
12880 as_Register($src1$$reg),
12881 as_Register($src2$$reg),
12882 Assembler::LSL,
12883 $src3$$constant & 0x3f);
12884 %}
12885
12886 ins_pipe(ialu_reg_reg_shift);
12887 %}
12888
12889 // This pattern is automatically generated from aarch64_ad.m4
12890 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12891
12892 // Shift Left followed by Shift Right.
12893 // This idiom is used by the compiler for the i2b bytecode etc.
12894 instruct sbfmL(iRegLNoSp dst, iRegL src, immI lshift_count, immI rshift_count)
12895 %{
12896 match(Set dst (RShiftL (LShiftL src lshift_count) rshift_count));
12897 ins_cost(INSN_COST * 2);
12898 format %{ "sbfm $dst, $src, $rshift_count - $lshift_count, #63 - $lshift_count" %}
12899 ins_encode %{
12900 int lshift = $lshift_count$$constant & 63;
12901 int rshift = $rshift_count$$constant & 63;
12902 int s = 63 - lshift;
12903 int r = (rshift - lshift) & 63;
12904 __ sbfm(as_Register($dst$$reg),
12905 as_Register($src$$reg),
12906 r, s);
12907 %}
12908
12909 ins_pipe(ialu_reg_shift);
12910 %}
12911
12912 // This pattern is automatically generated from aarch64_ad.m4
12913 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12914
12915 // Shift Left followed by Shift Right.
12916 // This idiom is used by the compiler for the i2b bytecode etc.
12917 instruct sbfmwI(iRegINoSp dst, iRegIorL2I src, immI lshift_count, immI rshift_count)
12918 %{
12919 match(Set dst (RShiftI (LShiftI src lshift_count) rshift_count));
12920 ins_cost(INSN_COST * 2);
12921 format %{ "sbfmw $dst, $src, $rshift_count - $lshift_count, #31 - $lshift_count" %}
12922 ins_encode %{
12923 int lshift = $lshift_count$$constant & 31;
12924 int rshift = $rshift_count$$constant & 31;
12925 int s = 31 - lshift;
12926 int r = (rshift - lshift) & 31;
12927 __ sbfmw(as_Register($dst$$reg),
12928 as_Register($src$$reg),
12929 r, s);
12930 %}
12931
12932 ins_pipe(ialu_reg_shift);
12933 %}
12934
12935 // This pattern is automatically generated from aarch64_ad.m4
12936 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12937
12938 // Shift Left followed by Shift Right.
12939 // This idiom is used by the compiler for the i2b bytecode etc.
12940 instruct ubfmL(iRegLNoSp dst, iRegL src, immI lshift_count, immI rshift_count)
12941 %{
12942 match(Set dst (URShiftL (LShiftL src lshift_count) rshift_count));
12943 ins_cost(INSN_COST * 2);
12944 format %{ "ubfm $dst, $src, $rshift_count - $lshift_count, #63 - $lshift_count" %}
12945 ins_encode %{
12946 int lshift = $lshift_count$$constant & 63;
12947 int rshift = $rshift_count$$constant & 63;
12948 int s = 63 - lshift;
12949 int r = (rshift - lshift) & 63;
12950 __ ubfm(as_Register($dst$$reg),
12951 as_Register($src$$reg),
12952 r, s);
12953 %}
12954
12955 ins_pipe(ialu_reg_shift);
12956 %}
12957
12958 // This pattern is automatically generated from aarch64_ad.m4
12959 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12960
12961 // Shift Left followed by Shift Right.
12962 // This idiom is used by the compiler for the i2b bytecode etc.
12963 instruct ubfmwI(iRegINoSp dst, iRegIorL2I src, immI lshift_count, immI rshift_count)
12964 %{
12965 match(Set dst (URShiftI (LShiftI src lshift_count) rshift_count));
12966 ins_cost(INSN_COST * 2);
12967 format %{ "ubfmw $dst, $src, $rshift_count - $lshift_count, #31 - $lshift_count" %}
12968 ins_encode %{
12969 int lshift = $lshift_count$$constant & 31;
12970 int rshift = $rshift_count$$constant & 31;
12971 int s = 31 - lshift;
12972 int r = (rshift - lshift) & 31;
12973 __ ubfmw(as_Register($dst$$reg),
12974 as_Register($src$$reg),
12975 r, s);
12976 %}
12977
12978 ins_pipe(ialu_reg_shift);
12979 %}
12980
12981 // Bitfield extract with shift & mask
12982
12983 // This pattern is automatically generated from aarch64_ad.m4
12984 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12985 instruct ubfxwI(iRegINoSp dst, iRegIorL2I src, immI rshift, immI_bitmask mask)
12986 %{
12987 match(Set dst (AndI (URShiftI src rshift) mask));
12988 // Make sure we are not going to exceed what ubfxw can do.
12989 predicate((exact_log2(n->in(2)->get_int() + 1) + (n->in(1)->in(2)->get_int() & 31)) <= (31 + 1));
12990
12991 ins_cost(INSN_COST);
12992 format %{ "ubfxw $dst, $src, $rshift, $mask" %}
12993 ins_encode %{
12994 int rshift = $rshift$$constant & 31;
12995 intptr_t mask = $mask$$constant;
12996 int width = exact_log2(mask+1);
12997 __ ubfxw(as_Register($dst$$reg),
12998 as_Register($src$$reg), rshift, width);
12999 %}
13000 ins_pipe(ialu_reg_shift);
13001 %}
13002
13003 // This pattern is automatically generated from aarch64_ad.m4
13004 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13005 instruct ubfxL(iRegLNoSp dst, iRegL src, immI rshift, immL_bitmask mask)
13006 %{
13007 match(Set dst (AndL (URShiftL src rshift) mask));
13008 // Make sure we are not going to exceed what ubfx can do.
13009 predicate((exact_log2_long(n->in(2)->get_long() + 1) + (n->in(1)->in(2)->get_int() & 63)) <= (63 + 1));
13010
13011 ins_cost(INSN_COST);
13012 format %{ "ubfx $dst, $src, $rshift, $mask" %}
13013 ins_encode %{
13014 int rshift = $rshift$$constant & 63;
13015 intptr_t mask = $mask$$constant;
13016 int width = exact_log2_long(mask+1);
13017 __ ubfx(as_Register($dst$$reg),
13018 as_Register($src$$reg), rshift, width);
13019 %}
13020 ins_pipe(ialu_reg_shift);
13021 %}
13022
13023
13024 // This pattern is automatically generated from aarch64_ad.m4
13025 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13026
13027 // We can use ubfx when extending an And with a mask when we know mask
13028 // is positive. We know that because immI_bitmask guarantees it.
13029 instruct ubfxIConvI2L(iRegLNoSp dst, iRegIorL2I src, immI rshift, immI_bitmask mask)
13030 %{
13031 match(Set dst (ConvI2L (AndI (URShiftI src rshift) mask)));
13032 // Make sure we are not going to exceed what ubfxw can do.
13033 predicate((exact_log2(n->in(1)->in(2)->get_int() + 1) + (n->in(1)->in(1)->in(2)->get_int() & 31)) <= (31 + 1));
13034
13035 ins_cost(INSN_COST * 2);
13036 format %{ "ubfx $dst, $src, $rshift, $mask" %}
13037 ins_encode %{
13038 int rshift = $rshift$$constant & 31;
13039 intptr_t mask = $mask$$constant;
13040 int width = exact_log2(mask+1);
13041 __ ubfx(as_Register($dst$$reg),
13042 as_Register($src$$reg), rshift, width);
13043 %}
13044 ins_pipe(ialu_reg_shift);
13045 %}
13046
13047
13048 // This pattern is automatically generated from aarch64_ad.m4
13049 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13050
13051 // We can use ubfiz when masking by a positive number and then left shifting the result.
13052 // We know that the mask is positive because immI_bitmask guarantees it.
13053 instruct ubfizwI(iRegINoSp dst, iRegIorL2I src, immI lshift, immI_bitmask mask)
13054 %{
13055 match(Set dst (LShiftI (AndI src mask) lshift));
13056 predicate((exact_log2(n->in(1)->in(2)->get_int() + 1) + (n->in(2)->get_int() & 31)) <= (31 + 1));
13057
13058 ins_cost(INSN_COST);
13059 format %{ "ubfizw $dst, $src, $lshift, $mask" %}
13060 ins_encode %{
13061 int lshift = $lshift$$constant & 31;
13062 intptr_t mask = $mask$$constant;
13063 int width = exact_log2(mask+1);
13064 __ ubfizw(as_Register($dst$$reg),
13065 as_Register($src$$reg), lshift, width);
13066 %}
13067 ins_pipe(ialu_reg_shift);
13068 %}
13069
13070 // This pattern is automatically generated from aarch64_ad.m4
13071 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13072
13073 // We can use ubfiz when masking by a positive number and then left shifting the result.
13074 // We know that the mask is positive because immL_bitmask guarantees it.
13075 instruct ubfizL(iRegLNoSp dst, iRegL src, immI lshift, immL_bitmask mask)
13076 %{
13077 match(Set dst (LShiftL (AndL src mask) lshift));
13078 predicate((exact_log2_long(n->in(1)->in(2)->get_long() + 1) + (n->in(2)->get_int() & 63)) <= (63 + 1));
13079
13080 ins_cost(INSN_COST);
13081 format %{ "ubfiz $dst, $src, $lshift, $mask" %}
13082 ins_encode %{
13083 int lshift = $lshift$$constant & 63;
13084 intptr_t mask = $mask$$constant;
13085 int width = exact_log2_long(mask+1);
13086 __ ubfiz(as_Register($dst$$reg),
13087 as_Register($src$$reg), lshift, width);
13088 %}
13089 ins_pipe(ialu_reg_shift);
13090 %}
13091
13092 // This pattern is automatically generated from aarch64_ad.m4
13093 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13094
13095 // We can use ubfiz when masking by a positive number and then left shifting the result.
13096 // We know that the mask is positive because immI_bitmask guarantees it.
13097 instruct ubfizwIConvI2L(iRegLNoSp dst, iRegIorL2I src, immI lshift, immI_bitmask mask)
13098 %{
13099 match(Set dst (ConvI2L (LShiftI (AndI src mask) lshift)));
13100 predicate((exact_log2(n->in(1)->in(1)->in(2)->get_int() + 1) + (n->in(1)->in(2)->get_int() & 31)) <= 31);
13101
13102 ins_cost(INSN_COST);
13103 format %{ "ubfizw $dst, $src, $lshift, $mask" %}
13104 ins_encode %{
13105 int lshift = $lshift$$constant & 31;
13106 intptr_t mask = $mask$$constant;
13107 int width = exact_log2(mask+1);
13108 __ ubfizw(as_Register($dst$$reg),
13109 as_Register($src$$reg), lshift, width);
13110 %}
13111 ins_pipe(ialu_reg_shift);
13112 %}
13113
13114 // This pattern is automatically generated from aarch64_ad.m4
13115 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13116
13117 // We can use ubfiz when masking by a positive number and then left shifting the result.
13118 // We know that the mask is positive because immL_bitmask guarantees it.
13119 instruct ubfizLConvL2I(iRegINoSp dst, iRegL src, immI lshift, immL_positive_bitmaskI mask)
13120 %{
13121 match(Set dst (ConvL2I (LShiftL (AndL src mask) lshift)));
13122 predicate((exact_log2_long(n->in(1)->in(1)->in(2)->get_long() + 1) + (n->in(1)->in(2)->get_int() & 63)) <= 31);
13123
13124 ins_cost(INSN_COST);
13125 format %{ "ubfiz $dst, $src, $lshift, $mask" %}
13126 ins_encode %{
13127 int lshift = $lshift$$constant & 63;
13128 intptr_t mask = $mask$$constant;
13129 int width = exact_log2_long(mask+1);
13130 __ ubfiz(as_Register($dst$$reg),
13131 as_Register($src$$reg), lshift, width);
13132 %}
13133 ins_pipe(ialu_reg_shift);
13134 %}
13135
13136
13137 // This pattern is automatically generated from aarch64_ad.m4
13138 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13139
13140 // If there is a convert I to L block between and AndI and a LShiftL, we can also match ubfiz
13141 instruct ubfizIConvI2L(iRegLNoSp dst, iRegIorL2I src, immI lshift, immI_bitmask mask)
13142 %{
13143 match(Set dst (LShiftL (ConvI2L (AndI src mask)) lshift));
13144 predicate((exact_log2(n->in(1)->in(1)->in(2)->get_int() + 1) + (n->in(2)->get_int() & 63)) <= (63 + 1));
13145
13146 ins_cost(INSN_COST);
13147 format %{ "ubfiz $dst, $src, $lshift, $mask" %}
13148 ins_encode %{
13149 int lshift = $lshift$$constant & 63;
13150 intptr_t mask = $mask$$constant;
13151 int width = exact_log2(mask+1);
13152 __ ubfiz(as_Register($dst$$reg),
13153 as_Register($src$$reg), lshift, width);
13154 %}
13155 ins_pipe(ialu_reg_shift);
13156 %}
13157
13158 // This pattern is automatically generated from aarch64_ad.m4
13159 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13160
13161 // If there is a convert L to I block between and AndL and a LShiftI, we can also match ubfiz
13162 instruct ubfizLConvL2Ix(iRegINoSp dst, iRegL src, immI lshift, immL_positive_bitmaskI mask)
13163 %{
13164 match(Set dst (LShiftI (ConvL2I (AndL src mask)) lshift));
13165 predicate((exact_log2_long(n->in(1)->in(1)->in(2)->get_long() + 1) + (n->in(2)->get_int() & 31)) <= 31);
13166
13167 ins_cost(INSN_COST);
13168 format %{ "ubfiz $dst, $src, $lshift, $mask" %}
13169 ins_encode %{
13170 int lshift = $lshift$$constant & 31;
13171 intptr_t mask = $mask$$constant;
13172 int width = exact_log2(mask+1);
13173 __ ubfiz(as_Register($dst$$reg),
13174 as_Register($src$$reg), lshift, width);
13175 %}
13176 ins_pipe(ialu_reg_shift);
13177 %}
13178
13179 // This pattern is automatically generated from aarch64_ad.m4
13180 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13181
13182 // Can skip int2long conversions after AND with small bitmask
13183 instruct ubfizIConvI2LAndI(iRegLNoSp dst, iRegI src, immI_bitmask msk)
13184 %{
13185 match(Set dst (ConvI2L (AndI src msk)));
13186 ins_cost(INSN_COST);
13187 format %{ "ubfiz $dst, $src, 0, exact_log2($msk + 1) " %}
13188 ins_encode %{
13189 __ ubfiz(as_Register($dst$$reg), as_Register($src$$reg), 0, exact_log2($msk$$constant + 1));
13190 %}
13191 ins_pipe(ialu_reg_shift);
13192 %}
13193
13194
13195 // Rotations
13196
13197 // This pattern is automatically generated from aarch64_ad.m4
13198 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13199 instruct extrOrL(iRegLNoSp dst, iRegL src1, iRegL src2, immI lshift, immI rshift, rFlagsReg cr)
13200 %{
13201 match(Set dst (OrL (LShiftL src1 lshift) (URShiftL src2 rshift)));
13202 predicate(0 == (((n->in(1)->in(2)->get_int() & 63) + (n->in(2)->in(2)->get_int() & 63)) & 63));
13203
13204 ins_cost(INSN_COST);
13205 format %{ "extr $dst, $src1, $src2, #$rshift" %}
13206
13207 ins_encode %{
13208 __ extr(as_Register($dst$$reg), as_Register($src1$$reg), as_Register($src2$$reg),
13209 $rshift$$constant & 63);
13210 %}
13211 ins_pipe(ialu_reg_reg_extr);
13212 %}
13213
13214
13215 // This pattern is automatically generated from aarch64_ad.m4
13216 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13217 instruct extrOrI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI lshift, immI rshift, rFlagsReg cr)
13218 %{
13219 match(Set dst (OrI (LShiftI src1 lshift) (URShiftI src2 rshift)));
13220 predicate(0 == (((n->in(1)->in(2)->get_int() & 31) + (n->in(2)->in(2)->get_int() & 31)) & 31));
13221
13222 ins_cost(INSN_COST);
13223 format %{ "extr $dst, $src1, $src2, #$rshift" %}
13224
13225 ins_encode %{
13226 __ extrw(as_Register($dst$$reg), as_Register($src1$$reg), as_Register($src2$$reg),
13227 $rshift$$constant & 31);
13228 %}
13229 ins_pipe(ialu_reg_reg_extr);
13230 %}
13231
13232
13233 // This pattern is automatically generated from aarch64_ad.m4
13234 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13235 instruct extrAddL(iRegLNoSp dst, iRegL src1, iRegL src2, immI lshift, immI rshift, rFlagsReg cr)
13236 %{
13237 match(Set dst (AddL (LShiftL src1 lshift) (URShiftL src2 rshift)));
13238 predicate(0 == (((n->in(1)->in(2)->get_int() & 63) + (n->in(2)->in(2)->get_int() & 63)) & 63));
13239
13240 ins_cost(INSN_COST);
13241 format %{ "extr $dst, $src1, $src2, #$rshift" %}
13242
13243 ins_encode %{
13244 __ extr(as_Register($dst$$reg), as_Register($src1$$reg), as_Register($src2$$reg),
13245 $rshift$$constant & 63);
13246 %}
13247 ins_pipe(ialu_reg_reg_extr);
13248 %}
13249
13250
13251 // This pattern is automatically generated from aarch64_ad.m4
13252 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13253 instruct extrAddI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI lshift, immI rshift, rFlagsReg cr)
13254 %{
13255 match(Set dst (AddI (LShiftI src1 lshift) (URShiftI src2 rshift)));
13256 predicate(0 == (((n->in(1)->in(2)->get_int() & 31) + (n->in(2)->in(2)->get_int() & 31)) & 31));
13257
13258 ins_cost(INSN_COST);
13259 format %{ "extr $dst, $src1, $src2, #$rshift" %}
13260
13261 ins_encode %{
13262 __ extrw(as_Register($dst$$reg), as_Register($src1$$reg), as_Register($src2$$reg),
13263 $rshift$$constant & 31);
13264 %}
13265 ins_pipe(ialu_reg_reg_extr);
13266 %}
13267
13268 // This pattern is automatically generated from aarch64_ad.m4
13269 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13270 instruct rorI_imm(iRegINoSp dst, iRegI src, immI shift)
13271 %{
13272 match(Set dst (RotateRight src shift));
13273
13274 ins_cost(INSN_COST);
13275 format %{ "ror $dst, $src, $shift" %}
13276
13277 ins_encode %{
13278 __ extrw(as_Register($dst$$reg), as_Register($src$$reg), as_Register($src$$reg),
13279 $shift$$constant & 0x1f);
13280 %}
13281 ins_pipe(ialu_reg_reg_vshift);
13282 %}
13283
13284 // This pattern is automatically generated from aarch64_ad.m4
13285 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13286 instruct rorL_imm(iRegLNoSp dst, iRegL src, immI shift)
13287 %{
13288 match(Set dst (RotateRight src shift));
13289
13290 ins_cost(INSN_COST);
13291 format %{ "ror $dst, $src, $shift" %}
13292
13293 ins_encode %{
13294 __ extr(as_Register($dst$$reg), as_Register($src$$reg), as_Register($src$$reg),
13295 $shift$$constant & 0x3f);
13296 %}
13297 ins_pipe(ialu_reg_reg_vshift);
13298 %}
13299
13300 // This pattern is automatically generated from aarch64_ad.m4
13301 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13302 instruct rorI_reg(iRegINoSp dst, iRegI src, iRegI shift)
13303 %{
13304 match(Set dst (RotateRight src shift));
13305
13306 ins_cost(INSN_COST);
13307 format %{ "ror $dst, $src, $shift" %}
13308
13309 ins_encode %{
13310 __ rorvw(as_Register($dst$$reg), as_Register($src$$reg), as_Register($shift$$reg));
13311 %}
13312 ins_pipe(ialu_reg_reg_vshift);
13313 %}
13314
13315 // This pattern is automatically generated from aarch64_ad.m4
13316 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13317 instruct rorL_reg(iRegLNoSp dst, iRegL src, iRegI shift)
13318 %{
13319 match(Set dst (RotateRight src shift));
13320
13321 ins_cost(INSN_COST);
13322 format %{ "ror $dst, $src, $shift" %}
13323
13324 ins_encode %{
13325 __ rorv(as_Register($dst$$reg), as_Register($src$$reg), as_Register($shift$$reg));
13326 %}
13327 ins_pipe(ialu_reg_reg_vshift);
13328 %}
13329
13330 // This pattern is automatically generated from aarch64_ad.m4
13331 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13332 instruct rolI_reg(iRegINoSp dst, iRegI src, iRegI shift)
13333 %{
13334 match(Set dst (RotateLeft src shift));
13335
13336 ins_cost(INSN_COST);
13337 format %{ "rol $dst, $src, $shift" %}
13338
13339 ins_encode %{
13340 __ subw(rscratch1, zr, as_Register($shift$$reg));
13341 __ rorvw(as_Register($dst$$reg), as_Register($src$$reg), rscratch1);
13342 %}
13343 ins_pipe(ialu_reg_reg_vshift);
13344 %}
13345
13346 // This pattern is automatically generated from aarch64_ad.m4
13347 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13348 instruct rolL_reg(iRegLNoSp dst, iRegL src, iRegI shift)
13349 %{
13350 match(Set dst (RotateLeft src shift));
13351
13352 ins_cost(INSN_COST);
13353 format %{ "rol $dst, $src, $shift" %}
13354
13355 ins_encode %{
13356 __ subw(rscratch1, zr, as_Register($shift$$reg));
13357 __ rorv(as_Register($dst$$reg), as_Register($src$$reg), rscratch1);
13358 %}
13359 ins_pipe(ialu_reg_reg_vshift);
13360 %}
13361
13362
13363 // Add/subtract (extended)
13364
13365 // This pattern is automatically generated from aarch64_ad.m4
13366 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13367 instruct AddExtI(iRegLNoSp dst, iRegL src1, iRegIorL2I src2, rFlagsReg cr)
13368 %{
13369 match(Set dst (AddL src1 (ConvI2L src2)));
13370 ins_cost(INSN_COST);
13371 format %{ "add $dst, $src1, $src2, sxtw" %}
13372
13373 ins_encode %{
13374 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13375 as_Register($src2$$reg), ext::sxtw);
13376 %}
13377 ins_pipe(ialu_reg_reg);
13378 %}
13379
13380 // This pattern is automatically generated from aarch64_ad.m4
13381 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13382 instruct SubExtI(iRegLNoSp dst, iRegL src1, iRegIorL2I src2, rFlagsReg cr)
13383 %{
13384 match(Set dst (SubL src1 (ConvI2L src2)));
13385 ins_cost(INSN_COST);
13386 format %{ "sub $dst, $src1, $src2, sxtw" %}
13387
13388 ins_encode %{
13389 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13390 as_Register($src2$$reg), ext::sxtw);
13391 %}
13392 ins_pipe(ialu_reg_reg);
13393 %}
13394
13395 // This pattern is automatically generated from aarch64_ad.m4
13396 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13397 instruct AddExtI_sxth(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_16 lshift, immI_16 rshift, rFlagsReg cr)
13398 %{
13399 match(Set dst (AddI src1 (RShiftI (LShiftI src2 lshift) rshift)));
13400 ins_cost(INSN_COST);
13401 format %{ "add $dst, $src1, $src2, sxth" %}
13402
13403 ins_encode %{
13404 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13405 as_Register($src2$$reg), ext::sxth);
13406 %}
13407 ins_pipe(ialu_reg_reg);
13408 %}
13409
13410 // This pattern is automatically generated from aarch64_ad.m4
13411 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13412 instruct AddExtI_sxtb(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_24 lshift, immI_24 rshift, rFlagsReg cr)
13413 %{
13414 match(Set dst (AddI src1 (RShiftI (LShiftI src2 lshift) rshift)));
13415 ins_cost(INSN_COST);
13416 format %{ "add $dst, $src1, $src2, sxtb" %}
13417
13418 ins_encode %{
13419 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13420 as_Register($src2$$reg), ext::sxtb);
13421 %}
13422 ins_pipe(ialu_reg_reg);
13423 %}
13424
13425 // This pattern is automatically generated from aarch64_ad.m4
13426 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13427 instruct AddExtI_uxtb(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_24 lshift, immI_24 rshift, rFlagsReg cr)
13428 %{
13429 match(Set dst (AddI src1 (URShiftI (LShiftI src2 lshift) rshift)));
13430 ins_cost(INSN_COST);
13431 format %{ "add $dst, $src1, $src2, uxtb" %}
13432
13433 ins_encode %{
13434 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13435 as_Register($src2$$reg), ext::uxtb);
13436 %}
13437 ins_pipe(ialu_reg_reg);
13438 %}
13439
13440 // This pattern is automatically generated from aarch64_ad.m4
13441 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13442 instruct AddExtL_sxth(iRegLNoSp dst, iRegL src1, iRegL src2, immI_48 lshift, immI_48 rshift, rFlagsReg cr)
13443 %{
13444 match(Set dst (AddL src1 (RShiftL (LShiftL src2 lshift) rshift)));
13445 ins_cost(INSN_COST);
13446 format %{ "add $dst, $src1, $src2, sxth" %}
13447
13448 ins_encode %{
13449 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13450 as_Register($src2$$reg), ext::sxth);
13451 %}
13452 ins_pipe(ialu_reg_reg);
13453 %}
13454
13455 // This pattern is automatically generated from aarch64_ad.m4
13456 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13457 instruct AddExtL_sxtw(iRegLNoSp dst, iRegL src1, iRegL src2, immI_32 lshift, immI_32 rshift, rFlagsReg cr)
13458 %{
13459 match(Set dst (AddL src1 (RShiftL (LShiftL src2 lshift) rshift)));
13460 ins_cost(INSN_COST);
13461 format %{ "add $dst, $src1, $src2, sxtw" %}
13462
13463 ins_encode %{
13464 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13465 as_Register($src2$$reg), ext::sxtw);
13466 %}
13467 ins_pipe(ialu_reg_reg);
13468 %}
13469
13470 // This pattern is automatically generated from aarch64_ad.m4
13471 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13472 instruct AddExtL_sxtb(iRegLNoSp dst, iRegL src1, iRegL src2, immI_56 lshift, immI_56 rshift, rFlagsReg cr)
13473 %{
13474 match(Set dst (AddL src1 (RShiftL (LShiftL src2 lshift) rshift)));
13475 ins_cost(INSN_COST);
13476 format %{ "add $dst, $src1, $src2, sxtb" %}
13477
13478 ins_encode %{
13479 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13480 as_Register($src2$$reg), ext::sxtb);
13481 %}
13482 ins_pipe(ialu_reg_reg);
13483 %}
13484
13485 // This pattern is automatically generated from aarch64_ad.m4
13486 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13487 instruct AddExtL_uxtb(iRegLNoSp dst, iRegL src1, iRegL src2, immI_56 lshift, immI_56 rshift, rFlagsReg cr)
13488 %{
13489 match(Set dst (AddL src1 (URShiftL (LShiftL src2 lshift) rshift)));
13490 ins_cost(INSN_COST);
13491 format %{ "add $dst, $src1, $src2, uxtb" %}
13492
13493 ins_encode %{
13494 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13495 as_Register($src2$$reg), ext::uxtb);
13496 %}
13497 ins_pipe(ialu_reg_reg);
13498 %}
13499
13500 // This pattern is automatically generated from aarch64_ad.m4
13501 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13502 instruct AddExtI_uxtb_and(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_255 mask, rFlagsReg cr)
13503 %{
13504 match(Set dst (AddI src1 (AndI src2 mask)));
13505 ins_cost(INSN_COST);
13506 format %{ "addw $dst, $src1, $src2, uxtb" %}
13507
13508 ins_encode %{
13509 __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
13510 as_Register($src2$$reg), ext::uxtb);
13511 %}
13512 ins_pipe(ialu_reg_reg);
13513 %}
13514
13515 // This pattern is automatically generated from aarch64_ad.m4
13516 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13517 instruct AddExtI_uxth_and(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_65535 mask, rFlagsReg cr)
13518 %{
13519 match(Set dst (AddI src1 (AndI src2 mask)));
13520 ins_cost(INSN_COST);
13521 format %{ "addw $dst, $src1, $src2, uxth" %}
13522
13523 ins_encode %{
13524 __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
13525 as_Register($src2$$reg), ext::uxth);
13526 %}
13527 ins_pipe(ialu_reg_reg);
13528 %}
13529
13530 // This pattern is automatically generated from aarch64_ad.m4
13531 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13532 instruct AddExtL_uxtb_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_255 mask, rFlagsReg cr)
13533 %{
13534 match(Set dst (AddL src1 (AndL src2 mask)));
13535 ins_cost(INSN_COST);
13536 format %{ "add $dst, $src1, $src2, uxtb" %}
13537
13538 ins_encode %{
13539 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13540 as_Register($src2$$reg), ext::uxtb);
13541 %}
13542 ins_pipe(ialu_reg_reg);
13543 %}
13544
13545 // This pattern is automatically generated from aarch64_ad.m4
13546 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13547 instruct AddExtL_uxth_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_65535 mask, rFlagsReg cr)
13548 %{
13549 match(Set dst (AddL src1 (AndL src2 mask)));
13550 ins_cost(INSN_COST);
13551 format %{ "add $dst, $src1, $src2, uxth" %}
13552
13553 ins_encode %{
13554 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13555 as_Register($src2$$reg), ext::uxth);
13556 %}
13557 ins_pipe(ialu_reg_reg);
13558 %}
13559
13560 // This pattern is automatically generated from aarch64_ad.m4
13561 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13562 instruct AddExtL_uxtw_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_4294967295 mask, rFlagsReg cr)
13563 %{
13564 match(Set dst (AddL src1 (AndL src2 mask)));
13565 ins_cost(INSN_COST);
13566 format %{ "add $dst, $src1, $src2, uxtw" %}
13567
13568 ins_encode %{
13569 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13570 as_Register($src2$$reg), ext::uxtw);
13571 %}
13572 ins_pipe(ialu_reg_reg);
13573 %}
13574
13575 // This pattern is automatically generated from aarch64_ad.m4
13576 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13577 instruct SubExtI_uxtb_and(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_255 mask, rFlagsReg cr)
13578 %{
13579 match(Set dst (SubI src1 (AndI src2 mask)));
13580 ins_cost(INSN_COST);
13581 format %{ "subw $dst, $src1, $src2, uxtb" %}
13582
13583 ins_encode %{
13584 __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
13585 as_Register($src2$$reg), ext::uxtb);
13586 %}
13587 ins_pipe(ialu_reg_reg);
13588 %}
13589
13590 // This pattern is automatically generated from aarch64_ad.m4
13591 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13592 instruct SubExtI_uxth_and(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_65535 mask, rFlagsReg cr)
13593 %{
13594 match(Set dst (SubI src1 (AndI src2 mask)));
13595 ins_cost(INSN_COST);
13596 format %{ "subw $dst, $src1, $src2, uxth" %}
13597
13598 ins_encode %{
13599 __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
13600 as_Register($src2$$reg), ext::uxth);
13601 %}
13602 ins_pipe(ialu_reg_reg);
13603 %}
13604
13605 // This pattern is automatically generated from aarch64_ad.m4
13606 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13607 instruct SubExtL_uxtb_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_255 mask, rFlagsReg cr)
13608 %{
13609 match(Set dst (SubL src1 (AndL src2 mask)));
13610 ins_cost(INSN_COST);
13611 format %{ "sub $dst, $src1, $src2, uxtb" %}
13612
13613 ins_encode %{
13614 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13615 as_Register($src2$$reg), ext::uxtb);
13616 %}
13617 ins_pipe(ialu_reg_reg);
13618 %}
13619
13620 // This pattern is automatically generated from aarch64_ad.m4
13621 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13622 instruct SubExtL_uxth_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_65535 mask, rFlagsReg cr)
13623 %{
13624 match(Set dst (SubL src1 (AndL src2 mask)));
13625 ins_cost(INSN_COST);
13626 format %{ "sub $dst, $src1, $src2, uxth" %}
13627
13628 ins_encode %{
13629 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13630 as_Register($src2$$reg), ext::uxth);
13631 %}
13632 ins_pipe(ialu_reg_reg);
13633 %}
13634
13635 // This pattern is automatically generated from aarch64_ad.m4
13636 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13637 instruct SubExtL_uxtw_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_4294967295 mask, rFlagsReg cr)
13638 %{
13639 match(Set dst (SubL src1 (AndL src2 mask)));
13640 ins_cost(INSN_COST);
13641 format %{ "sub $dst, $src1, $src2, uxtw" %}
13642
13643 ins_encode %{
13644 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13645 as_Register($src2$$reg), ext::uxtw);
13646 %}
13647 ins_pipe(ialu_reg_reg);
13648 %}
13649
13650
13651 // This pattern is automatically generated from aarch64_ad.m4
13652 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13653 instruct AddExtL_sxtb_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_56 lshift1, immI_56 rshift1, rFlagsReg cr)
13654 %{
13655 match(Set dst (AddL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
13656 ins_cost(1.9 * INSN_COST);
13657 format %{ "add $dst, $src1, $src2, sxtb #lshift2" %}
13658
13659 ins_encode %{
13660 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13661 as_Register($src2$$reg), ext::sxtb, ($lshift2$$constant));
13662 %}
13663 ins_pipe(ialu_reg_reg_shift);
13664 %}
13665
13666 // This pattern is automatically generated from aarch64_ad.m4
13667 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13668 instruct AddExtL_sxth_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_48 lshift1, immI_48 rshift1, rFlagsReg cr)
13669 %{
13670 match(Set dst (AddL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
13671 ins_cost(1.9 * INSN_COST);
13672 format %{ "add $dst, $src1, $src2, sxth #lshift2" %}
13673
13674 ins_encode %{
13675 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13676 as_Register($src2$$reg), ext::sxth, ($lshift2$$constant));
13677 %}
13678 ins_pipe(ialu_reg_reg_shift);
13679 %}
13680
13681 // This pattern is automatically generated from aarch64_ad.m4
13682 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13683 instruct AddExtL_sxtw_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_32 lshift1, immI_32 rshift1, rFlagsReg cr)
13684 %{
13685 match(Set dst (AddL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
13686 ins_cost(1.9 * INSN_COST);
13687 format %{ "add $dst, $src1, $src2, sxtw #lshift2" %}
13688
13689 ins_encode %{
13690 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13691 as_Register($src2$$reg), ext::sxtw, ($lshift2$$constant));
13692 %}
13693 ins_pipe(ialu_reg_reg_shift);
13694 %}
13695
13696 // This pattern is automatically generated from aarch64_ad.m4
13697 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13698 instruct SubExtL_sxtb_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_56 lshift1, immI_56 rshift1, rFlagsReg cr)
13699 %{
13700 match(Set dst (SubL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
13701 ins_cost(1.9 * INSN_COST);
13702 format %{ "sub $dst, $src1, $src2, sxtb #lshift2" %}
13703
13704 ins_encode %{
13705 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13706 as_Register($src2$$reg), ext::sxtb, ($lshift2$$constant));
13707 %}
13708 ins_pipe(ialu_reg_reg_shift);
13709 %}
13710
13711 // This pattern is automatically generated from aarch64_ad.m4
13712 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13713 instruct SubExtL_sxth_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_48 lshift1, immI_48 rshift1, rFlagsReg cr)
13714 %{
13715 match(Set dst (SubL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
13716 ins_cost(1.9 * INSN_COST);
13717 format %{ "sub $dst, $src1, $src2, sxth #lshift2" %}
13718
13719 ins_encode %{
13720 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13721 as_Register($src2$$reg), ext::sxth, ($lshift2$$constant));
13722 %}
13723 ins_pipe(ialu_reg_reg_shift);
13724 %}
13725
13726 // This pattern is automatically generated from aarch64_ad.m4
13727 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13728 instruct SubExtL_sxtw_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_32 lshift1, immI_32 rshift1, rFlagsReg cr)
13729 %{
13730 match(Set dst (SubL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
13731 ins_cost(1.9 * INSN_COST);
13732 format %{ "sub $dst, $src1, $src2, sxtw #lshift2" %}
13733
13734 ins_encode %{
13735 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13736 as_Register($src2$$reg), ext::sxtw, ($lshift2$$constant));
13737 %}
13738 ins_pipe(ialu_reg_reg_shift);
13739 %}
13740
13741 // This pattern is automatically generated from aarch64_ad.m4
13742 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13743 instruct AddExtI_sxtb_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immIExt lshift2, immI_24 lshift1, immI_24 rshift1, rFlagsReg cr)
13744 %{
13745 match(Set dst (AddI src1 (LShiftI (RShiftI (LShiftI src2 lshift1) rshift1) lshift2)));
13746 ins_cost(1.9 * INSN_COST);
13747 format %{ "addw $dst, $src1, $src2, sxtb #lshift2" %}
13748
13749 ins_encode %{
13750 __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
13751 as_Register($src2$$reg), ext::sxtb, ($lshift2$$constant));
13752 %}
13753 ins_pipe(ialu_reg_reg_shift);
13754 %}
13755
13756 // This pattern is automatically generated from aarch64_ad.m4
13757 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13758 instruct AddExtI_sxth_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immIExt lshift2, immI_16 lshift1, immI_16 rshift1, rFlagsReg cr)
13759 %{
13760 match(Set dst (AddI src1 (LShiftI (RShiftI (LShiftI src2 lshift1) rshift1) lshift2)));
13761 ins_cost(1.9 * INSN_COST);
13762 format %{ "addw $dst, $src1, $src2, sxth #lshift2" %}
13763
13764 ins_encode %{
13765 __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
13766 as_Register($src2$$reg), ext::sxth, ($lshift2$$constant));
13767 %}
13768 ins_pipe(ialu_reg_reg_shift);
13769 %}
13770
13771 // This pattern is automatically generated from aarch64_ad.m4
13772 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13773 instruct SubExtI_sxtb_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immIExt lshift2, immI_24 lshift1, immI_24 rshift1, rFlagsReg cr)
13774 %{
13775 match(Set dst (SubI src1 (LShiftI (RShiftI (LShiftI src2 lshift1) rshift1) lshift2)));
13776 ins_cost(1.9 * INSN_COST);
13777 format %{ "subw $dst, $src1, $src2, sxtb #lshift2" %}
13778
13779 ins_encode %{
13780 __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
13781 as_Register($src2$$reg), ext::sxtb, ($lshift2$$constant));
13782 %}
13783 ins_pipe(ialu_reg_reg_shift);
13784 %}
13785
13786 // This pattern is automatically generated from aarch64_ad.m4
13787 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13788 instruct SubExtI_sxth_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immIExt lshift2, immI_16 lshift1, immI_16 rshift1, rFlagsReg cr)
13789 %{
13790 match(Set dst (SubI src1 (LShiftI (RShiftI (LShiftI src2 lshift1) rshift1) lshift2)));
13791 ins_cost(1.9 * INSN_COST);
13792 format %{ "subw $dst, $src1, $src2, sxth #lshift2" %}
13793
13794 ins_encode %{
13795 __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
13796 as_Register($src2$$reg), ext::sxth, ($lshift2$$constant));
13797 %}
13798 ins_pipe(ialu_reg_reg_shift);
13799 %}
13800
13801 // This pattern is automatically generated from aarch64_ad.m4
13802 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13803 instruct AddExtI_shift(iRegLNoSp dst, iRegL src1, iRegIorL2I src2, immIExt lshift, rFlagsReg cr)
13804 %{
13805 match(Set dst (AddL src1 (LShiftL (ConvI2L src2) lshift)));
13806 ins_cost(1.9 * INSN_COST);
13807 format %{ "add $dst, $src1, $src2, sxtw #lshift" %}
13808
13809 ins_encode %{
13810 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13811 as_Register($src2$$reg), ext::sxtw, ($lshift$$constant));
13812 %}
13813 ins_pipe(ialu_reg_reg_shift);
13814 %}
13815
13816 // This pattern is automatically generated from aarch64_ad.m4
13817 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13818 instruct SubExtI_shift(iRegLNoSp dst, iRegL src1, iRegIorL2I src2, immIExt lshift, rFlagsReg cr)
13819 %{
13820 match(Set dst (SubL src1 (LShiftL (ConvI2L src2) lshift)));
13821 ins_cost(1.9 * INSN_COST);
13822 format %{ "sub $dst, $src1, $src2, sxtw #lshift" %}
13823
13824 ins_encode %{
13825 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13826 as_Register($src2$$reg), ext::sxtw, ($lshift$$constant));
13827 %}
13828 ins_pipe(ialu_reg_reg_shift);
13829 %}
13830
13831 // This pattern is automatically generated from aarch64_ad.m4
13832 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13833 instruct AddExtL_uxtb_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_255 mask, immIExt lshift, rFlagsReg cr)
13834 %{
13835 match(Set dst (AddL src1 (LShiftL (AndL src2 mask) lshift)));
13836 ins_cost(1.9 * INSN_COST);
13837 format %{ "add $dst, $src1, $src2, uxtb #lshift" %}
13838
13839 ins_encode %{
13840 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13841 as_Register($src2$$reg), ext::uxtb, ($lshift$$constant));
13842 %}
13843 ins_pipe(ialu_reg_reg_shift);
13844 %}
13845
13846 // This pattern is automatically generated from aarch64_ad.m4
13847 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13848 instruct AddExtL_uxth_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_65535 mask, immIExt lshift, rFlagsReg cr)
13849 %{
13850 match(Set dst (AddL src1 (LShiftL (AndL src2 mask) lshift)));
13851 ins_cost(1.9 * INSN_COST);
13852 format %{ "add $dst, $src1, $src2, uxth #lshift" %}
13853
13854 ins_encode %{
13855 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13856 as_Register($src2$$reg), ext::uxth, ($lshift$$constant));
13857 %}
13858 ins_pipe(ialu_reg_reg_shift);
13859 %}
13860
13861 // This pattern is automatically generated from aarch64_ad.m4
13862 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13863 instruct AddExtL_uxtw_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_4294967295 mask, immIExt lshift, rFlagsReg cr)
13864 %{
13865 match(Set dst (AddL src1 (LShiftL (AndL src2 mask) lshift)));
13866 ins_cost(1.9 * INSN_COST);
13867 format %{ "add $dst, $src1, $src2, uxtw #lshift" %}
13868
13869 ins_encode %{
13870 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13871 as_Register($src2$$reg), ext::uxtw, ($lshift$$constant));
13872 %}
13873 ins_pipe(ialu_reg_reg_shift);
13874 %}
13875
13876 // This pattern is automatically generated from aarch64_ad.m4
13877 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13878 instruct SubExtL_uxtb_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_255 mask, immIExt lshift, rFlagsReg cr)
13879 %{
13880 match(Set dst (SubL src1 (LShiftL (AndL src2 mask) lshift)));
13881 ins_cost(1.9 * INSN_COST);
13882 format %{ "sub $dst, $src1, $src2, uxtb #lshift" %}
13883
13884 ins_encode %{
13885 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13886 as_Register($src2$$reg), ext::uxtb, ($lshift$$constant));
13887 %}
13888 ins_pipe(ialu_reg_reg_shift);
13889 %}
13890
13891 // This pattern is automatically generated from aarch64_ad.m4
13892 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13893 instruct SubExtL_uxth_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_65535 mask, immIExt lshift, rFlagsReg cr)
13894 %{
13895 match(Set dst (SubL src1 (LShiftL (AndL src2 mask) lshift)));
13896 ins_cost(1.9 * INSN_COST);
13897 format %{ "sub $dst, $src1, $src2, uxth #lshift" %}
13898
13899 ins_encode %{
13900 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13901 as_Register($src2$$reg), ext::uxth, ($lshift$$constant));
13902 %}
13903 ins_pipe(ialu_reg_reg_shift);
13904 %}
13905
13906 // This pattern is automatically generated from aarch64_ad.m4
13907 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13908 instruct SubExtL_uxtw_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_4294967295 mask, immIExt lshift, rFlagsReg cr)
13909 %{
13910 match(Set dst (SubL src1 (LShiftL (AndL src2 mask) lshift)));
13911 ins_cost(1.9 * INSN_COST);
13912 format %{ "sub $dst, $src1, $src2, uxtw #lshift" %}
13913
13914 ins_encode %{
13915 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13916 as_Register($src2$$reg), ext::uxtw, ($lshift$$constant));
13917 %}
13918 ins_pipe(ialu_reg_reg_shift);
13919 %}
13920
13921 // This pattern is automatically generated from aarch64_ad.m4
13922 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13923 instruct AddExtI_uxtb_and_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_255 mask, immIExt lshift, rFlagsReg cr)
13924 %{
13925 match(Set dst (AddI src1 (LShiftI (AndI src2 mask) lshift)));
13926 ins_cost(1.9 * INSN_COST);
13927 format %{ "addw $dst, $src1, $src2, uxtb #lshift" %}
13928
13929 ins_encode %{
13930 __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
13931 as_Register($src2$$reg), ext::uxtb, ($lshift$$constant));
13932 %}
13933 ins_pipe(ialu_reg_reg_shift);
13934 %}
13935
13936 // This pattern is automatically generated from aarch64_ad.m4
13937 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13938 instruct AddExtI_uxth_and_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_65535 mask, immIExt lshift, rFlagsReg cr)
13939 %{
13940 match(Set dst (AddI src1 (LShiftI (AndI src2 mask) lshift)));
13941 ins_cost(1.9 * INSN_COST);
13942 format %{ "addw $dst, $src1, $src2, uxth #lshift" %}
13943
13944 ins_encode %{
13945 __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
13946 as_Register($src2$$reg), ext::uxth, ($lshift$$constant));
13947 %}
13948 ins_pipe(ialu_reg_reg_shift);
13949 %}
13950
13951 // This pattern is automatically generated from aarch64_ad.m4
13952 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13953 instruct SubExtI_uxtb_and_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_255 mask, immIExt lshift, rFlagsReg cr)
13954 %{
13955 match(Set dst (SubI src1 (LShiftI (AndI src2 mask) lshift)));
13956 ins_cost(1.9 * INSN_COST);
13957 format %{ "subw $dst, $src1, $src2, uxtb #lshift" %}
13958
13959 ins_encode %{
13960 __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
13961 as_Register($src2$$reg), ext::uxtb, ($lshift$$constant));
13962 %}
13963 ins_pipe(ialu_reg_reg_shift);
13964 %}
13965
13966 // This pattern is automatically generated from aarch64_ad.m4
13967 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13968 instruct SubExtI_uxth_and_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_65535 mask, immIExt lshift, rFlagsReg cr)
13969 %{
13970 match(Set dst (SubI src1 (LShiftI (AndI src2 mask) lshift)));
13971 ins_cost(1.9 * INSN_COST);
13972 format %{ "subw $dst, $src1, $src2, uxth #lshift" %}
13973
13974 ins_encode %{
13975 __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
13976 as_Register($src2$$reg), ext::uxth, ($lshift$$constant));
13977 %}
13978 ins_pipe(ialu_reg_reg_shift);
13979 %}
13980
13981 // This pattern is automatically generated from aarch64_ad.m4
13982 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13983 instruct cmovI_reg_reg_lt(iRegINoSp dst, iRegI src1, iRegI src2, rFlagsReg cr)
13984 %{
13985 effect(DEF dst, USE src1, USE src2, USE cr);
13986 ins_cost(INSN_COST * 2);
13987 format %{ "cselw $dst, $src1, $src2 lt\t" %}
13988
13989 ins_encode %{
13990 __ cselw($dst$$Register,
13991 $src1$$Register,
13992 $src2$$Register,
13993 Assembler::LT);
13994 %}
13995 ins_pipe(icond_reg_reg);
13996 %}
13997
13998 // This pattern is automatically generated from aarch64_ad.m4
13999 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
14000 instruct cmovI_reg_reg_gt(iRegINoSp dst, iRegI src1, iRegI src2, rFlagsReg cr)
14001 %{
14002 effect(DEF dst, USE src1, USE src2, USE cr);
14003 ins_cost(INSN_COST * 2);
14004 format %{ "cselw $dst, $src1, $src2 gt\t" %}
14005
14006 ins_encode %{
14007 __ cselw($dst$$Register,
14008 $src1$$Register,
14009 $src2$$Register,
14010 Assembler::GT);
14011 %}
14012 ins_pipe(icond_reg_reg);
14013 %}
14014
14015 // This pattern is automatically generated from aarch64_ad.m4
14016 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
14017 instruct cmovI_reg_imm0_lt(iRegINoSp dst, iRegI src1, rFlagsReg cr)
14018 %{
14019 effect(DEF dst, USE src1, USE cr);
14020 ins_cost(INSN_COST * 2);
14021 format %{ "cselw $dst, $src1, zr lt\t" %}
14022
14023 ins_encode %{
14024 __ cselw($dst$$Register,
14025 $src1$$Register,
14026 zr,
14027 Assembler::LT);
14028 %}
14029 ins_pipe(icond_reg);
14030 %}
14031
14032 // This pattern is automatically generated from aarch64_ad.m4
14033 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
14034 instruct cmovI_reg_imm0_gt(iRegINoSp dst, iRegI src1, rFlagsReg cr)
14035 %{
14036 effect(DEF dst, USE src1, USE cr);
14037 ins_cost(INSN_COST * 2);
14038 format %{ "cselw $dst, $src1, zr gt\t" %}
14039
14040 ins_encode %{
14041 __ cselw($dst$$Register,
14042 $src1$$Register,
14043 zr,
14044 Assembler::GT);
14045 %}
14046 ins_pipe(icond_reg);
14047 %}
14048
14049 // This pattern is automatically generated from aarch64_ad.m4
14050 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
14051 instruct cmovI_reg_imm1_le(iRegINoSp dst, iRegI src1, rFlagsReg cr)
14052 %{
14053 effect(DEF dst, USE src1, USE cr);
14054 ins_cost(INSN_COST * 2);
14055 format %{ "csincw $dst, $src1, zr le\t" %}
14056
14057 ins_encode %{
14058 __ csincw($dst$$Register,
14059 $src1$$Register,
14060 zr,
14061 Assembler::LE);
14062 %}
14063 ins_pipe(icond_reg);
14064 %}
14065
14066 // This pattern is automatically generated from aarch64_ad.m4
14067 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
14068 instruct cmovI_reg_imm1_gt(iRegINoSp dst, iRegI src1, rFlagsReg cr)
14069 %{
14070 effect(DEF dst, USE src1, USE cr);
14071 ins_cost(INSN_COST * 2);
14072 format %{ "csincw $dst, $src1, zr gt\t" %}
14073
14074 ins_encode %{
14075 __ csincw($dst$$Register,
14076 $src1$$Register,
14077 zr,
14078 Assembler::GT);
14079 %}
14080 ins_pipe(icond_reg);
14081 %}
14082
14083 // This pattern is automatically generated from aarch64_ad.m4
14084 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
14085 instruct cmovI_reg_immM1_lt(iRegINoSp dst, iRegI src1, rFlagsReg cr)
14086 %{
14087 effect(DEF dst, USE src1, USE cr);
14088 ins_cost(INSN_COST * 2);
14089 format %{ "csinvw $dst, $src1, zr lt\t" %}
14090
14091 ins_encode %{
14092 __ csinvw($dst$$Register,
14093 $src1$$Register,
14094 zr,
14095 Assembler::LT);
14096 %}
14097 ins_pipe(icond_reg);
14098 %}
14099
14100 // This pattern is automatically generated from aarch64_ad.m4
14101 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
14102 instruct cmovI_reg_immM1_ge(iRegINoSp dst, iRegI src1, rFlagsReg cr)
14103 %{
14104 effect(DEF dst, USE src1, USE cr);
14105 ins_cost(INSN_COST * 2);
14106 format %{ "csinvw $dst, $src1, zr ge\t" %}
14107
14108 ins_encode %{
14109 __ csinvw($dst$$Register,
14110 $src1$$Register,
14111 zr,
14112 Assembler::GE);
14113 %}
14114 ins_pipe(icond_reg);
14115 %}
14116
14117 // This pattern is automatically generated from aarch64_ad.m4
14118 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
14119 instruct minI_reg_imm0(iRegINoSp dst, iRegIorL2I src, immI0 imm)
14120 %{
14121 match(Set dst (MinI src imm));
14122 ins_cost(INSN_COST * 3);
14123 expand %{
14124 rFlagsReg cr;
14125 compI_reg_imm0(cr, src);
14126 cmovI_reg_imm0_lt(dst, src, cr);
14127 %}
14128 %}
14129
14130 // This pattern is automatically generated from aarch64_ad.m4
14131 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
14132 instruct minI_imm0_reg(iRegINoSp dst, immI0 imm, iRegIorL2I src)
14133 %{
14134 match(Set dst (MinI imm src));
14135 ins_cost(INSN_COST * 3);
14136 expand %{
14137 rFlagsReg cr;
14138 compI_reg_imm0(cr, src);
14139 cmovI_reg_imm0_lt(dst, src, cr);
14140 %}
14141 %}
14142
14143 // This pattern is automatically generated from aarch64_ad.m4
14144 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
14145 instruct minI_reg_imm1(iRegINoSp dst, iRegIorL2I src, immI_1 imm)
14146 %{
14147 match(Set dst (MinI src imm));
14148 ins_cost(INSN_COST * 3);
14149 expand %{
14150 rFlagsReg cr;
14151 compI_reg_imm0(cr, src);
14152 cmovI_reg_imm1_le(dst, src, cr);
14153 %}
14154 %}
14155
14156 // This pattern is automatically generated from aarch64_ad.m4
14157 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
14158 instruct minI_imm1_reg(iRegINoSp dst, immI_1 imm, iRegIorL2I src)
14159 %{
14160 match(Set dst (MinI imm src));
14161 ins_cost(INSN_COST * 3);
14162 expand %{
14163 rFlagsReg cr;
14164 compI_reg_imm0(cr, src);
14165 cmovI_reg_imm1_le(dst, src, cr);
14166 %}
14167 %}
14168
14169 // This pattern is automatically generated from aarch64_ad.m4
14170 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
14171 instruct minI_reg_immM1(iRegINoSp dst, iRegIorL2I src, immI_M1 imm)
14172 %{
14173 match(Set dst (MinI src imm));
14174 ins_cost(INSN_COST * 3);
14175 expand %{
14176 rFlagsReg cr;
14177 compI_reg_imm0(cr, src);
14178 cmovI_reg_immM1_lt(dst, src, cr);
14179 %}
14180 %}
14181
14182 // This pattern is automatically generated from aarch64_ad.m4
14183 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
14184 instruct minI_immM1_reg(iRegINoSp dst, immI_M1 imm, iRegIorL2I src)
14185 %{
14186 match(Set dst (MinI imm src));
14187 ins_cost(INSN_COST * 3);
14188 expand %{
14189 rFlagsReg cr;
14190 compI_reg_imm0(cr, src);
14191 cmovI_reg_immM1_lt(dst, src, cr);
14192 %}
14193 %}
14194
14195 // This pattern is automatically generated from aarch64_ad.m4
14196 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
14197 instruct maxI_reg_imm0(iRegINoSp dst, iRegIorL2I src, immI0 imm)
14198 %{
14199 match(Set dst (MaxI src imm));
14200 ins_cost(INSN_COST * 3);
14201 expand %{
14202 rFlagsReg cr;
14203 compI_reg_imm0(cr, src);
14204 cmovI_reg_imm0_gt(dst, src, cr);
14205 %}
14206 %}
14207
14208 // This pattern is automatically generated from aarch64_ad.m4
14209 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
14210 instruct maxI_imm0_reg(iRegINoSp dst, immI0 imm, iRegIorL2I src)
14211 %{
14212 match(Set dst (MaxI imm src));
14213 ins_cost(INSN_COST * 3);
14214 expand %{
14215 rFlagsReg cr;
14216 compI_reg_imm0(cr, src);
14217 cmovI_reg_imm0_gt(dst, src, cr);
14218 %}
14219 %}
14220
14221 // This pattern is automatically generated from aarch64_ad.m4
14222 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
14223 instruct maxI_reg_imm1(iRegINoSp dst, iRegIorL2I src, immI_1 imm)
14224 %{
14225 match(Set dst (MaxI src imm));
14226 ins_cost(INSN_COST * 3);
14227 expand %{
14228 rFlagsReg cr;
14229 compI_reg_imm0(cr, src);
14230 cmovI_reg_imm1_gt(dst, src, cr);
14231 %}
14232 %}
14233
14234 // This pattern is automatically generated from aarch64_ad.m4
14235 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
14236 instruct maxI_imm1_reg(iRegINoSp dst, immI_1 imm, iRegIorL2I src)
14237 %{
14238 match(Set dst (MaxI imm src));
14239 ins_cost(INSN_COST * 3);
14240 expand %{
14241 rFlagsReg cr;
14242 compI_reg_imm0(cr, src);
14243 cmovI_reg_imm1_gt(dst, src, cr);
14244 %}
14245 %}
14246
14247 // This pattern is automatically generated from aarch64_ad.m4
14248 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
14249 instruct maxI_reg_immM1(iRegINoSp dst, iRegIorL2I src, immI_M1 imm)
14250 %{
14251 match(Set dst (MaxI src imm));
14252 ins_cost(INSN_COST * 3);
14253 expand %{
14254 rFlagsReg cr;
14255 compI_reg_imm0(cr, src);
14256 cmovI_reg_immM1_ge(dst, src, cr);
14257 %}
14258 %}
14259
14260 // This pattern is automatically generated from aarch64_ad.m4
14261 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
14262 instruct maxI_immM1_reg(iRegINoSp dst, immI_M1 imm, iRegIorL2I src)
14263 %{
14264 match(Set dst (MaxI imm src));
14265 ins_cost(INSN_COST * 3);
14266 expand %{
14267 rFlagsReg cr;
14268 compI_reg_imm0(cr, src);
14269 cmovI_reg_immM1_ge(dst, src, cr);
14270 %}
14271 %}
14272
14273
14274
14275 // END This section of the file is automatically generated. Do not edit --------------
14276
14277
14278 // ============================================================================
14279 // Floating Point Arithmetic Instructions
14280
14281 instruct addF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
14282 match(Set dst (AddF src1 src2));
14283
14284 ins_cost(INSN_COST * 5);
14285 format %{ "fadds $dst, $src1, $src2" %}
14286
14287 ins_encode %{
14288 __ fadds(as_FloatRegister($dst$$reg),
14289 as_FloatRegister($src1$$reg),
14290 as_FloatRegister($src2$$reg));
14291 %}
14292
14293 ins_pipe(fp_dop_reg_reg_s);
14294 %}
14295
14296 instruct addD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
14297 match(Set dst (AddD src1 src2));
14298
14299 ins_cost(INSN_COST * 5);
14300 format %{ "faddd $dst, $src1, $src2" %}
14301
14302 ins_encode %{
14303 __ faddd(as_FloatRegister($dst$$reg),
14304 as_FloatRegister($src1$$reg),
14305 as_FloatRegister($src2$$reg));
14306 %}
14307
14308 ins_pipe(fp_dop_reg_reg_d);
14309 %}
14310
14311 instruct subF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
14312 match(Set dst (SubF src1 src2));
14313
14314 ins_cost(INSN_COST * 5);
14315 format %{ "fsubs $dst, $src1, $src2" %}
14316
14317 ins_encode %{
14318 __ fsubs(as_FloatRegister($dst$$reg),
14319 as_FloatRegister($src1$$reg),
14320 as_FloatRegister($src2$$reg));
14321 %}
14322
14323 ins_pipe(fp_dop_reg_reg_s);
14324 %}
14325
14326 instruct subD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
14327 match(Set dst (SubD src1 src2));
14328
14329 ins_cost(INSN_COST * 5);
14330 format %{ "fsubd $dst, $src1, $src2" %}
14331
14332 ins_encode %{
14333 __ fsubd(as_FloatRegister($dst$$reg),
14334 as_FloatRegister($src1$$reg),
14335 as_FloatRegister($src2$$reg));
14336 %}
14337
14338 ins_pipe(fp_dop_reg_reg_d);
14339 %}
14340
14341 instruct mulF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
14342 match(Set dst (MulF src1 src2));
14343
14344 ins_cost(INSN_COST * 6);
14345 format %{ "fmuls $dst, $src1, $src2" %}
14346
14347 ins_encode %{
14348 __ fmuls(as_FloatRegister($dst$$reg),
14349 as_FloatRegister($src1$$reg),
14350 as_FloatRegister($src2$$reg));
14351 %}
14352
14353 ins_pipe(fp_dop_reg_reg_s);
14354 %}
14355
14356 instruct mulD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
14357 match(Set dst (MulD src1 src2));
14358
14359 ins_cost(INSN_COST * 6);
14360 format %{ "fmuld $dst, $src1, $src2" %}
14361
14362 ins_encode %{
14363 __ fmuld(as_FloatRegister($dst$$reg),
14364 as_FloatRegister($src1$$reg),
14365 as_FloatRegister($src2$$reg));
14366 %}
14367
14368 ins_pipe(fp_dop_reg_reg_d);
14369 %}
14370
14371 // src1 * src2 + src3
14372 instruct maddF_reg_reg(vRegF dst, vRegF src1, vRegF src2, vRegF src3) %{
14373 predicate(UseFMA);
14374 match(Set dst (FmaF src3 (Binary src1 src2)));
14375
14376 format %{ "fmadds $dst, $src1, $src2, $src3" %}
14377
14378 ins_encode %{
14379 __ fmadds(as_FloatRegister($dst$$reg),
14380 as_FloatRegister($src1$$reg),
14381 as_FloatRegister($src2$$reg),
14382 as_FloatRegister($src3$$reg));
14383 %}
14384
14385 ins_pipe(pipe_class_default);
14386 %}
14387
14388 // src1 * src2 + src3
14389 instruct maddD_reg_reg(vRegD dst, vRegD src1, vRegD src2, vRegD src3) %{
14390 predicate(UseFMA);
14391 match(Set dst (FmaD src3 (Binary src1 src2)));
14392
14393 format %{ "fmaddd $dst, $src1, $src2, $src3" %}
14394
14395 ins_encode %{
14396 __ fmaddd(as_FloatRegister($dst$$reg),
14397 as_FloatRegister($src1$$reg),
14398 as_FloatRegister($src2$$reg),
14399 as_FloatRegister($src3$$reg));
14400 %}
14401
14402 ins_pipe(pipe_class_default);
14403 %}
14404
14405 // -src1 * src2 + src3
14406 instruct msubF_reg_reg(vRegF dst, vRegF src1, vRegF src2, vRegF src3) %{
14407 predicate(UseFMA);
14408 match(Set dst (FmaF src3 (Binary (NegF src1) src2)));
14409 match(Set dst (FmaF src3 (Binary src1 (NegF src2))));
14410
14411 format %{ "fmsubs $dst, $src1, $src2, $src3" %}
14412
14413 ins_encode %{
14414 __ fmsubs(as_FloatRegister($dst$$reg),
14415 as_FloatRegister($src1$$reg),
14416 as_FloatRegister($src2$$reg),
14417 as_FloatRegister($src3$$reg));
14418 %}
14419
14420 ins_pipe(pipe_class_default);
14421 %}
14422
14423 // -src1 * src2 + src3
14424 instruct msubD_reg_reg(vRegD dst, vRegD src1, vRegD src2, vRegD src3) %{
14425 predicate(UseFMA);
14426 match(Set dst (FmaD src3 (Binary (NegD src1) src2)));
14427 match(Set dst (FmaD src3 (Binary src1 (NegD src2))));
14428
14429 format %{ "fmsubd $dst, $src1, $src2, $src3" %}
14430
14431 ins_encode %{
14432 __ fmsubd(as_FloatRegister($dst$$reg),
14433 as_FloatRegister($src1$$reg),
14434 as_FloatRegister($src2$$reg),
14435 as_FloatRegister($src3$$reg));
14436 %}
14437
14438 ins_pipe(pipe_class_default);
14439 %}
14440
14441 // -src1 * src2 - src3
14442 instruct mnaddF_reg_reg(vRegF dst, vRegF src1, vRegF src2, vRegF src3) %{
14443 predicate(UseFMA);
14444 match(Set dst (FmaF (NegF src3) (Binary (NegF src1) src2)));
14445 match(Set dst (FmaF (NegF src3) (Binary src1 (NegF src2))));
14446
14447 format %{ "fnmadds $dst, $src1, $src2, $src3" %}
14448
14449 ins_encode %{
14450 __ fnmadds(as_FloatRegister($dst$$reg),
14451 as_FloatRegister($src1$$reg),
14452 as_FloatRegister($src2$$reg),
14453 as_FloatRegister($src3$$reg));
14454 %}
14455
14456 ins_pipe(pipe_class_default);
14457 %}
14458
14459 // -src1 * src2 - src3
14460 instruct mnaddD_reg_reg(vRegD dst, vRegD src1, vRegD src2, vRegD src3) %{
14461 predicate(UseFMA);
14462 match(Set dst (FmaD (NegD src3) (Binary (NegD src1) src2)));
14463 match(Set dst (FmaD (NegD src3) (Binary src1 (NegD src2))));
14464
14465 format %{ "fnmaddd $dst, $src1, $src2, $src3" %}
14466
14467 ins_encode %{
14468 __ fnmaddd(as_FloatRegister($dst$$reg),
14469 as_FloatRegister($src1$$reg),
14470 as_FloatRegister($src2$$reg),
14471 as_FloatRegister($src3$$reg));
14472 %}
14473
14474 ins_pipe(pipe_class_default);
14475 %}
14476
14477 // src1 * src2 - src3
14478 instruct mnsubF_reg_reg(vRegF dst, vRegF src1, vRegF src2, vRegF src3, immF0 zero) %{
14479 predicate(UseFMA);
14480 match(Set dst (FmaF (NegF src3) (Binary src1 src2)));
14481
14482 format %{ "fnmsubs $dst, $src1, $src2, $src3" %}
14483
14484 ins_encode %{
14485 __ fnmsubs(as_FloatRegister($dst$$reg),
14486 as_FloatRegister($src1$$reg),
14487 as_FloatRegister($src2$$reg),
14488 as_FloatRegister($src3$$reg));
14489 %}
14490
14491 ins_pipe(pipe_class_default);
14492 %}
14493
14494 // src1 * src2 - src3
14495 instruct mnsubD_reg_reg(vRegD dst, vRegD src1, vRegD src2, vRegD src3, immD0 zero) %{
14496 predicate(UseFMA);
14497 match(Set dst (FmaD (NegD src3) (Binary src1 src2)));
14498
14499 format %{ "fnmsubd $dst, $src1, $src2, $src3" %}
14500
14501 ins_encode %{
14502 // n.b. insn name should be fnmsubd
14503 __ fnmsub(as_FloatRegister($dst$$reg),
14504 as_FloatRegister($src1$$reg),
14505 as_FloatRegister($src2$$reg),
14506 as_FloatRegister($src3$$reg));
14507 %}
14508
14509 ins_pipe(pipe_class_default);
14510 %}
14511
14512
14513 // Math.max(FF)F
14514 instruct maxF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
14515 match(Set dst (MaxF src1 src2));
14516
14517 format %{ "fmaxs $dst, $src1, $src2" %}
14518 ins_encode %{
14519 __ fmaxs(as_FloatRegister($dst$$reg),
14520 as_FloatRegister($src1$$reg),
14521 as_FloatRegister($src2$$reg));
14522 %}
14523
14524 ins_pipe(fp_dop_reg_reg_s);
14525 %}
14526
14527 // Math.min(FF)F
14528 instruct minF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
14529 match(Set dst (MinF src1 src2));
14530
14531 format %{ "fmins $dst, $src1, $src2" %}
14532 ins_encode %{
14533 __ fmins(as_FloatRegister($dst$$reg),
14534 as_FloatRegister($src1$$reg),
14535 as_FloatRegister($src2$$reg));
14536 %}
14537
14538 ins_pipe(fp_dop_reg_reg_s);
14539 %}
14540
14541 // Math.max(DD)D
14542 instruct maxD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
14543 match(Set dst (MaxD src1 src2));
14544
14545 format %{ "fmaxd $dst, $src1, $src2" %}
14546 ins_encode %{
14547 __ fmaxd(as_FloatRegister($dst$$reg),
14548 as_FloatRegister($src1$$reg),
14549 as_FloatRegister($src2$$reg));
14550 %}
14551
14552 ins_pipe(fp_dop_reg_reg_d);
14553 %}
14554
14555 // Math.min(DD)D
14556 instruct minD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
14557 match(Set dst (MinD src1 src2));
14558
14559 format %{ "fmind $dst, $src1, $src2" %}
14560 ins_encode %{
14561 __ fmind(as_FloatRegister($dst$$reg),
14562 as_FloatRegister($src1$$reg),
14563 as_FloatRegister($src2$$reg));
14564 %}
14565
14566 ins_pipe(fp_dop_reg_reg_d);
14567 %}
14568
14569
14570 instruct divF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
14571 match(Set dst (DivF src1 src2));
14572
14573 ins_cost(INSN_COST * 18);
14574 format %{ "fdivs $dst, $src1, $src2" %}
14575
14576 ins_encode %{
14577 __ fdivs(as_FloatRegister($dst$$reg),
14578 as_FloatRegister($src1$$reg),
14579 as_FloatRegister($src2$$reg));
14580 %}
14581
14582 ins_pipe(fp_div_s);
14583 %}
14584
14585 instruct divD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
14586 match(Set dst (DivD src1 src2));
14587
14588 ins_cost(INSN_COST * 32);
14589 format %{ "fdivd $dst, $src1, $src2" %}
14590
14591 ins_encode %{
14592 __ fdivd(as_FloatRegister($dst$$reg),
14593 as_FloatRegister($src1$$reg),
14594 as_FloatRegister($src2$$reg));
14595 %}
14596
14597 ins_pipe(fp_div_d);
14598 %}
14599
14600 instruct negF_reg_reg(vRegF dst, vRegF src) %{
14601 match(Set dst (NegF src));
14602
14603 ins_cost(INSN_COST * 3);
14604 format %{ "fneg $dst, $src" %}
14605
14606 ins_encode %{
14607 __ fnegs(as_FloatRegister($dst$$reg),
14608 as_FloatRegister($src$$reg));
14609 %}
14610
14611 ins_pipe(fp_uop_s);
14612 %}
14613
14614 instruct negD_reg_reg(vRegD dst, vRegD src) %{
14615 match(Set dst (NegD src));
14616
14617 ins_cost(INSN_COST * 3);
14618 format %{ "fnegd $dst, $src" %}
14619
14620 ins_encode %{
14621 __ fnegd(as_FloatRegister($dst$$reg),
14622 as_FloatRegister($src$$reg));
14623 %}
14624
14625 ins_pipe(fp_uop_d);
14626 %}
14627
14628 instruct absI_reg(iRegINoSp dst, iRegIorL2I src, rFlagsReg cr)
14629 %{
14630 match(Set dst (AbsI src));
14631
14632 effect(KILL cr);
14633 ins_cost(INSN_COST * 2);
14634 format %{ "cmpw $src, zr\n\t"
14635 "cnegw $dst, $src, Assembler::LT\t# int abs"
14636 %}
14637
14638 ins_encode %{
14639 __ cmpw(as_Register($src$$reg), zr);
14640 __ cnegw(as_Register($dst$$reg), as_Register($src$$reg), Assembler::LT);
14641 %}
14642 ins_pipe(pipe_class_default);
14643 %}
14644
14645 instruct absL_reg(iRegLNoSp dst, iRegL src, rFlagsReg cr)
14646 %{
14647 match(Set dst (AbsL src));
14648
14649 effect(KILL cr);
14650 ins_cost(INSN_COST * 2);
14651 format %{ "cmp $src, zr\n\t"
14652 "cneg $dst, $src, Assembler::LT\t# long abs"
14653 %}
14654
14655 ins_encode %{
14656 __ cmp(as_Register($src$$reg), zr);
14657 __ cneg(as_Register($dst$$reg), as_Register($src$$reg), Assembler::LT);
14658 %}
14659 ins_pipe(pipe_class_default);
14660 %}
14661
14662 instruct absF_reg(vRegF dst, vRegF src) %{
14663 match(Set dst (AbsF src));
14664
14665 ins_cost(INSN_COST * 3);
14666 format %{ "fabss $dst, $src" %}
14667 ins_encode %{
14668 __ fabss(as_FloatRegister($dst$$reg),
14669 as_FloatRegister($src$$reg));
14670 %}
14671
14672 ins_pipe(fp_uop_s);
14673 %}
14674
14675 instruct absD_reg(vRegD dst, vRegD src) %{
14676 match(Set dst (AbsD src));
14677
14678 ins_cost(INSN_COST * 3);
14679 format %{ "fabsd $dst, $src" %}
14680 ins_encode %{
14681 __ fabsd(as_FloatRegister($dst$$reg),
14682 as_FloatRegister($src$$reg));
14683 %}
14684
14685 ins_pipe(fp_uop_d);
14686 %}
14687
14688 instruct absdF_reg(vRegF dst, vRegF src1, vRegF src2) %{
14689 match(Set dst (AbsF (SubF src1 src2)));
14690
14691 ins_cost(INSN_COST * 3);
14692 format %{ "fabds $dst, $src1, $src2" %}
14693 ins_encode %{
14694 __ fabds(as_FloatRegister($dst$$reg),
14695 as_FloatRegister($src1$$reg),
14696 as_FloatRegister($src2$$reg));
14697 %}
14698
14699 ins_pipe(fp_uop_s);
14700 %}
14701
14702 instruct absdD_reg(vRegD dst, vRegD src1, vRegD src2) %{
14703 match(Set dst (AbsD (SubD src1 src2)));
14704
14705 ins_cost(INSN_COST * 3);
14706 format %{ "fabdd $dst, $src1, $src2" %}
14707 ins_encode %{
14708 __ fabdd(as_FloatRegister($dst$$reg),
14709 as_FloatRegister($src1$$reg),
14710 as_FloatRegister($src2$$reg));
14711 %}
14712
14713 ins_pipe(fp_uop_d);
14714 %}
14715
14716 instruct sqrtD_reg(vRegD dst, vRegD src) %{
14717 match(Set dst (SqrtD src));
14718
14719 ins_cost(INSN_COST * 50);
14720 format %{ "fsqrtd $dst, $src" %}
14721 ins_encode %{
14722 __ fsqrtd(as_FloatRegister($dst$$reg),
14723 as_FloatRegister($src$$reg));
14724 %}
14725
14726 ins_pipe(fp_div_s);
14727 %}
14728
14729 instruct sqrtF_reg(vRegF dst, vRegF src) %{
14730 match(Set dst (SqrtF src));
14731
14732 ins_cost(INSN_COST * 50);
14733 format %{ "fsqrts $dst, $src" %}
14734 ins_encode %{
14735 __ fsqrts(as_FloatRegister($dst$$reg),
14736 as_FloatRegister($src$$reg));
14737 %}
14738
14739 ins_pipe(fp_div_d);
14740 %}
14741
14742 // Math.rint, floor, ceil
14743 instruct roundD_reg(vRegD dst, vRegD src, immI rmode) %{
14744 match(Set dst (RoundDoubleMode src rmode));
14745 format %{ "frint $dst, $src, $rmode" %}
14746 ins_encode %{
14747 switch ($rmode$$constant) {
14748 case RoundDoubleModeNode::rmode_rint:
14749 __ frintnd(as_FloatRegister($dst$$reg),
14750 as_FloatRegister($src$$reg));
14751 break;
14752 case RoundDoubleModeNode::rmode_floor:
14753 __ frintmd(as_FloatRegister($dst$$reg),
14754 as_FloatRegister($src$$reg));
14755 break;
14756 case RoundDoubleModeNode::rmode_ceil:
14757 __ frintpd(as_FloatRegister($dst$$reg),
14758 as_FloatRegister($src$$reg));
14759 break;
14760 }
14761 %}
14762 ins_pipe(fp_uop_d);
14763 %}
14764
14765 instruct copySignD_reg(vRegD dst, vRegD src1, vRegD src2, vRegD zero) %{
14766 match(Set dst (CopySignD src1 (Binary src2 zero)));
14767 effect(TEMP_DEF dst, USE src1, USE src2, USE zero);
14768 format %{ "CopySignD $dst $src1 $src2" %}
14769 ins_encode %{
14770 FloatRegister dst = as_FloatRegister($dst$$reg),
14771 src1 = as_FloatRegister($src1$$reg),
14772 src2 = as_FloatRegister($src2$$reg),
14773 zero = as_FloatRegister($zero$$reg);
14774 __ fnegd(dst, zero);
14775 __ bsl(dst, __ T8B, src2, src1);
14776 %}
14777 ins_pipe(fp_uop_d);
14778 %}
14779
14780 instruct copySignF_reg(vRegF dst, vRegF src1, vRegF src2) %{
14781 match(Set dst (CopySignF src1 src2));
14782 effect(TEMP_DEF dst, USE src1, USE src2);
14783 format %{ "CopySignF $dst $src1 $src2" %}
14784 ins_encode %{
14785 FloatRegister dst = as_FloatRegister($dst$$reg),
14786 src1 = as_FloatRegister($src1$$reg),
14787 src2 = as_FloatRegister($src2$$reg);
14788 __ movi(dst, __ T2S, 0x80, 24);
14789 __ bsl(dst, __ T8B, src2, src1);
14790 %}
14791 ins_pipe(fp_uop_d);
14792 %}
14793
14794 instruct signumD_reg(vRegD dst, vRegD src, vRegD zero, vRegD one) %{
14795 match(Set dst (SignumD src (Binary zero one)));
14796 effect(TEMP_DEF dst, USE src, USE zero, USE one);
14797 format %{ "signumD $dst, $src" %}
14798 ins_encode %{
14799 FloatRegister src = as_FloatRegister($src$$reg),
14800 dst = as_FloatRegister($dst$$reg),
14801 zero = as_FloatRegister($zero$$reg),
14802 one = as_FloatRegister($one$$reg);
14803 __ facgtd(dst, src, zero); // dst=0 for +-0.0 and NaN. 0xFFF..F otherwise
14804 __ ushrd(dst, dst, 1); // dst=0 for +-0.0 and NaN. 0x7FF..F otherwise
14805 // Bit selection instruction gets bit from "one" for each enabled bit in
14806 // "dst", otherwise gets a bit from "src". For "src" that contains +-0.0 or
14807 // NaN the whole "src" will be copied because "dst" is zero. For all other
14808 // "src" values dst is 0x7FF..F, which means only the sign bit is copied
14809 // from "src", and all other bits are copied from 1.0.
14810 __ bsl(dst, __ T8B, one, src);
14811 %}
14812 ins_pipe(fp_uop_d);
14813 %}
14814
14815 instruct signumF_reg(vRegF dst, vRegF src, vRegF zero, vRegF one) %{
14816 match(Set dst (SignumF src (Binary zero one)));
14817 effect(TEMP_DEF dst, USE src, USE zero, USE one);
14818 format %{ "signumF $dst, $src" %}
14819 ins_encode %{
14820 FloatRegister src = as_FloatRegister($src$$reg),
14821 dst = as_FloatRegister($dst$$reg),
14822 zero = as_FloatRegister($zero$$reg),
14823 one = as_FloatRegister($one$$reg);
14824 __ facgts(dst, src, zero); // dst=0 for +-0.0 and NaN. 0xFFF..F otherwise
14825 __ ushr(dst, __ T2S, dst, 1); // dst=0 for +-0.0 and NaN. 0x7FF..F otherwise
14826 // Bit selection instruction gets bit from "one" for each enabled bit in
14827 // "dst", otherwise gets a bit from "src". For "src" that contains +-0.0 or
14828 // NaN the whole "src" will be copied because "dst" is zero. For all other
14829 // "src" values dst is 0x7FF..F, which means only the sign bit is copied
14830 // from "src", and all other bits are copied from 1.0.
14831 __ bsl(dst, __ T8B, one, src);
14832 %}
14833 ins_pipe(fp_uop_d);
14834 %}
14835
14836 instruct onspinwait() %{
14837 match(OnSpinWait);
14838 ins_cost(INSN_COST);
14839
14840 format %{ "onspinwait" %}
14841
14842 ins_encode %{
14843 __ spin_wait();
14844 %}
14845 ins_pipe(pipe_class_empty);
14846 %}
14847
14848 // ============================================================================
14849 // Logical Instructions
14850
14851 // Integer Logical Instructions
14852
14853 // And Instructions
14854
14855
14856 instruct andI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, rFlagsReg cr) %{
14857 match(Set dst (AndI src1 src2));
14858
14859 format %{ "andw $dst, $src1, $src2\t# int" %}
14860
14861 ins_cost(INSN_COST);
14862 ins_encode %{
14863 __ andw(as_Register($dst$$reg),
14864 as_Register($src1$$reg),
14865 as_Register($src2$$reg));
14866 %}
14867
14868 ins_pipe(ialu_reg_reg);
14869 %}
14870
14871 instruct andI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immILog src2, rFlagsReg cr) %{
14872 match(Set dst (AndI src1 src2));
14873
14874 format %{ "andsw $dst, $src1, $src2\t# int" %}
14875
14876 ins_cost(INSN_COST);
14877 ins_encode %{
14878 __ andw(as_Register($dst$$reg),
14879 as_Register($src1$$reg),
14880 (uint64_t)($src2$$constant));
14881 %}
14882
14883 ins_pipe(ialu_reg_imm);
14884 %}
14885
14886 // Or Instructions
14887
14888 instruct orI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
14889 match(Set dst (OrI src1 src2));
14890
14891 format %{ "orrw $dst, $src1, $src2\t# int" %}
14892
14893 ins_cost(INSN_COST);
14894 ins_encode %{
14895 __ orrw(as_Register($dst$$reg),
14896 as_Register($src1$$reg),
14897 as_Register($src2$$reg));
14898 %}
14899
14900 ins_pipe(ialu_reg_reg);
14901 %}
14902
14903 instruct orI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immILog src2) %{
14904 match(Set dst (OrI src1 src2));
14905
14906 format %{ "orrw $dst, $src1, $src2\t# int" %}
14907
14908 ins_cost(INSN_COST);
14909 ins_encode %{
14910 __ orrw(as_Register($dst$$reg),
14911 as_Register($src1$$reg),
14912 (uint64_t)($src2$$constant));
14913 %}
14914
14915 ins_pipe(ialu_reg_imm);
14916 %}
14917
14918 // Xor Instructions
14919
14920 instruct xorI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
14921 match(Set dst (XorI src1 src2));
14922
14923 format %{ "eorw $dst, $src1, $src2\t# int" %}
14924
14925 ins_cost(INSN_COST);
14926 ins_encode %{
14927 __ eorw(as_Register($dst$$reg),
14928 as_Register($src1$$reg),
14929 as_Register($src2$$reg));
14930 %}
14931
14932 ins_pipe(ialu_reg_reg);
14933 %}
14934
14935 instruct xorI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immILog src2) %{
14936 match(Set dst (XorI src1 src2));
14937
14938 format %{ "eorw $dst, $src1, $src2\t# int" %}
14939
14940 ins_cost(INSN_COST);
14941 ins_encode %{
14942 __ eorw(as_Register($dst$$reg),
14943 as_Register($src1$$reg),
14944 (uint64_t)($src2$$constant));
14945 %}
14946
14947 ins_pipe(ialu_reg_imm);
14948 %}
14949
14950 // Long Logical Instructions
14951 // TODO
14952
14953 instruct andL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2, rFlagsReg cr) %{
14954 match(Set dst (AndL src1 src2));
14955
14956 format %{ "and $dst, $src1, $src2\t# int" %}
14957
14958 ins_cost(INSN_COST);
14959 ins_encode %{
14960 __ andr(as_Register($dst$$reg),
14961 as_Register($src1$$reg),
14962 as_Register($src2$$reg));
14963 %}
14964
14965 ins_pipe(ialu_reg_reg);
14966 %}
14967
14968 instruct andL_reg_imm(iRegLNoSp dst, iRegL src1, immLLog src2, rFlagsReg cr) %{
14969 match(Set dst (AndL src1 src2));
14970
14971 format %{ "and $dst, $src1, $src2\t# int" %}
14972
14973 ins_cost(INSN_COST);
14974 ins_encode %{
14975 __ andr(as_Register($dst$$reg),
14976 as_Register($src1$$reg),
14977 (uint64_t)($src2$$constant));
14978 %}
14979
14980 ins_pipe(ialu_reg_imm);
14981 %}
14982
14983 // Or Instructions
14984
14985 instruct orL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
14986 match(Set dst (OrL src1 src2));
14987
14988 format %{ "orr $dst, $src1, $src2\t# int" %}
14989
14990 ins_cost(INSN_COST);
14991 ins_encode %{
14992 __ orr(as_Register($dst$$reg),
14993 as_Register($src1$$reg),
14994 as_Register($src2$$reg));
14995 %}
14996
14997 ins_pipe(ialu_reg_reg);
14998 %}
14999
15000 instruct orL_reg_imm(iRegLNoSp dst, iRegL src1, immLLog src2) %{
15001 match(Set dst (OrL src1 src2));
15002
15003 format %{ "orr $dst, $src1, $src2\t# int" %}
15004
15005 ins_cost(INSN_COST);
15006 ins_encode %{
15007 __ orr(as_Register($dst$$reg),
15008 as_Register($src1$$reg),
15009 (uint64_t)($src2$$constant));
15010 %}
15011
15012 ins_pipe(ialu_reg_imm);
15013 %}
15014
15015 // Xor Instructions
15016
15017 instruct xorL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
15018 match(Set dst (XorL src1 src2));
15019
15020 format %{ "eor $dst, $src1, $src2\t# int" %}
15021
15022 ins_cost(INSN_COST);
15023 ins_encode %{
15024 __ eor(as_Register($dst$$reg),
15025 as_Register($src1$$reg),
15026 as_Register($src2$$reg));
15027 %}
15028
15029 ins_pipe(ialu_reg_reg);
15030 %}
15031
15032 instruct xorL_reg_imm(iRegLNoSp dst, iRegL src1, immLLog src2) %{
15033 match(Set dst (XorL src1 src2));
15034
15035 ins_cost(INSN_COST);
15036 format %{ "eor $dst, $src1, $src2\t# int" %}
15037
15038 ins_encode %{
15039 __ eor(as_Register($dst$$reg),
15040 as_Register($src1$$reg),
15041 (uint64_t)($src2$$constant));
15042 %}
15043
15044 ins_pipe(ialu_reg_imm);
15045 %}
15046
15047 instruct convI2L_reg_reg(iRegLNoSp dst, iRegIorL2I src)
15048 %{
15049 match(Set dst (ConvI2L src));
15050
15051 ins_cost(INSN_COST);
15052 format %{ "sxtw $dst, $src\t# i2l" %}
15053 ins_encode %{
15054 __ sbfm($dst$$Register, $src$$Register, 0, 31);
15055 %}
15056 ins_pipe(ialu_reg_shift);
15057 %}
15058
15059 // this pattern occurs in bigmath arithmetic
15060 instruct convUI2L_reg_reg(iRegLNoSp dst, iRegIorL2I src, immL_32bits mask)
15061 %{
15062 match(Set dst (AndL (ConvI2L src) mask));
15063
15064 ins_cost(INSN_COST);
15065 format %{ "ubfm $dst, $src, 0, 31\t# ui2l" %}
15066 ins_encode %{
15067 __ ubfm($dst$$Register, $src$$Register, 0, 31);
15068 %}
15069
15070 ins_pipe(ialu_reg_shift);
15071 %}
15072
15073 instruct convL2I_reg(iRegINoSp dst, iRegL src) %{
15074 match(Set dst (ConvL2I src));
15075
15076 ins_cost(INSN_COST);
15077 format %{ "movw $dst, $src \t// l2i" %}
15078
15079 ins_encode %{
15080 __ movw(as_Register($dst$$reg), as_Register($src$$reg));
15081 %}
15082
15083 ins_pipe(ialu_reg);
15084 %}
15085
15086 instruct convI2B(iRegINoSp dst, iRegIorL2I src, rFlagsReg cr)
15087 %{
15088 match(Set dst (Conv2B src));
15089 effect(KILL cr);
15090
15091 format %{
15092 "cmpw $src, zr\n\t"
15093 "cset $dst, ne"
15094 %}
15095
15096 ins_encode %{
15097 __ cmpw(as_Register($src$$reg), zr);
15098 __ cset(as_Register($dst$$reg), Assembler::NE);
15099 %}
15100
15101 ins_pipe(ialu_reg);
15102 %}
15103
15104 instruct convP2B(iRegINoSp dst, iRegP src, rFlagsReg cr)
15105 %{
15106 match(Set dst (Conv2B src));
15107 effect(KILL cr);
15108
15109 format %{
15110 "cmp $src, zr\n\t"
15111 "cset $dst, ne"
15112 %}
15113
15114 ins_encode %{
15115 __ cmp(as_Register($src$$reg), zr);
15116 __ cset(as_Register($dst$$reg), Assembler::NE);
15117 %}
15118
15119 ins_pipe(ialu_reg);
15120 %}
15121
15122 instruct convD2F_reg(vRegF dst, vRegD src) %{
15123 match(Set dst (ConvD2F src));
15124
15125 ins_cost(INSN_COST * 5);
15126 format %{ "fcvtd $dst, $src \t// d2f" %}
15127
15128 ins_encode %{
15129 __ fcvtd(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg));
15130 %}
15131
15132 ins_pipe(fp_d2f);
15133 %}
15134
15135 instruct convF2D_reg(vRegD dst, vRegF src) %{
15136 match(Set dst (ConvF2D src));
15137
15138 ins_cost(INSN_COST * 5);
15139 format %{ "fcvts $dst, $src \t// f2d" %}
15140
15141 ins_encode %{
15142 __ fcvts(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg));
15143 %}
15144
15145 ins_pipe(fp_f2d);
15146 %}
15147
15148 instruct convF2I_reg_reg(iRegINoSp dst, vRegF src) %{
15149 match(Set dst (ConvF2I src));
15150
15151 ins_cost(INSN_COST * 5);
15152 format %{ "fcvtzsw $dst, $src \t// f2i" %}
15153
15154 ins_encode %{
15155 __ fcvtzsw(as_Register($dst$$reg), as_FloatRegister($src$$reg));
15156 %}
15157
15158 ins_pipe(fp_f2i);
15159 %}
15160
15161 instruct convF2L_reg_reg(iRegLNoSp dst, vRegF src) %{
15162 match(Set dst (ConvF2L src));
15163
15164 ins_cost(INSN_COST * 5);
15165 format %{ "fcvtzs $dst, $src \t// f2l" %}
15166
15167 ins_encode %{
15168 __ fcvtzs(as_Register($dst$$reg), as_FloatRegister($src$$reg));
15169 %}
15170
15171 ins_pipe(fp_f2l);
15172 %}
15173
15174 instruct convI2F_reg_reg(vRegF dst, iRegIorL2I src) %{
15175 match(Set dst (ConvI2F src));
15176
15177 ins_cost(INSN_COST * 5);
15178 format %{ "scvtfws $dst, $src \t// i2f" %}
15179
15180 ins_encode %{
15181 __ scvtfws(as_FloatRegister($dst$$reg), as_Register($src$$reg));
15182 %}
15183
15184 ins_pipe(fp_i2f);
15185 %}
15186
15187 instruct convL2F_reg_reg(vRegF dst, iRegL src) %{
15188 match(Set dst (ConvL2F src));
15189
15190 ins_cost(INSN_COST * 5);
15191 format %{ "scvtfs $dst, $src \t// l2f" %}
15192
15193 ins_encode %{
15194 __ scvtfs(as_FloatRegister($dst$$reg), as_Register($src$$reg));
15195 %}
15196
15197 ins_pipe(fp_l2f);
15198 %}
15199
15200 instruct convD2I_reg_reg(iRegINoSp dst, vRegD src) %{
15201 match(Set dst (ConvD2I src));
15202
15203 ins_cost(INSN_COST * 5);
15204 format %{ "fcvtzdw $dst, $src \t// d2i" %}
15205
15206 ins_encode %{
15207 __ fcvtzdw(as_Register($dst$$reg), as_FloatRegister($src$$reg));
15208 %}
15209
15210 ins_pipe(fp_d2i);
15211 %}
15212
15213 instruct convD2L_reg_reg(iRegLNoSp dst, vRegD src) %{
15214 match(Set dst (ConvD2L src));
15215
15216 ins_cost(INSN_COST * 5);
15217 format %{ "fcvtzd $dst, $src \t// d2l" %}
15218
15219 ins_encode %{
15220 __ fcvtzd(as_Register($dst$$reg), as_FloatRegister($src$$reg));
15221 %}
15222
15223 ins_pipe(fp_d2l);
15224 %}
15225
15226 instruct convI2D_reg_reg(vRegD dst, iRegIorL2I src) %{
15227 match(Set dst (ConvI2D src));
15228
15229 ins_cost(INSN_COST * 5);
15230 format %{ "scvtfwd $dst, $src \t// i2d" %}
15231
15232 ins_encode %{
15233 __ scvtfwd(as_FloatRegister($dst$$reg), as_Register($src$$reg));
15234 %}
15235
15236 ins_pipe(fp_i2d);
15237 %}
15238
15239 instruct convL2D_reg_reg(vRegD dst, iRegL src) %{
15240 match(Set dst (ConvL2D src));
15241
15242 ins_cost(INSN_COST * 5);
15243 format %{ "scvtfd $dst, $src \t// l2d" %}
15244
15245 ins_encode %{
15246 __ scvtfd(as_FloatRegister($dst$$reg), as_Register($src$$reg));
15247 %}
15248
15249 ins_pipe(fp_l2d);
15250 %}
15251
15252 // stack <-> reg and reg <-> reg shuffles with no conversion
15253
15254 instruct MoveF2I_stack_reg(iRegINoSp dst, stackSlotF src) %{
15255
15256 match(Set dst (MoveF2I src));
15257
15258 effect(DEF dst, USE src);
15259
15260 ins_cost(4 * INSN_COST);
15261
15262 format %{ "ldrw $dst, $src\t# MoveF2I_stack_reg" %}
15263
15264 ins_encode %{
15265 __ ldrw($dst$$Register, Address(sp, $src$$disp));
15266 %}
15267
15268 ins_pipe(iload_reg_reg);
15269
15270 %}
15271
15272 instruct MoveI2F_stack_reg(vRegF dst, stackSlotI src) %{
15273
15274 match(Set dst (MoveI2F src));
15275
15276 effect(DEF dst, USE src);
15277
15278 ins_cost(4 * INSN_COST);
15279
15280 format %{ "ldrs $dst, $src\t# MoveI2F_stack_reg" %}
15281
15282 ins_encode %{
15283 __ ldrs(as_FloatRegister($dst$$reg), Address(sp, $src$$disp));
15284 %}
15285
15286 ins_pipe(pipe_class_memory);
15287
15288 %}
15289
15290 instruct MoveD2L_stack_reg(iRegLNoSp dst, stackSlotD src) %{
15291
15292 match(Set dst (MoveD2L src));
15293
15294 effect(DEF dst, USE src);
15295
15296 ins_cost(4 * INSN_COST);
15297
15298 format %{ "ldr $dst, $src\t# MoveD2L_stack_reg" %}
15299
15300 ins_encode %{
15301 __ ldr($dst$$Register, Address(sp, $src$$disp));
15302 %}
15303
15304 ins_pipe(iload_reg_reg);
15305
15306 %}
15307
15308 instruct MoveL2D_stack_reg(vRegD dst, stackSlotL src) %{
15309
15310 match(Set dst (MoveL2D src));
15311
15312 effect(DEF dst, USE src);
15313
15314 ins_cost(4 * INSN_COST);
15315
15316 format %{ "ldrd $dst, $src\t# MoveL2D_stack_reg" %}
15317
15318 ins_encode %{
15319 __ ldrd(as_FloatRegister($dst$$reg), Address(sp, $src$$disp));
15320 %}
15321
15322 ins_pipe(pipe_class_memory);
15323
15324 %}
15325
15326 instruct MoveF2I_reg_stack(stackSlotI dst, vRegF src) %{
15327
15328 match(Set dst (MoveF2I src));
15329
15330 effect(DEF dst, USE src);
15331
15332 ins_cost(INSN_COST);
15333
15334 format %{ "strs $src, $dst\t# MoveF2I_reg_stack" %}
15335
15336 ins_encode %{
15337 __ strs(as_FloatRegister($src$$reg), Address(sp, $dst$$disp));
15338 %}
15339
15340 ins_pipe(pipe_class_memory);
15341
15342 %}
15343
15344 instruct MoveI2F_reg_stack(stackSlotF dst, iRegI src) %{
15345
15346 match(Set dst (MoveI2F src));
15347
15348 effect(DEF dst, USE src);
15349
15350 ins_cost(INSN_COST);
15351
15352 format %{ "strw $src, $dst\t# MoveI2F_reg_stack" %}
15353
15354 ins_encode %{
15355 __ strw($src$$Register, Address(sp, $dst$$disp));
15356 %}
15357
15358 ins_pipe(istore_reg_reg);
15359
15360 %}
15361
15362 instruct MoveD2L_reg_stack(stackSlotL dst, vRegD src) %{
15363
15364 match(Set dst (MoveD2L src));
15365
15366 effect(DEF dst, USE src);
15367
15368 ins_cost(INSN_COST);
15369
15370 format %{ "strd $dst, $src\t# MoveD2L_reg_stack" %}
15371
15372 ins_encode %{
15373 __ strd(as_FloatRegister($src$$reg), Address(sp, $dst$$disp));
15374 %}
15375
15376 ins_pipe(pipe_class_memory);
15377
15378 %}
15379
15380 instruct MoveL2D_reg_stack(stackSlotD dst, iRegL src) %{
15381
15382 match(Set dst (MoveL2D src));
15383
15384 effect(DEF dst, USE src);
15385
15386 ins_cost(INSN_COST);
15387
15388 format %{ "str $src, $dst\t# MoveL2D_reg_stack" %}
15389
15390 ins_encode %{
15391 __ str($src$$Register, Address(sp, $dst$$disp));
15392 %}
15393
15394 ins_pipe(istore_reg_reg);
15395
15396 %}
15397
15398 instruct MoveF2I_reg_reg(iRegINoSp dst, vRegF src) %{
15399
15400 match(Set dst (MoveF2I src));
15401
15402 effect(DEF dst, USE src);
15403
15404 ins_cost(INSN_COST);
15405
15406 format %{ "fmovs $dst, $src\t# MoveF2I_reg_reg" %}
15407
15408 ins_encode %{
15409 __ fmovs($dst$$Register, as_FloatRegister($src$$reg));
15410 %}
15411
15412 ins_pipe(fp_f2i);
15413
15414 %}
15415
15416 instruct MoveI2F_reg_reg(vRegF dst, iRegI src) %{
15417
15418 match(Set dst (MoveI2F src));
15419
15420 effect(DEF dst, USE src);
15421
15422 ins_cost(INSN_COST);
15423
15424 format %{ "fmovs $dst, $src\t# MoveI2F_reg_reg" %}
15425
15426 ins_encode %{
15427 __ fmovs(as_FloatRegister($dst$$reg), $src$$Register);
15428 %}
15429
15430 ins_pipe(fp_i2f);
15431
15432 %}
15433
15434 instruct MoveD2L_reg_reg(iRegLNoSp dst, vRegD src) %{
15435
15436 match(Set dst (MoveD2L src));
15437
15438 effect(DEF dst, USE src);
15439
15440 ins_cost(INSN_COST);
15441
15442 format %{ "fmovd $dst, $src\t# MoveD2L_reg_reg" %}
15443
15444 ins_encode %{
15445 __ fmovd($dst$$Register, as_FloatRegister($src$$reg));
15446 %}
15447
15448 ins_pipe(fp_d2l);
15449
15450 %}
15451
15452 instruct MoveL2D_reg_reg(vRegD dst, iRegL src) %{
15453
15454 match(Set dst (MoveL2D src));
15455
15456 effect(DEF dst, USE src);
15457
15458 ins_cost(INSN_COST);
15459
15460 format %{ "fmovd $dst, $src\t# MoveL2D_reg_reg" %}
15461
15462 ins_encode %{
15463 __ fmovd(as_FloatRegister($dst$$reg), $src$$Register);
15464 %}
15465
15466 ins_pipe(fp_l2d);
15467
15468 %}
15469
15470 // ============================================================================
15471 // clearing of an array
15472
15473 instruct clearArray_reg_reg(iRegL_R11 cnt, iRegP_R10 base, Universe dummy, rFlagsReg cr)
15474 %{
15475 match(Set dummy (ClearArray cnt base));
15476 effect(USE_KILL cnt, USE_KILL base, KILL cr);
15477
15478 ins_cost(4 * INSN_COST);
15479 format %{ "ClearArray $cnt, $base" %}
15480
15481 ins_encode %{
15482 address tpc = __ zero_words($base$$Register, $cnt$$Register);
15483 if (tpc == NULL) {
15484 ciEnv::current()->record_failure("CodeCache is full");
15485 return;
15486 }
15487 %}
15488
15489 ins_pipe(pipe_class_memory);
15490 %}
15491
15492 instruct clearArray_imm_reg(immL cnt, iRegP_R10 base, iRegL_R11 temp, Universe dummy, rFlagsReg cr)
15493 %{
15494 predicate((uint64_t)n->in(2)->get_long()
15495 < (uint64_t)(BlockZeroingLowLimit >> LogBytesPerWord));
15496 match(Set dummy (ClearArray cnt base));
15497 effect(TEMP temp, USE_KILL base, KILL cr);
15498
15499 ins_cost(4 * INSN_COST);
15500 format %{ "ClearArray $cnt, $base" %}
15501
15502 ins_encode %{
15503 address tpc = __ zero_words($base$$Register, (uint64_t)$cnt$$constant);
15504 if (tpc == NULL) {
15505 ciEnv::current()->record_failure("CodeCache is full");
15506 return;
15507 }
15508 %}
15509
15510 ins_pipe(pipe_class_memory);
15511 %}
15512
15513 // ============================================================================
15514 // Overflow Math Instructions
15515
15516 instruct overflowAddI_reg_reg(rFlagsReg cr, iRegIorL2I op1, iRegIorL2I op2)
15517 %{
15518 match(Set cr (OverflowAddI op1 op2));
15519
15520 format %{ "cmnw $op1, $op2\t# overflow check int" %}
15521 ins_cost(INSN_COST);
15522 ins_encode %{
15523 __ cmnw($op1$$Register, $op2$$Register);
15524 %}
15525
15526 ins_pipe(icmp_reg_reg);
15527 %}
15528
15529 instruct overflowAddI_reg_imm(rFlagsReg cr, iRegIorL2I op1, immIAddSub op2)
15530 %{
15531 match(Set cr (OverflowAddI op1 op2));
15532
15533 format %{ "cmnw $op1, $op2\t# overflow check int" %}
15534 ins_cost(INSN_COST);
15535 ins_encode %{
15536 __ cmnw($op1$$Register, $op2$$constant);
15537 %}
15538
15539 ins_pipe(icmp_reg_imm);
15540 %}
15541
15542 instruct overflowAddL_reg_reg(rFlagsReg cr, iRegL op1, iRegL op2)
15543 %{
15544 match(Set cr (OverflowAddL op1 op2));
15545
15546 format %{ "cmn $op1, $op2\t# overflow check long" %}
15547 ins_cost(INSN_COST);
15548 ins_encode %{
15549 __ cmn($op1$$Register, $op2$$Register);
15550 %}
15551
15552 ins_pipe(icmp_reg_reg);
15553 %}
15554
15555 instruct overflowAddL_reg_imm(rFlagsReg cr, iRegL op1, immLAddSub op2)
15556 %{
15557 match(Set cr (OverflowAddL op1 op2));
15558
15559 format %{ "adds zr, $op1, $op2\t# overflow check long" %}
15560 ins_cost(INSN_COST);
15561 ins_encode %{
15562 __ adds(zr, $op1$$Register, $op2$$constant);
15563 %}
15564
15565 ins_pipe(icmp_reg_imm);
15566 %}
15567
15568 instruct overflowSubI_reg_reg(rFlagsReg cr, iRegIorL2I op1, iRegIorL2I op2)
15569 %{
15570 match(Set cr (OverflowSubI op1 op2));
15571
15572 format %{ "cmpw $op1, $op2\t# overflow check int" %}
15573 ins_cost(INSN_COST);
15574 ins_encode %{
15575 __ cmpw($op1$$Register, $op2$$Register);
15576 %}
15577
15578 ins_pipe(icmp_reg_reg);
15579 %}
15580
15581 instruct overflowSubI_reg_imm(rFlagsReg cr, iRegIorL2I op1, immIAddSub op2)
15582 %{
15583 match(Set cr (OverflowSubI op1 op2));
15584
15585 format %{ "cmpw $op1, $op2\t# overflow check int" %}
15586 ins_cost(INSN_COST);
15587 ins_encode %{
15588 __ cmpw($op1$$Register, $op2$$constant);
15589 %}
15590
15591 ins_pipe(icmp_reg_imm);
15592 %}
15593
15594 instruct overflowSubL_reg_reg(rFlagsReg cr, iRegL op1, iRegL op2)
15595 %{
15596 match(Set cr (OverflowSubL op1 op2));
15597
15598 format %{ "cmp $op1, $op2\t# overflow check long" %}
15599 ins_cost(INSN_COST);
15600 ins_encode %{
15601 __ cmp($op1$$Register, $op2$$Register);
15602 %}
15603
15604 ins_pipe(icmp_reg_reg);
15605 %}
15606
15607 instruct overflowSubL_reg_imm(rFlagsReg cr, iRegL op1, immLAddSub op2)
15608 %{
15609 match(Set cr (OverflowSubL op1 op2));
15610
15611 format %{ "cmp $op1, $op2\t# overflow check long" %}
15612 ins_cost(INSN_COST);
15613 ins_encode %{
15614 __ subs(zr, $op1$$Register, $op2$$constant);
15615 %}
15616
15617 ins_pipe(icmp_reg_imm);
15618 %}
15619
15620 instruct overflowNegI_reg(rFlagsReg cr, immI0 zero, iRegIorL2I op1)
15621 %{
15622 match(Set cr (OverflowSubI zero op1));
15623
15624 format %{ "cmpw zr, $op1\t# overflow check int" %}
15625 ins_cost(INSN_COST);
15626 ins_encode %{
15627 __ cmpw(zr, $op1$$Register);
15628 %}
15629
15630 ins_pipe(icmp_reg_imm);
15631 %}
15632
15633 instruct overflowNegL_reg(rFlagsReg cr, immI0 zero, iRegL op1)
15634 %{
15635 match(Set cr (OverflowSubL zero op1));
15636
15637 format %{ "cmp zr, $op1\t# overflow check long" %}
15638 ins_cost(INSN_COST);
15639 ins_encode %{
15640 __ cmp(zr, $op1$$Register);
15641 %}
15642
15643 ins_pipe(icmp_reg_imm);
15644 %}
15645
15646 instruct overflowMulI_reg(rFlagsReg cr, iRegIorL2I op1, iRegIorL2I op2)
15647 %{
15648 match(Set cr (OverflowMulI op1 op2));
15649
15650 format %{ "smull rscratch1, $op1, $op2\t# overflow check int\n\t"
15651 "cmp rscratch1, rscratch1, sxtw\n\t"
15652 "movw rscratch1, #0x80000000\n\t"
15653 "cselw rscratch1, rscratch1, zr, NE\n\t"
15654 "cmpw rscratch1, #1" %}
15655 ins_cost(5 * INSN_COST);
15656 ins_encode %{
15657 __ smull(rscratch1, $op1$$Register, $op2$$Register);
15658 __ subs(zr, rscratch1, rscratch1, ext::sxtw); // NE => overflow
15659 __ movw(rscratch1, 0x80000000); // Develop 0 (EQ),
15660 __ cselw(rscratch1, rscratch1, zr, Assembler::NE); // or 0x80000000 (NE)
15661 __ cmpw(rscratch1, 1); // 0x80000000 - 1 => VS
15662 %}
15663
15664 ins_pipe(pipe_slow);
15665 %}
15666
15667 instruct overflowMulI_reg_branch(cmpOp cmp, iRegIorL2I op1, iRegIorL2I op2, label labl, rFlagsReg cr)
15668 %{
15669 match(If cmp (OverflowMulI op1 op2));
15670 predicate(n->in(1)->as_Bool()->_test._test == BoolTest::overflow
15671 || n->in(1)->as_Bool()->_test._test == BoolTest::no_overflow);
15672 effect(USE labl, KILL cr);
15673
15674 format %{ "smull rscratch1, $op1, $op2\t# overflow check int\n\t"
15675 "cmp rscratch1, rscratch1, sxtw\n\t"
15676 "b$cmp $labl" %}
15677 ins_cost(3 * INSN_COST); // Branch is rare so treat as INSN_COST
15678 ins_encode %{
15679 Label* L = $labl$$label;
15680 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15681 __ smull(rscratch1, $op1$$Register, $op2$$Register);
15682 __ subs(zr, rscratch1, rscratch1, ext::sxtw); // NE => overflow
15683 __ br(cond == Assembler::VS ? Assembler::NE : Assembler::EQ, *L);
15684 %}
15685
15686 ins_pipe(pipe_serial);
15687 %}
15688
15689 instruct overflowMulL_reg(rFlagsReg cr, iRegL op1, iRegL op2)
15690 %{
15691 match(Set cr (OverflowMulL op1 op2));
15692
15693 format %{ "mul rscratch1, $op1, $op2\t#overflow check long\n\t"
15694 "smulh rscratch2, $op1, $op2\n\t"
15695 "cmp rscratch2, rscratch1, ASR #63\n\t"
15696 "movw rscratch1, #0x80000000\n\t"
15697 "cselw rscratch1, rscratch1, zr, NE\n\t"
15698 "cmpw rscratch1, #1" %}
15699 ins_cost(6 * INSN_COST);
15700 ins_encode %{
15701 __ mul(rscratch1, $op1$$Register, $op2$$Register); // Result bits 0..63
15702 __ smulh(rscratch2, $op1$$Register, $op2$$Register); // Result bits 64..127
15703 __ cmp(rscratch2, rscratch1, Assembler::ASR, 63); // Top is pure sign ext
15704 __ movw(rscratch1, 0x80000000); // Develop 0 (EQ),
15705 __ cselw(rscratch1, rscratch1, zr, Assembler::NE); // or 0x80000000 (NE)
15706 __ cmpw(rscratch1, 1); // 0x80000000 - 1 => VS
15707 %}
15708
15709 ins_pipe(pipe_slow);
15710 %}
15711
15712 instruct overflowMulL_reg_branch(cmpOp cmp, iRegL op1, iRegL op2, label labl, rFlagsReg cr)
15713 %{
15714 match(If cmp (OverflowMulL op1 op2));
15715 predicate(n->in(1)->as_Bool()->_test._test == BoolTest::overflow
15716 || n->in(1)->as_Bool()->_test._test == BoolTest::no_overflow);
15717 effect(USE labl, KILL cr);
15718
15719 format %{ "mul rscratch1, $op1, $op2\t#overflow check long\n\t"
15720 "smulh rscratch2, $op1, $op2\n\t"
15721 "cmp rscratch2, rscratch1, ASR #63\n\t"
15722 "b$cmp $labl" %}
15723 ins_cost(4 * INSN_COST); // Branch is rare so treat as INSN_COST
15724 ins_encode %{
15725 Label* L = $labl$$label;
15726 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15727 __ mul(rscratch1, $op1$$Register, $op2$$Register); // Result bits 0..63
15728 __ smulh(rscratch2, $op1$$Register, $op2$$Register); // Result bits 64..127
15729 __ cmp(rscratch2, rscratch1, Assembler::ASR, 63); // Top is pure sign ext
15730 __ br(cond == Assembler::VS ? Assembler::NE : Assembler::EQ, *L);
15731 %}
15732
15733 ins_pipe(pipe_serial);
15734 %}
15735
15736 // ============================================================================
15737 // Compare Instructions
15738
15739 instruct compI_reg_reg(rFlagsReg cr, iRegI op1, iRegI op2)
15740 %{
15741 match(Set cr (CmpI op1 op2));
15742
15743 effect(DEF cr, USE op1, USE op2);
15744
15745 ins_cost(INSN_COST);
15746 format %{ "cmpw $op1, $op2" %}
15747
15748 ins_encode(aarch64_enc_cmpw(op1, op2));
15749
15750 ins_pipe(icmp_reg_reg);
15751 %}
15752
15753 instruct compI_reg_immI0(rFlagsReg cr, iRegI op1, immI0 zero)
15754 %{
15755 match(Set cr (CmpI op1 zero));
15756
15757 effect(DEF cr, USE op1);
15758
15759 ins_cost(INSN_COST);
15760 format %{ "cmpw $op1, 0" %}
15761
15762 ins_encode(aarch64_enc_cmpw_imm_addsub(op1, zero));
15763
15764 ins_pipe(icmp_reg_imm);
15765 %}
15766
15767 instruct compI_reg_immIAddSub(rFlagsReg cr, iRegI op1, immIAddSub op2)
15768 %{
15769 match(Set cr (CmpI op1 op2));
15770
15771 effect(DEF cr, USE op1);
15772
15773 ins_cost(INSN_COST);
15774 format %{ "cmpw $op1, $op2" %}
15775
15776 ins_encode(aarch64_enc_cmpw_imm_addsub(op1, op2));
15777
15778 ins_pipe(icmp_reg_imm);
15779 %}
15780
15781 instruct compI_reg_immI(rFlagsReg cr, iRegI op1, immI op2)
15782 %{
15783 match(Set cr (CmpI op1 op2));
15784
15785 effect(DEF cr, USE op1);
15786
15787 ins_cost(INSN_COST * 2);
15788 format %{ "cmpw $op1, $op2" %}
15789
15790 ins_encode(aarch64_enc_cmpw_imm(op1, op2));
15791
15792 ins_pipe(icmp_reg_imm);
15793 %}
15794
15795 // Unsigned compare Instructions; really, same as signed compare
15796 // except it should only be used to feed an If or a CMovI which takes a
15797 // cmpOpU.
15798
15799 instruct compU_reg_reg(rFlagsRegU cr, iRegI op1, iRegI op2)
15800 %{
15801 match(Set cr (CmpU op1 op2));
15802
15803 effect(DEF cr, USE op1, USE op2);
15804
15805 ins_cost(INSN_COST);
15806 format %{ "cmpw $op1, $op2\t# unsigned" %}
15807
15808 ins_encode(aarch64_enc_cmpw(op1, op2));
15809
15810 ins_pipe(icmp_reg_reg);
15811 %}
15812
15813 instruct compU_reg_immI0(rFlagsRegU cr, iRegI op1, immI0 zero)
15814 %{
15815 match(Set cr (CmpU op1 zero));
15816
15817 effect(DEF cr, USE op1);
15818
15819 ins_cost(INSN_COST);
15820 format %{ "cmpw $op1, #0\t# unsigned" %}
15821
15822 ins_encode(aarch64_enc_cmpw_imm_addsub(op1, zero));
15823
15824 ins_pipe(icmp_reg_imm);
15825 %}
15826
15827 instruct compU_reg_immIAddSub(rFlagsRegU cr, iRegI op1, immIAddSub op2)
15828 %{
15829 match(Set cr (CmpU op1 op2));
15830
15831 effect(DEF cr, USE op1);
15832
15833 ins_cost(INSN_COST);
15834 format %{ "cmpw $op1, $op2\t# unsigned" %}
15835
15836 ins_encode(aarch64_enc_cmpw_imm_addsub(op1, op2));
15837
15838 ins_pipe(icmp_reg_imm);
15839 %}
15840
15841 instruct compU_reg_immI(rFlagsRegU cr, iRegI op1, immI op2)
15842 %{
15843 match(Set cr (CmpU op1 op2));
15844
15845 effect(DEF cr, USE op1);
15846
15847 ins_cost(INSN_COST * 2);
15848 format %{ "cmpw $op1, $op2\t# unsigned" %}
15849
15850 ins_encode(aarch64_enc_cmpw_imm(op1, op2));
15851
15852 ins_pipe(icmp_reg_imm);
15853 %}
15854
15855 instruct compL_reg_reg(rFlagsReg cr, iRegL op1, iRegL op2)
15856 %{
15857 match(Set cr (CmpL op1 op2));
15858
15859 effect(DEF cr, USE op1, USE op2);
15860
15861 ins_cost(INSN_COST);
15862 format %{ "cmp $op1, $op2" %}
15863
15864 ins_encode(aarch64_enc_cmp(op1, op2));
15865
15866 ins_pipe(icmp_reg_reg);
15867 %}
15868
15869 instruct compL_reg_immL0(rFlagsReg cr, iRegL op1, immL0 zero)
15870 %{
15871 match(Set cr (CmpL op1 zero));
15872
15873 effect(DEF cr, USE op1);
15874
15875 ins_cost(INSN_COST);
15876 format %{ "tst $op1" %}
15877
15878 ins_encode(aarch64_enc_cmp_imm_addsub(op1, zero));
15879
15880 ins_pipe(icmp_reg_imm);
15881 %}
15882
15883 instruct compL_reg_immLAddSub(rFlagsReg cr, iRegL op1, immLAddSub op2)
15884 %{
15885 match(Set cr (CmpL op1 op2));
15886
15887 effect(DEF cr, USE op1);
15888
15889 ins_cost(INSN_COST);
15890 format %{ "cmp $op1, $op2" %}
15891
15892 ins_encode(aarch64_enc_cmp_imm_addsub(op1, op2));
15893
15894 ins_pipe(icmp_reg_imm);
15895 %}
15896
15897 instruct compL_reg_immL(rFlagsReg cr, iRegL op1, immL op2)
15898 %{
15899 match(Set cr (CmpL op1 op2));
15900
15901 effect(DEF cr, USE op1);
15902
15903 ins_cost(INSN_COST * 2);
15904 format %{ "cmp $op1, $op2" %}
15905
15906 ins_encode(aarch64_enc_cmp_imm(op1, op2));
15907
15908 ins_pipe(icmp_reg_imm);
15909 %}
15910
15911 instruct compUL_reg_reg(rFlagsRegU cr, iRegL op1, iRegL op2)
15912 %{
15913 match(Set cr (CmpUL op1 op2));
15914
15915 effect(DEF cr, USE op1, USE op2);
15916
15917 ins_cost(INSN_COST);
15918 format %{ "cmp $op1, $op2" %}
15919
15920 ins_encode(aarch64_enc_cmp(op1, op2));
15921
15922 ins_pipe(icmp_reg_reg);
15923 %}
15924
15925 instruct compUL_reg_immL0(rFlagsRegU cr, iRegL op1, immL0 zero)
15926 %{
15927 match(Set cr (CmpUL op1 zero));
15928
15929 effect(DEF cr, USE op1);
15930
15931 ins_cost(INSN_COST);
15932 format %{ "tst $op1" %}
15933
15934 ins_encode(aarch64_enc_cmp_imm_addsub(op1, zero));
15935
15936 ins_pipe(icmp_reg_imm);
15937 %}
15938
15939 instruct compUL_reg_immLAddSub(rFlagsRegU cr, iRegL op1, immLAddSub op2)
15940 %{
15941 match(Set cr (CmpUL op1 op2));
15942
15943 effect(DEF cr, USE op1);
15944
15945 ins_cost(INSN_COST);
15946 format %{ "cmp $op1, $op2" %}
15947
15948 ins_encode(aarch64_enc_cmp_imm_addsub(op1, op2));
15949
15950 ins_pipe(icmp_reg_imm);
15951 %}
15952
15953 instruct compUL_reg_immL(rFlagsRegU cr, iRegL op1, immL op2)
15954 %{
15955 match(Set cr (CmpUL op1 op2));
15956
15957 effect(DEF cr, USE op1);
15958
15959 ins_cost(INSN_COST * 2);
15960 format %{ "cmp $op1, $op2" %}
15961
15962 ins_encode(aarch64_enc_cmp_imm(op1, op2));
15963
15964 ins_pipe(icmp_reg_imm);
15965 %}
15966
15967 instruct compP_reg_reg(rFlagsRegU cr, iRegP op1, iRegP op2)
15968 %{
15969 match(Set cr (CmpP op1 op2));
15970
15971 effect(DEF cr, USE op1, USE op2);
15972
15973 ins_cost(INSN_COST);
15974 format %{ "cmp $op1, $op2\t // ptr" %}
15975
15976 ins_encode(aarch64_enc_cmpp(op1, op2));
15977
15978 ins_pipe(icmp_reg_reg);
15979 %}
15980
15981 instruct compN_reg_reg(rFlagsRegU cr, iRegN op1, iRegN op2)
15982 %{
15983 match(Set cr (CmpN op1 op2));
15984
15985 effect(DEF cr, USE op1, USE op2);
15986
15987 ins_cost(INSN_COST);
15988 format %{ "cmp $op1, $op2\t // compressed ptr" %}
15989
15990 ins_encode(aarch64_enc_cmpn(op1, op2));
15991
15992 ins_pipe(icmp_reg_reg);
15993 %}
15994
15995 instruct testP_reg(rFlagsRegU cr, iRegP op1, immP0 zero)
15996 %{
15997 match(Set cr (CmpP op1 zero));
15998
15999 effect(DEF cr, USE op1, USE zero);
16000
16001 ins_cost(INSN_COST);
16002 format %{ "cmp $op1, 0\t // ptr" %}
16003
16004 ins_encode(aarch64_enc_testp(op1));
16005
16006 ins_pipe(icmp_reg_imm);
16007 %}
16008
16009 instruct testN_reg(rFlagsRegU cr, iRegN op1, immN0 zero)
16010 %{
16011 match(Set cr (CmpN op1 zero));
16012
16013 effect(DEF cr, USE op1, USE zero);
16014
16015 ins_cost(INSN_COST);
16016 format %{ "cmp $op1, 0\t // compressed ptr" %}
16017
16018 ins_encode(aarch64_enc_testn(op1));
16019
16020 ins_pipe(icmp_reg_imm);
16021 %}
16022
16023 // FP comparisons
16024 //
16025 // n.b. CmpF/CmpD set a normal flags reg which then gets compared
16026 // using normal cmpOp. See declaration of rFlagsReg for details.
16027
16028 instruct compF_reg_reg(rFlagsReg cr, vRegF src1, vRegF src2)
16029 %{
16030 match(Set cr (CmpF src1 src2));
16031
16032 ins_cost(3 * INSN_COST);
16033 format %{ "fcmps $src1, $src2" %}
16034
16035 ins_encode %{
16036 __ fcmps(as_FloatRegister($src1$$reg), as_FloatRegister($src2$$reg));
16037 %}
16038
16039 ins_pipe(pipe_class_compare);
16040 %}
16041
16042 instruct compF_reg_zero(rFlagsReg cr, vRegF src1, immF0 src2)
16043 %{
16044 match(Set cr (CmpF src1 src2));
16045
16046 ins_cost(3 * INSN_COST);
16047 format %{ "fcmps $src1, 0.0" %}
16048
16049 ins_encode %{
16050 __ fcmps(as_FloatRegister($src1$$reg), 0.0);
16051 %}
16052
16053 ins_pipe(pipe_class_compare);
16054 %}
16055 // FROM HERE
16056
16057 instruct compD_reg_reg(rFlagsReg cr, vRegD src1, vRegD src2)
16058 %{
16059 match(Set cr (CmpD src1 src2));
16060
16061 ins_cost(3 * INSN_COST);
16062 format %{ "fcmpd $src1, $src2" %}
16063
16064 ins_encode %{
16065 __ fcmpd(as_FloatRegister($src1$$reg), as_FloatRegister($src2$$reg));
16066 %}
16067
16068 ins_pipe(pipe_class_compare);
16069 %}
16070
16071 instruct compD_reg_zero(rFlagsReg cr, vRegD src1, immD0 src2)
16072 %{
16073 match(Set cr (CmpD src1 src2));
16074
16075 ins_cost(3 * INSN_COST);
16076 format %{ "fcmpd $src1, 0.0" %}
16077
16078 ins_encode %{
16079 __ fcmpd(as_FloatRegister($src1$$reg), 0.0);
16080 %}
16081
16082 ins_pipe(pipe_class_compare);
16083 %}
16084
16085 instruct compF3_reg_reg(iRegINoSp dst, vRegF src1, vRegF src2, rFlagsReg cr)
16086 %{
16087 match(Set dst (CmpF3 src1 src2));
16088 effect(KILL cr);
16089
16090 ins_cost(5 * INSN_COST);
16091 format %{ "fcmps $src1, $src2\n\t"
16092 "csinvw($dst, zr, zr, eq\n\t"
16093 "csnegw($dst, $dst, $dst, lt)"
16094 %}
16095
16096 ins_encode %{
16097 Label done;
16098 FloatRegister s1 = as_FloatRegister($src1$$reg);
16099 FloatRegister s2 = as_FloatRegister($src2$$reg);
16100 Register d = as_Register($dst$$reg);
16101 __ fcmps(s1, s2);
16102 // installs 0 if EQ else -1
16103 __ csinvw(d, zr, zr, Assembler::EQ);
16104 // keeps -1 if less or unordered else installs 1
16105 __ csnegw(d, d, d, Assembler::LT);
16106 __ bind(done);
16107 %}
16108
16109 ins_pipe(pipe_class_default);
16110
16111 %}
16112
16113 instruct compD3_reg_reg(iRegINoSp dst, vRegD src1, vRegD src2, rFlagsReg cr)
16114 %{
16115 match(Set dst (CmpD3 src1 src2));
16116 effect(KILL cr);
16117
16118 ins_cost(5 * INSN_COST);
16119 format %{ "fcmpd $src1, $src2\n\t"
16120 "csinvw($dst, zr, zr, eq\n\t"
16121 "csnegw($dst, $dst, $dst, lt)"
16122 %}
16123
16124 ins_encode %{
16125 Label done;
16126 FloatRegister s1 = as_FloatRegister($src1$$reg);
16127 FloatRegister s2 = as_FloatRegister($src2$$reg);
16128 Register d = as_Register($dst$$reg);
16129 __ fcmpd(s1, s2);
16130 // installs 0 if EQ else -1
16131 __ csinvw(d, zr, zr, Assembler::EQ);
16132 // keeps -1 if less or unordered else installs 1
16133 __ csnegw(d, d, d, Assembler::LT);
16134 __ bind(done);
16135 %}
16136 ins_pipe(pipe_class_default);
16137
16138 %}
16139
16140 instruct compF3_reg_immF0(iRegINoSp dst, vRegF src1, immF0 zero, rFlagsReg cr)
16141 %{
16142 match(Set dst (CmpF3 src1 zero));
16143 effect(KILL cr);
16144
16145 ins_cost(5 * INSN_COST);
16146 format %{ "fcmps $src1, 0.0\n\t"
16147 "csinvw($dst, zr, zr, eq\n\t"
16148 "csnegw($dst, $dst, $dst, lt)"
16149 %}
16150
16151 ins_encode %{
16152 Label done;
16153 FloatRegister s1 = as_FloatRegister($src1$$reg);
16154 Register d = as_Register($dst$$reg);
16155 __ fcmps(s1, 0.0);
16156 // installs 0 if EQ else -1
16157 __ csinvw(d, zr, zr, Assembler::EQ);
16158 // keeps -1 if less or unordered else installs 1
16159 __ csnegw(d, d, d, Assembler::LT);
16160 __ bind(done);
16161 %}
16162
16163 ins_pipe(pipe_class_default);
16164
16165 %}
16166
16167 instruct compD3_reg_immD0(iRegINoSp dst, vRegD src1, immD0 zero, rFlagsReg cr)
16168 %{
16169 match(Set dst (CmpD3 src1 zero));
16170 effect(KILL cr);
16171
16172 ins_cost(5 * INSN_COST);
16173 format %{ "fcmpd $src1, 0.0\n\t"
16174 "csinvw($dst, zr, zr, eq\n\t"
16175 "csnegw($dst, $dst, $dst, lt)"
16176 %}
16177
16178 ins_encode %{
16179 Label done;
16180 FloatRegister s1 = as_FloatRegister($src1$$reg);
16181 Register d = as_Register($dst$$reg);
16182 __ fcmpd(s1, 0.0);
16183 // installs 0 if EQ else -1
16184 __ csinvw(d, zr, zr, Assembler::EQ);
16185 // keeps -1 if less or unordered else installs 1
16186 __ csnegw(d, d, d, Assembler::LT);
16187 __ bind(done);
16188 %}
16189 ins_pipe(pipe_class_default);
16190
16191 %}
16192
16193 instruct cmpLTMask_reg_reg(iRegINoSp dst, iRegIorL2I p, iRegIorL2I q, rFlagsReg cr)
16194 %{
16195 match(Set dst (CmpLTMask p q));
16196 effect(KILL cr);
16197
16198 ins_cost(3 * INSN_COST);
16199
16200 format %{ "cmpw $p, $q\t# cmpLTMask\n\t"
16201 "csetw $dst, lt\n\t"
16202 "subw $dst, zr, $dst"
16203 %}
16204
16205 ins_encode %{
16206 __ cmpw(as_Register($p$$reg), as_Register($q$$reg));
16207 __ csetw(as_Register($dst$$reg), Assembler::LT);
16208 __ subw(as_Register($dst$$reg), zr, as_Register($dst$$reg));
16209 %}
16210
16211 ins_pipe(ialu_reg_reg);
16212 %}
16213
16214 instruct cmpLTMask_reg_zero(iRegINoSp dst, iRegIorL2I src, immI0 zero, rFlagsReg cr)
16215 %{
16216 match(Set dst (CmpLTMask src zero));
16217 effect(KILL cr);
16218
16219 ins_cost(INSN_COST);
16220
16221 format %{ "asrw $dst, $src, #31\t# cmpLTMask0" %}
16222
16223 ins_encode %{
16224 __ asrw(as_Register($dst$$reg), as_Register($src$$reg), 31);
16225 %}
16226
16227 ins_pipe(ialu_reg_shift);
16228 %}
16229
16230 // ============================================================================
16231 // Max and Min
16232
16233 // Like compI_reg_reg or compI_reg_immI0 but without match rule and second zero parameter.
16234
16235 instruct compI_reg_imm0(rFlagsReg cr, iRegI src)
16236 %{
16237 effect(DEF cr, USE src);
16238 ins_cost(INSN_COST);
16239 format %{ "cmpw $src, 0" %}
16240
16241 ins_encode %{
16242 __ cmpw($src$$Register, 0);
16243 %}
16244 ins_pipe(icmp_reg_imm);
16245 %}
16246
16247 instruct minI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2)
16248 %{
16249 match(Set dst (MinI src1 src2));
16250 ins_cost(INSN_COST * 3);
16251
16252 expand %{
16253 rFlagsReg cr;
16254 compI_reg_reg(cr, src1, src2);
16255 cmovI_reg_reg_lt(dst, src1, src2, cr);
16256 %}
16257 %}
16258
16259 instruct maxI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2)
16260 %{
16261 match(Set dst (MaxI src1 src2));
16262 ins_cost(INSN_COST * 3);
16263
16264 expand %{
16265 rFlagsReg cr;
16266 compI_reg_reg(cr, src1, src2);
16267 cmovI_reg_reg_gt(dst, src1, src2, cr);
16268 %}
16269 %}
16270
16271
16272 // ============================================================================
16273 // Branch Instructions
16274
16275 // Direct Branch.
16276 instruct branch(label lbl)
16277 %{
16278 match(Goto);
16279
16280 effect(USE lbl);
16281
16282 ins_cost(BRANCH_COST);
16283 format %{ "b $lbl" %}
16284
16285 ins_encode(aarch64_enc_b(lbl));
16286
16287 ins_pipe(pipe_branch);
16288 %}
16289
16290 // Conditional Near Branch
16291 instruct branchCon(cmpOp cmp, rFlagsReg cr, label lbl)
16292 %{
16293 // Same match rule as `branchConFar'.
16294 match(If cmp cr);
16295
16296 effect(USE lbl);
16297
16298 ins_cost(BRANCH_COST);
16299 // If set to 1 this indicates that the current instruction is a
16300 // short variant of a long branch. This avoids using this
16301 // instruction in first-pass matching. It will then only be used in
16302 // the `Shorten_branches' pass.
16303 // ins_short_branch(1);
16304 format %{ "b$cmp $lbl" %}
16305
16306 ins_encode(aarch64_enc_br_con(cmp, lbl));
16307
16308 ins_pipe(pipe_branch_cond);
16309 %}
16310
16311 // Conditional Near Branch Unsigned
16312 instruct branchConU(cmpOpU cmp, rFlagsRegU cr, label lbl)
16313 %{
16314 // Same match rule as `branchConFar'.
16315 match(If cmp cr);
16316
16317 effect(USE lbl);
16318
16319 ins_cost(BRANCH_COST);
16320 // If set to 1 this indicates that the current instruction is a
16321 // short variant of a long branch. This avoids using this
16322 // instruction in first-pass matching. It will then only be used in
16323 // the `Shorten_branches' pass.
16324 // ins_short_branch(1);
16325 format %{ "b$cmp $lbl\t# unsigned" %}
16326
16327 ins_encode(aarch64_enc_br_conU(cmp, lbl));
16328
16329 ins_pipe(pipe_branch_cond);
16330 %}
16331
16332 // Make use of CBZ and CBNZ. These instructions, as well as being
16333 // shorter than (cmp; branch), have the additional benefit of not
16334 // killing the flags.
16335
16336 instruct cmpI_imm0_branch(cmpOpEqNe cmp, iRegIorL2I op1, immI0 op2, label labl, rFlagsReg cr) %{
16337 match(If cmp (CmpI op1 op2));
16338 effect(USE labl);
16339
16340 ins_cost(BRANCH_COST);
16341 format %{ "cbw$cmp $op1, $labl" %}
16342 ins_encode %{
16343 Label* L = $labl$$label;
16344 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
16345 if (cond == Assembler::EQ)
16346 __ cbzw($op1$$Register, *L);
16347 else
16348 __ cbnzw($op1$$Register, *L);
16349 %}
16350 ins_pipe(pipe_cmp_branch);
16351 %}
16352
16353 instruct cmpL_imm0_branch(cmpOpEqNe cmp, iRegL op1, immL0 op2, label labl, rFlagsReg cr) %{
16354 match(If cmp (CmpL op1 op2));
16355 effect(USE labl);
16356
16357 ins_cost(BRANCH_COST);
16358 format %{ "cb$cmp $op1, $labl" %}
16359 ins_encode %{
16360 Label* L = $labl$$label;
16361 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
16362 if (cond == Assembler::EQ)
16363 __ cbz($op1$$Register, *L);
16364 else
16365 __ cbnz($op1$$Register, *L);
16366 %}
16367 ins_pipe(pipe_cmp_branch);
16368 %}
16369
16370 instruct cmpP_imm0_branch(cmpOpEqNe cmp, iRegP op1, immP0 op2, label labl, rFlagsReg cr) %{
16371 match(If cmp (CmpP op1 op2));
16372 effect(USE labl);
16373
16374 ins_cost(BRANCH_COST);
16375 format %{ "cb$cmp $op1, $labl" %}
16376 ins_encode %{
16377 Label* L = $labl$$label;
16378 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
16379 if (cond == Assembler::EQ)
16380 __ cbz($op1$$Register, *L);
16381 else
16382 __ cbnz($op1$$Register, *L);
16383 %}
16384 ins_pipe(pipe_cmp_branch);
16385 %}
16386
16387 instruct cmpN_imm0_branch(cmpOpEqNe cmp, iRegN op1, immN0 op2, label labl, rFlagsReg cr) %{
16388 match(If cmp (CmpN op1 op2));
16389 effect(USE labl);
16390
16391 ins_cost(BRANCH_COST);
16392 format %{ "cbw$cmp $op1, $labl" %}
16393 ins_encode %{
16394 Label* L = $labl$$label;
16395 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
16396 if (cond == Assembler::EQ)
16397 __ cbzw($op1$$Register, *L);
16398 else
16399 __ cbnzw($op1$$Register, *L);
16400 %}
16401 ins_pipe(pipe_cmp_branch);
16402 %}
16403
16404 instruct cmpP_narrowOop_imm0_branch(cmpOpEqNe cmp, iRegN oop, immP0 zero, label labl, rFlagsReg cr) %{
16405 match(If cmp (CmpP (DecodeN oop) zero));
16406 effect(USE labl);
16407
16408 ins_cost(BRANCH_COST);
16409 format %{ "cb$cmp $oop, $labl" %}
16410 ins_encode %{
16411 Label* L = $labl$$label;
16412 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
16413 if (cond == Assembler::EQ)
16414 __ cbzw($oop$$Register, *L);
16415 else
16416 __ cbnzw($oop$$Register, *L);
16417 %}
16418 ins_pipe(pipe_cmp_branch);
16419 %}
16420
16421 instruct cmpUI_imm0_branch(cmpOpUEqNeLtGe cmp, iRegIorL2I op1, immI0 op2, label labl, rFlagsRegU cr) %{
16422 match(If cmp (CmpU op1 op2));
16423 effect(USE labl);
16424
16425 ins_cost(BRANCH_COST);
16426 format %{ "cbw$cmp $op1, $labl" %}
16427 ins_encode %{
16428 Label* L = $labl$$label;
16429 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
16430 if (cond == Assembler::EQ || cond == Assembler::LS)
16431 __ cbzw($op1$$Register, *L);
16432 else
16433 __ cbnzw($op1$$Register, *L);
16434 %}
16435 ins_pipe(pipe_cmp_branch);
16436 %}
16437
16438 instruct cmpUL_imm0_branch(cmpOpUEqNeLtGe cmp, iRegL op1, immL0 op2, label labl, rFlagsRegU cr) %{
16439 match(If cmp (CmpUL op1 op2));
16440 effect(USE labl);
16441
16442 ins_cost(BRANCH_COST);
16443 format %{ "cb$cmp $op1, $labl" %}
16444 ins_encode %{
16445 Label* L = $labl$$label;
16446 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
16447 if (cond == Assembler::EQ || cond == Assembler::LS)
16448 __ cbz($op1$$Register, *L);
16449 else
16450 __ cbnz($op1$$Register, *L);
16451 %}
16452 ins_pipe(pipe_cmp_branch);
16453 %}
16454
16455 // Test bit and Branch
16456
16457 // Patterns for short (< 32KiB) variants
16458 instruct cmpL_branch_sign(cmpOpLtGe cmp, iRegL op1, immL0 op2, label labl) %{
16459 match(If cmp (CmpL op1 op2));
16460 effect(USE labl);
16461
16462 ins_cost(BRANCH_COST);
16463 format %{ "cb$cmp $op1, $labl # long" %}
16464 ins_encode %{
16465 Label* L = $labl$$label;
16466 Assembler::Condition cond =
16467 ((Assembler::Condition)$cmp$$cmpcode == Assembler::LT) ? Assembler::NE : Assembler::EQ;
16468 __ tbr(cond, $op1$$Register, 63, *L);
16469 %}
16470 ins_pipe(pipe_cmp_branch);
16471 ins_short_branch(1);
16472 %}
16473
16474 instruct cmpI_branch_sign(cmpOpLtGe cmp, iRegIorL2I op1, immI0 op2, label labl) %{
16475 match(If cmp (CmpI op1 op2));
16476 effect(USE labl);
16477
16478 ins_cost(BRANCH_COST);
16479 format %{ "cb$cmp $op1, $labl # int" %}
16480 ins_encode %{
16481 Label* L = $labl$$label;
16482 Assembler::Condition cond =
16483 ((Assembler::Condition)$cmp$$cmpcode == Assembler::LT) ? Assembler::NE : Assembler::EQ;
16484 __ tbr(cond, $op1$$Register, 31, *L);
16485 %}
16486 ins_pipe(pipe_cmp_branch);
16487 ins_short_branch(1);
16488 %}
16489
16490 instruct cmpL_branch_bit(cmpOpEqNe cmp, iRegL op1, immL op2, immL0 op3, label labl) %{
16491 match(If cmp (CmpL (AndL op1 op2) op3));
16492 predicate(is_power_of_2((julong)n->in(2)->in(1)->in(2)->get_long()));
16493 effect(USE labl);
16494
16495 ins_cost(BRANCH_COST);
16496 format %{ "tb$cmp $op1, $op2, $labl" %}
16497 ins_encode %{
16498 Label* L = $labl$$label;
16499 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
16500 int bit = exact_log2_long($op2$$constant);
16501 __ tbr(cond, $op1$$Register, bit, *L);
16502 %}
16503 ins_pipe(pipe_cmp_branch);
16504 ins_short_branch(1);
16505 %}
16506
16507 instruct cmpI_branch_bit(cmpOpEqNe cmp, iRegIorL2I op1, immI op2, immI0 op3, label labl) %{
16508 match(If cmp (CmpI (AndI op1 op2) op3));
16509 predicate(is_power_of_2((juint)n->in(2)->in(1)->in(2)->get_int()));
16510 effect(USE labl);
16511
16512 ins_cost(BRANCH_COST);
16513 format %{ "tb$cmp $op1, $op2, $labl" %}
16514 ins_encode %{
16515 Label* L = $labl$$label;
16516 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
16517 int bit = exact_log2((juint)$op2$$constant);
16518 __ tbr(cond, $op1$$Register, bit, *L);
16519 %}
16520 ins_pipe(pipe_cmp_branch);
16521 ins_short_branch(1);
16522 %}
16523
16524 // And far variants
16525 instruct far_cmpL_branch_sign(cmpOpLtGe cmp, iRegL op1, immL0 op2, label labl) %{
16526 match(If cmp (CmpL op1 op2));
16527 effect(USE labl);
16528
16529 ins_cost(BRANCH_COST);
16530 format %{ "cb$cmp $op1, $labl # long" %}
16531 ins_encode %{
16532 Label* L = $labl$$label;
16533 Assembler::Condition cond =
16534 ((Assembler::Condition)$cmp$$cmpcode == Assembler::LT) ? Assembler::NE : Assembler::EQ;
16535 __ tbr(cond, $op1$$Register, 63, *L, /*far*/true);
16536 %}
16537 ins_pipe(pipe_cmp_branch);
16538 %}
16539
16540 instruct far_cmpI_branch_sign(cmpOpLtGe cmp, iRegIorL2I op1, immI0 op2, label labl) %{
16541 match(If cmp (CmpI op1 op2));
16542 effect(USE labl);
16543
16544 ins_cost(BRANCH_COST);
16545 format %{ "cb$cmp $op1, $labl # int" %}
16546 ins_encode %{
16547 Label* L = $labl$$label;
16548 Assembler::Condition cond =
16549 ((Assembler::Condition)$cmp$$cmpcode == Assembler::LT) ? Assembler::NE : Assembler::EQ;
16550 __ tbr(cond, $op1$$Register, 31, *L, /*far*/true);
16551 %}
16552 ins_pipe(pipe_cmp_branch);
16553 %}
16554
16555 instruct far_cmpL_branch_bit(cmpOpEqNe cmp, iRegL op1, immL op2, immL0 op3, label labl) %{
16556 match(If cmp (CmpL (AndL op1 op2) op3));
16557 predicate(is_power_of_2((julong)n->in(2)->in(1)->in(2)->get_long()));
16558 effect(USE labl);
16559
16560 ins_cost(BRANCH_COST);
16561 format %{ "tb$cmp $op1, $op2, $labl" %}
16562 ins_encode %{
16563 Label* L = $labl$$label;
16564 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
16565 int bit = exact_log2_long($op2$$constant);
16566 __ tbr(cond, $op1$$Register, bit, *L, /*far*/true);
16567 %}
16568 ins_pipe(pipe_cmp_branch);
16569 %}
16570
16571 instruct far_cmpI_branch_bit(cmpOpEqNe cmp, iRegIorL2I op1, immI op2, immI0 op3, label labl) %{
16572 match(If cmp (CmpI (AndI op1 op2) op3));
16573 predicate(is_power_of_2((juint)n->in(2)->in(1)->in(2)->get_int()));
16574 effect(USE labl);
16575
16576 ins_cost(BRANCH_COST);
16577 format %{ "tb$cmp $op1, $op2, $labl" %}
16578 ins_encode %{
16579 Label* L = $labl$$label;
16580 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
16581 int bit = exact_log2((juint)$op2$$constant);
16582 __ tbr(cond, $op1$$Register, bit, *L, /*far*/true);
16583 %}
16584 ins_pipe(pipe_cmp_branch);
16585 %}
16586
16587 // Test bits
16588
16589 instruct cmpL_and(cmpOp cmp, iRegL op1, immL op2, immL0 op3, rFlagsReg cr) %{
16590 match(Set cr (CmpL (AndL op1 op2) op3));
16591 predicate(Assembler::operand_valid_for_logical_immediate
16592 (/*is_32*/false, n->in(1)->in(2)->get_long()));
16593
16594 ins_cost(INSN_COST);
16595 format %{ "tst $op1, $op2 # long" %}
16596 ins_encode %{
16597 __ tst($op1$$Register, $op2$$constant);
16598 %}
16599 ins_pipe(ialu_reg_reg);
16600 %}
16601
16602 instruct cmpI_and(cmpOp cmp, iRegIorL2I op1, immI op2, immI0 op3, rFlagsReg cr) %{
16603 match(Set cr (CmpI (AndI op1 op2) op3));
16604 predicate(Assembler::operand_valid_for_logical_immediate
16605 (/*is_32*/true, n->in(1)->in(2)->get_int()));
16606
16607 ins_cost(INSN_COST);
16608 format %{ "tst $op1, $op2 # int" %}
16609 ins_encode %{
16610 __ tstw($op1$$Register, $op2$$constant);
16611 %}
16612 ins_pipe(ialu_reg_reg);
16613 %}
16614
16615 instruct cmpL_and_reg(cmpOp cmp, iRegL op1, iRegL op2, immL0 op3, rFlagsReg cr) %{
16616 match(Set cr (CmpL (AndL op1 op2) op3));
16617
16618 ins_cost(INSN_COST);
16619 format %{ "tst $op1, $op2 # long" %}
16620 ins_encode %{
16621 __ tst($op1$$Register, $op2$$Register);
16622 %}
16623 ins_pipe(ialu_reg_reg);
16624 %}
16625
16626 instruct cmpI_and_reg(cmpOp cmp, iRegIorL2I op1, iRegIorL2I op2, immI0 op3, rFlagsReg cr) %{
16627 match(Set cr (CmpI (AndI op1 op2) op3));
16628
16629 ins_cost(INSN_COST);
16630 format %{ "tstw $op1, $op2 # int" %}
16631 ins_encode %{
16632 __ tstw($op1$$Register, $op2$$Register);
16633 %}
16634 ins_pipe(ialu_reg_reg);
16635 %}
16636
16637
16638 // Conditional Far Branch
16639 // Conditional Far Branch Unsigned
16640 // TODO: fixme
16641
16642 // counted loop end branch near
16643 instruct branchLoopEnd(cmpOp cmp, rFlagsReg cr, label lbl)
16644 %{
16645 match(CountedLoopEnd cmp cr);
16646
16647 effect(USE lbl);
16648
16649 ins_cost(BRANCH_COST);
16650 // short variant.
16651 // ins_short_branch(1);
16652 format %{ "b$cmp $lbl \t// counted loop end" %}
16653
16654 ins_encode(aarch64_enc_br_con(cmp, lbl));
16655
16656 ins_pipe(pipe_branch);
16657 %}
16658
16659 // counted loop end branch near Unsigned
16660 instruct branchLoopEndU(cmpOpU cmp, rFlagsRegU cr, label lbl)
16661 %{
16662 match(CountedLoopEnd cmp cr);
16663
16664 effect(USE lbl);
16665
16666 ins_cost(BRANCH_COST);
16667 // short variant.
16668 // ins_short_branch(1);
16669 format %{ "b$cmp $lbl \t// counted loop end unsigned" %}
16670
16671 ins_encode(aarch64_enc_br_conU(cmp, lbl));
16672
16673 ins_pipe(pipe_branch);
16674 %}
16675
16676 // counted loop end branch far
16677 // counted loop end branch far unsigned
16678 // TODO: fixme
16679
16680 // ============================================================================
16681 // inlined locking and unlocking
16682
16683 instruct cmpFastLock(rFlagsReg cr, iRegP object, iRegP box, iRegPNoSp tmp, iRegPNoSp tmp2)
16684 %{
16685 match(Set cr (FastLock object box));
16686 effect(TEMP tmp, TEMP tmp2);
16687
16688 // TODO
16689 // identify correct cost
16690 ins_cost(5 * INSN_COST);
16691 format %{ "fastlock $object,$box\t! kills $tmp,$tmp2" %}
16692
16693 ins_encode(aarch64_enc_fast_lock(object, box, tmp, tmp2));
16694
16695 ins_pipe(pipe_serial);
16696 %}
16697
16698 instruct cmpFastUnlock(rFlagsReg cr, iRegP object, iRegP box, iRegPNoSp tmp, iRegPNoSp tmp2)
16699 %{
16700 match(Set cr (FastUnlock object box));
16701 effect(TEMP tmp, TEMP tmp2);
16702
16703 ins_cost(5 * INSN_COST);
16704 format %{ "fastunlock $object,$box\t! kills $tmp, $tmp2" %}
16705
16706 ins_encode(aarch64_enc_fast_unlock(object, box, tmp, tmp2));
16707
16708 ins_pipe(pipe_serial);
16709 %}
16710
16711
16712 // ============================================================================
16713 // Safepoint Instructions
16714
16715 // TODO
16716 // provide a near and far version of this code
16717
16718 instruct safePoint(rFlagsReg cr, iRegP poll)
16719 %{
16720 match(SafePoint poll);
16721 effect(KILL cr);
16722
16723 format %{
16724 "ldrw zr, [$poll]\t# Safepoint: poll for GC"
16725 %}
16726 ins_encode %{
16727 __ read_polling_page(as_Register($poll$$reg), relocInfo::poll_type);
16728 %}
16729 ins_pipe(pipe_serial); // ins_pipe(iload_reg_mem);
16730 %}
16731
16732
16733 // ============================================================================
16734 // Procedure Call/Return Instructions
16735
16736 // Call Java Static Instruction
16737
16738 instruct CallStaticJavaDirect(method meth)
16739 %{
16740 match(CallStaticJava);
16741
16742 effect(USE meth);
16743
16744 ins_cost(CALL_COST);
16745
16746 format %{ "call,static $meth \t// ==> " %}
16747
16748 ins_encode(aarch64_enc_java_static_call(meth),
16749 aarch64_enc_call_epilog);
16750
16751 ins_pipe(pipe_class_call);
16752 %}
16753
16754 // TO HERE
16755
16756 // Call Java Dynamic Instruction
16757 instruct CallDynamicJavaDirect(method meth)
16758 %{
16759 match(CallDynamicJava);
16760
16761 effect(USE meth);
16762
16763 ins_cost(CALL_COST);
16764
16765 format %{ "CALL,dynamic $meth \t// ==> " %}
16766
16767 ins_encode(aarch64_enc_java_dynamic_call(meth),
16768 aarch64_enc_call_epilog);
16769
16770 ins_pipe(pipe_class_call);
16771 %}
16772
16773 // Call Runtime Instruction
16774
16775 instruct CallRuntimeDirect(method meth)
16776 %{
16777 match(CallRuntime);
16778
16779 effect(USE meth);
16780
16781 ins_cost(CALL_COST);
16782
16783 format %{ "CALL, runtime $meth" %}
16784
16785 ins_encode( aarch64_enc_java_to_runtime(meth) );
16786
16787 ins_pipe(pipe_class_call);
16788 %}
16789
16790 // Call Runtime Instruction
16791
16792 instruct CallLeafDirect(method meth)
16793 %{
16794 match(CallLeaf);
16795
16796 effect(USE meth);
16797
16798 ins_cost(CALL_COST);
16799
16800 format %{ "CALL, runtime leaf $meth" %}
16801
16802 ins_encode( aarch64_enc_java_to_runtime(meth) );
16803
16804 ins_pipe(pipe_class_call);
16805 %}
16806
16807 // Call Runtime Instruction
16808
16809 instruct CallLeafNoFPDirect(method meth)
16810 %{
16811 match(CallLeafNoFP);
16812
16813 effect(USE meth);
16814
16815 ins_cost(CALL_COST);
16816
16817 format %{ "CALL, runtime leaf nofp $meth" %}
16818
16819 ins_encode( aarch64_enc_java_to_runtime(meth) );
16820
16821 ins_pipe(pipe_class_call);
16822 %}
16823
16824 instruct CallNativeDirect(method meth)
16825 %{
16826 match(CallNative);
16827
16828 effect(USE meth);
16829
16830 ins_cost(CALL_COST);
16831
16832 format %{ "CALL, native $meth" %}
16833
16834 ins_encode( aarch64_enc_java_to_runtime(meth) );
16835
16836 ins_pipe(pipe_class_call);
16837 %}
16838
16839 // Tail Call; Jump from runtime stub to Java code.
16840 // Also known as an 'interprocedural jump'.
16841 // Target of jump will eventually return to caller.
16842 // TailJump below removes the return address.
16843 instruct TailCalljmpInd(iRegPNoSp jump_target, inline_cache_RegP method_ptr)
16844 %{
16845 match(TailCall jump_target method_ptr);
16846
16847 ins_cost(CALL_COST);
16848
16849 format %{ "br $jump_target\t# $method_ptr holds method" %}
16850
16851 ins_encode(aarch64_enc_tail_call(jump_target));
16852
16853 ins_pipe(pipe_class_call);
16854 %}
16855
16856 instruct TailjmpInd(iRegPNoSp jump_target, iRegP_R0 ex_oop)
16857 %{
16858 match(TailJump jump_target ex_oop);
16859
16860 ins_cost(CALL_COST);
16861
16862 format %{ "br $jump_target\t# $ex_oop holds exception oop" %}
16863
16864 ins_encode(aarch64_enc_tail_jmp(jump_target));
16865
16866 ins_pipe(pipe_class_call);
16867 %}
16868
16869 // Create exception oop: created by stack-crawling runtime code.
16870 // Created exception is now available to this handler, and is setup
16871 // just prior to jumping to this handler. No code emitted.
16872 // TODO check
16873 // should ex_oop be in r0? intel uses rax, ppc cannot use r0 so uses rarg1
16874 instruct CreateException(iRegP_R0 ex_oop)
16875 %{
16876 match(Set ex_oop (CreateEx));
16877
16878 format %{ " -- \t// exception oop; no code emitted" %}
16879
16880 size(0);
16881
16882 ins_encode( /*empty*/ );
16883
16884 ins_pipe(pipe_class_empty);
16885 %}
16886
16887 // Rethrow exception: The exception oop will come in the first
16888 // argument position. Then JUMP (not call) to the rethrow stub code.
16889 instruct RethrowException() %{
16890 match(Rethrow);
16891 ins_cost(CALL_COST);
16892
16893 format %{ "b rethrow_stub" %}
16894
16895 ins_encode( aarch64_enc_rethrow() );
16896
16897 ins_pipe(pipe_class_call);
16898 %}
16899
16900
16901 // Return Instruction
16902 // epilog node loads ret address into lr as part of frame pop
16903 instruct Ret()
16904 %{
16905 match(Return);
16906
16907 format %{ "ret\t// return register" %}
16908
16909 ins_encode( aarch64_enc_ret() );
16910
16911 ins_pipe(pipe_branch);
16912 %}
16913
16914 // Die now.
16915 instruct ShouldNotReachHere() %{
16916 match(Halt);
16917
16918 ins_cost(CALL_COST);
16919 format %{ "ShouldNotReachHere" %}
16920
16921 ins_encode %{
16922 if (is_reachable()) {
16923 __ stop(_halt_reason);
16924 }
16925 %}
16926
16927 ins_pipe(pipe_class_default);
16928 %}
16929
16930 // ============================================================================
16931 // Partial Subtype Check
16932 //
16933 // superklass array for an instance of the superklass. Set a hidden
16934 // internal cache on a hit (cache is checked with exposed code in
16935 // gen_subtype_check()). Return NZ for a miss or zero for a hit. The
16936 // encoding ALSO sets flags.
16937
16938 instruct partialSubtypeCheck(iRegP_R4 sub, iRegP_R0 super, iRegP_R2 temp, iRegP_R5 result, rFlagsReg cr)
16939 %{
16940 match(Set result (PartialSubtypeCheck sub super));
16941 effect(KILL cr, KILL temp);
16942
16943 ins_cost(1100); // slightly larger than the next version
16944 format %{ "partialSubtypeCheck $result, $sub, $super" %}
16945
16946 ins_encode(aarch64_enc_partial_subtype_check(sub, super, temp, result));
16947
16948 opcode(0x1); // Force zero of result reg on hit
16949
16950 ins_pipe(pipe_class_memory);
16951 %}
16952
16953 instruct partialSubtypeCheckVsZero(iRegP_R4 sub, iRegP_R0 super, iRegP_R2 temp, iRegP_R5 result, immP0 zero, rFlagsReg cr)
16954 %{
16955 match(Set cr (CmpP (PartialSubtypeCheck sub super) zero));
16956 effect(KILL temp, KILL result);
16957
16958 ins_cost(1100); // slightly larger than the next version
16959 format %{ "partialSubtypeCheck $result, $sub, $super == 0" %}
16960
16961 ins_encode(aarch64_enc_partial_subtype_check(sub, super, temp, result));
16962
16963 opcode(0x0); // Don't zero result reg on hit
16964
16965 ins_pipe(pipe_class_memory);
16966 %}
16967
16968 instruct string_compareU(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
16969 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2, rFlagsReg cr)
16970 %{
16971 predicate(((StrCompNode*)n)->encoding() == StrIntrinsicNode::UU);
16972 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
16973 effect(KILL tmp1, KILL tmp2, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
16974
16975 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # KILL $tmp1" %}
16976 ins_encode %{
16977 // Count is in 8-bit bytes; non-Compact chars are 16 bits.
16978 __ string_compare($str1$$Register, $str2$$Register,
16979 $cnt1$$Register, $cnt2$$Register, $result$$Register,
16980 $tmp1$$Register, $tmp2$$Register,
16981 fnoreg, fnoreg, fnoreg, StrIntrinsicNode::UU);
16982 %}
16983 ins_pipe(pipe_class_memory);
16984 %}
16985
16986 instruct string_compareL(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
16987 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2, rFlagsReg cr)
16988 %{
16989 predicate(((StrCompNode*)n)->encoding() == StrIntrinsicNode::LL);
16990 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
16991 effect(KILL tmp1, KILL tmp2, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
16992
16993 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # KILL $tmp1" %}
16994 ins_encode %{
16995 __ string_compare($str1$$Register, $str2$$Register,
16996 $cnt1$$Register, $cnt2$$Register, $result$$Register,
16997 $tmp1$$Register, $tmp2$$Register,
16998 fnoreg, fnoreg, fnoreg, StrIntrinsicNode::LL);
16999 %}
17000 ins_pipe(pipe_class_memory);
17001 %}
17002
17003 instruct string_compareUL(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
17004 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
17005 vRegD_V0 vtmp1, vRegD_V1 vtmp2, vRegD_V2 vtmp3, rFlagsReg cr)
17006 %{
17007 predicate(((StrCompNode*)n)->encoding() == StrIntrinsicNode::UL);
17008 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
17009 effect(KILL tmp1, KILL tmp2, KILL vtmp1, KILL vtmp2, KILL vtmp3,
17010 USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
17011
17012 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # KILL $tmp1, $tmp2, $vtmp1, $vtmp2, $vtmp3" %}
17013 ins_encode %{
17014 __ string_compare($str1$$Register, $str2$$Register,
17015 $cnt1$$Register, $cnt2$$Register, $result$$Register,
17016 $tmp1$$Register, $tmp2$$Register,
17017 $vtmp1$$FloatRegister, $vtmp2$$FloatRegister,
17018 $vtmp3$$FloatRegister, StrIntrinsicNode::UL);
17019 %}
17020 ins_pipe(pipe_class_memory);
17021 %}
17022
17023 instruct string_compareLU(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
17024 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
17025 vRegD_V0 vtmp1, vRegD_V1 vtmp2, vRegD_V2 vtmp3, rFlagsReg cr)
17026 %{
17027 predicate(((StrCompNode*)n)->encoding() == StrIntrinsicNode::LU);
17028 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
17029 effect(KILL tmp1, KILL tmp2, KILL vtmp1, KILL vtmp2, KILL vtmp3,
17030 USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
17031
17032 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # KILL $tmp1, $tmp2, $vtmp1, $vtmp2, $vtmp3" %}
17033 ins_encode %{
17034 __ string_compare($str1$$Register, $str2$$Register,
17035 $cnt1$$Register, $cnt2$$Register, $result$$Register,
17036 $tmp1$$Register, $tmp2$$Register,
17037 $vtmp1$$FloatRegister, $vtmp2$$FloatRegister,
17038 $vtmp3$$FloatRegister,StrIntrinsicNode::LU);
17039 %}
17040 ins_pipe(pipe_class_memory);
17041 %}
17042
17043 instruct string_indexofUU(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2, iRegI_R2 cnt2,
17044 iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2,
17045 iRegINoSp tmp3, iRegINoSp tmp4, iRegINoSp tmp5, iRegINoSp tmp6,
17046 vRegD_V0 vtmp0, vRegD_V1 vtmp1, rFlagsReg cr)
17047 %{
17048 predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UU);
17049 match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 cnt2)));
17050 effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2,
17051 TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, TEMP tmp5, TEMP tmp6,
17052 TEMP vtmp0, TEMP vtmp1, KILL cr);
17053 format %{ "String IndexOf $str1,$cnt1,$str2,$cnt2 -> $result (UU) "
17054 "# KILL $str1 $cnt1 $str2 $cnt2 $tmp1 $tmp2 $tmp3 $tmp4 $tmp5 $tmp6 V0-V1 cr" %}
17055
17056 ins_encode %{
17057 __ string_indexof($str1$$Register, $str2$$Register,
17058 $cnt1$$Register, $cnt2$$Register,
17059 $tmp1$$Register, $tmp2$$Register,
17060 $tmp3$$Register, $tmp4$$Register,
17061 $tmp5$$Register, $tmp6$$Register,
17062 -1, $result$$Register, StrIntrinsicNode::UU);
17063 %}
17064 ins_pipe(pipe_class_memory);
17065 %}
17066
17067 instruct string_indexofLL(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2, iRegI_R2 cnt2,
17068 iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2, iRegINoSp tmp3,
17069 iRegINoSp tmp4, iRegINoSp tmp5, iRegINoSp tmp6,
17070 vRegD_V0 vtmp0, vRegD_V1 vtmp1, rFlagsReg cr)
17071 %{
17072 predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::LL);
17073 match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 cnt2)));
17074 effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2,
17075 TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, TEMP tmp5, TEMP tmp6,
17076 TEMP vtmp0, TEMP vtmp1, KILL cr);
17077 format %{ "String IndexOf $str1,$cnt1,$str2,$cnt2 -> $result (LL) "
17078 "# KILL $str1 $cnt1 $str2 $cnt2 $tmp1 $tmp2 $tmp3 $tmp4 $tmp5 $tmp6 V0-V1 cr" %}
17079
17080 ins_encode %{
17081 __ string_indexof($str1$$Register, $str2$$Register,
17082 $cnt1$$Register, $cnt2$$Register,
17083 $tmp1$$Register, $tmp2$$Register,
17084 $tmp3$$Register, $tmp4$$Register,
17085 $tmp5$$Register, $tmp6$$Register,
17086 -1, $result$$Register, StrIntrinsicNode::LL);
17087 %}
17088 ins_pipe(pipe_class_memory);
17089 %}
17090
17091 instruct string_indexofUL(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2, iRegI_R2 cnt2,
17092 iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2,iRegINoSp tmp3,
17093 iRegINoSp tmp4, iRegINoSp tmp5, iRegINoSp tmp6,
17094 vRegD_V0 vtmp0, vRegD_V1 vtmp1, rFlagsReg cr)
17095 %{
17096 predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UL);
17097 match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 cnt2)));
17098 effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2,
17099 TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, TEMP tmp5,
17100 TEMP tmp6, TEMP vtmp0, TEMP vtmp1, KILL cr);
17101 format %{ "String IndexOf $str1,$cnt1,$str2,$cnt2 -> $result (UL) "
17102 "# KILL $str1 cnt1 $str2 $cnt2 $tmp1 $tmp2 $tmp3 $tmp4 $tmp5 $tmp6 V0-V1 cr" %}
17103
17104 ins_encode %{
17105 __ string_indexof($str1$$Register, $str2$$Register,
17106 $cnt1$$Register, $cnt2$$Register,
17107 $tmp1$$Register, $tmp2$$Register,
17108 $tmp3$$Register, $tmp4$$Register,
17109 $tmp5$$Register, $tmp6$$Register,
17110 -1, $result$$Register, StrIntrinsicNode::UL);
17111 %}
17112 ins_pipe(pipe_class_memory);
17113 %}
17114
17115 instruct string_indexof_conUU(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2,
17116 immI_le_4 int_cnt2, iRegI_R0 result, iRegINoSp tmp1,
17117 iRegINoSp tmp2, iRegINoSp tmp3, iRegINoSp tmp4, rFlagsReg cr)
17118 %{
17119 predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UU);
17120 match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 int_cnt2)));
17121 effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1,
17122 TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, KILL cr);
17123 format %{ "String IndexOf $str1,$cnt1,$str2,$int_cnt2 -> $result (UU) "
17124 "# KILL $str1 $cnt1 $str2 $tmp1 $tmp2 $tmp3 $tmp4 cr" %}
17125
17126 ins_encode %{
17127 int icnt2 = (int)$int_cnt2$$constant;
17128 __ string_indexof($str1$$Register, $str2$$Register,
17129 $cnt1$$Register, zr,
17130 $tmp1$$Register, $tmp2$$Register,
17131 $tmp3$$Register, $tmp4$$Register, zr, zr,
17132 icnt2, $result$$Register, StrIntrinsicNode::UU);
17133 %}
17134 ins_pipe(pipe_class_memory);
17135 %}
17136
17137 instruct string_indexof_conLL(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2,
17138 immI_le_4 int_cnt2, iRegI_R0 result, iRegINoSp tmp1,
17139 iRegINoSp tmp2, iRegINoSp tmp3, iRegINoSp tmp4, rFlagsReg cr)
17140 %{
17141 predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::LL);
17142 match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 int_cnt2)));
17143 effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1,
17144 TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, KILL cr);
17145 format %{ "String IndexOf $str1,$cnt1,$str2,$int_cnt2 -> $result (LL) "
17146 "# KILL $str1 $cnt1 $str2 $tmp1 $tmp2 $tmp3 $tmp4 cr" %}
17147
17148 ins_encode %{
17149 int icnt2 = (int)$int_cnt2$$constant;
17150 __ string_indexof($str1$$Register, $str2$$Register,
17151 $cnt1$$Register, zr,
17152 $tmp1$$Register, $tmp2$$Register,
17153 $tmp3$$Register, $tmp4$$Register, zr, zr,
17154 icnt2, $result$$Register, StrIntrinsicNode::LL);
17155 %}
17156 ins_pipe(pipe_class_memory);
17157 %}
17158
17159 instruct string_indexof_conUL(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2,
17160 immI_1 int_cnt2, iRegI_R0 result, iRegINoSp tmp1,
17161 iRegINoSp tmp2, iRegINoSp tmp3, iRegINoSp tmp4, rFlagsReg cr)
17162 %{
17163 predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UL);
17164 match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 int_cnt2)));
17165 effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1,
17166 TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, KILL cr);
17167 format %{ "String IndexOf $str1,$cnt1,$str2,$int_cnt2 -> $result (UL) "
17168 "# KILL $str1 $cnt1 $str2 $tmp1 $tmp2 $tmp3 $tmp4 cr" %}
17169
17170 ins_encode %{
17171 int icnt2 = (int)$int_cnt2$$constant;
17172 __ string_indexof($str1$$Register, $str2$$Register,
17173 $cnt1$$Register, zr,
17174 $tmp1$$Register, $tmp2$$Register,
17175 $tmp3$$Register, $tmp4$$Register, zr, zr,
17176 icnt2, $result$$Register, StrIntrinsicNode::UL);
17177 %}
17178 ins_pipe(pipe_class_memory);
17179 %}
17180
17181 instruct string_indexof_char(iRegP_R1 str1, iRegI_R2 cnt1, iRegI_R3 ch,
17182 iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2,
17183 iRegINoSp tmp3, rFlagsReg cr)
17184 %{
17185 match(Set result (StrIndexOfChar (Binary str1 cnt1) ch));
17186 predicate(((StrIndexOfCharNode*)n)->encoding() == StrIntrinsicNode::U);
17187 effect(USE_KILL str1, USE_KILL cnt1, USE_KILL ch,
17188 TEMP tmp1, TEMP tmp2, TEMP tmp3, KILL cr);
17189
17190 format %{ "StringUTF16 IndexOf char[] $str1,$cnt1,$ch -> $result" %}
17191
17192 ins_encode %{
17193 __ string_indexof_char($str1$$Register, $cnt1$$Register, $ch$$Register,
17194 $result$$Register, $tmp1$$Register, $tmp2$$Register,
17195 $tmp3$$Register);
17196 %}
17197 ins_pipe(pipe_class_memory);
17198 %}
17199
17200 instruct stringL_indexof_char(iRegP_R1 str1, iRegI_R2 cnt1, iRegI_R3 ch,
17201 iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2,
17202 iRegINoSp tmp3, rFlagsReg cr)
17203 %{
17204 match(Set result (StrIndexOfChar (Binary str1 cnt1) ch));
17205 predicate(((StrIndexOfCharNode*)n)->encoding() == StrIntrinsicNode::L);
17206 effect(USE_KILL str1, USE_KILL cnt1, USE_KILL ch,
17207 TEMP tmp1, TEMP tmp2, TEMP tmp3, KILL cr);
17208
17209 format %{ "StringLatin1 IndexOf char[] $str1,$cnt1,$ch -> $result" %}
17210
17211 ins_encode %{
17212 __ stringL_indexof_char($str1$$Register, $cnt1$$Register, $ch$$Register,
17213 $result$$Register, $tmp1$$Register, $tmp2$$Register,
17214 $tmp3$$Register);
17215 %}
17216 ins_pipe(pipe_class_memory);
17217 %}
17218
17219 instruct string_equalsL(iRegP_R1 str1, iRegP_R3 str2, iRegI_R4 cnt,
17220 iRegI_R0 result, rFlagsReg cr)
17221 %{
17222 predicate(((StrEqualsNode*)n)->encoding() == StrIntrinsicNode::LL);
17223 match(Set result (StrEquals (Binary str1 str2) cnt));
17224 effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt, KILL cr);
17225
17226 format %{ "String Equals $str1,$str2,$cnt -> $result" %}
17227 ins_encode %{
17228 // Count is in 8-bit bytes; non-Compact chars are 16 bits.
17229 __ string_equals($str1$$Register, $str2$$Register,
17230 $result$$Register, $cnt$$Register, 1);
17231 %}
17232 ins_pipe(pipe_class_memory);
17233 %}
17234
17235 instruct string_equalsU(iRegP_R1 str1, iRegP_R3 str2, iRegI_R4 cnt,
17236 iRegI_R0 result, rFlagsReg cr)
17237 %{
17238 predicate(((StrEqualsNode*)n)->encoding() == StrIntrinsicNode::UU);
17239 match(Set result (StrEquals (Binary str1 str2) cnt));
17240 effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt, KILL cr);
17241
17242 format %{ "String Equals $str1,$str2,$cnt -> $result" %}
17243 ins_encode %{
17244 // Count is in 8-bit bytes; non-Compact chars are 16 bits.
17245 __ string_equals($str1$$Register, $str2$$Register,
17246 $result$$Register, $cnt$$Register, 2);
17247 %}
17248 ins_pipe(pipe_class_memory);
17249 %}
17250
17251 instruct array_equalsB(iRegP_R1 ary1, iRegP_R2 ary2, iRegI_R0 result,
17252 iRegP_R3 tmp1, iRegP_R4 tmp2, iRegP_R5 tmp3,
17253 vRegD_V0 vtmp0, vRegD_V1 vtmp1, vRegD_V2 vtmp2, vRegD_V3 vtmp3,
17254 vRegD_V4 vtmp4, vRegD_V5 vtmp5, vRegD_V6 vtmp6, vRegD_V7 vtmp7,
17255 iRegP_R10 tmp, rFlagsReg cr)
17256 %{
17257 predicate(((AryEqNode*)n)->encoding() == StrIntrinsicNode::LL);
17258 match(Set result (AryEq ary1 ary2));
17259 effect(KILL tmp, USE_KILL ary1, USE_KILL ary2, TEMP tmp1, TEMP tmp2, TEMP tmp3,
17260 TEMP vtmp0, TEMP vtmp1, TEMP vtmp2, TEMP vtmp3, TEMP vtmp4, TEMP vtmp5,
17261 TEMP vtmp6, TEMP vtmp7, KILL cr);
17262
17263 format %{ "Array Equals $ary1,ary2 -> $result # KILL $ary1 $ary2 $tmp $tmp1 $tmp2 $tmp3 V0-V7 cr" %}
17264 ins_encode %{
17265 address tpc = __ arrays_equals($ary1$$Register, $ary2$$Register,
17266 $tmp1$$Register, $tmp2$$Register, $tmp3$$Register,
17267 $result$$Register, $tmp$$Register, 1);
17268 if (tpc == NULL) {
17269 ciEnv::current()->record_failure("CodeCache is full");
17270 return;
17271 }
17272 %}
17273 ins_pipe(pipe_class_memory);
17274 %}
17275
17276 instruct array_equalsC(iRegP_R1 ary1, iRegP_R2 ary2, iRegI_R0 result,
17277 iRegP_R3 tmp1, iRegP_R4 tmp2, iRegP_R5 tmp3,
17278 vRegD_V0 vtmp0, vRegD_V1 vtmp1, vRegD_V2 vtmp2, vRegD_V3 vtmp3,
17279 vRegD_V4 vtmp4, vRegD_V5 vtmp5, vRegD_V6 vtmp6, vRegD_V7 vtmp7,
17280 iRegP_R10 tmp, rFlagsReg cr)
17281 %{
17282 predicate(((AryEqNode*)n)->encoding() == StrIntrinsicNode::UU);
17283 match(Set result (AryEq ary1 ary2));
17284 effect(KILL tmp, USE_KILL ary1, USE_KILL ary2, TEMP tmp1, TEMP tmp2, TEMP tmp3,
17285 TEMP vtmp0, TEMP vtmp1, TEMP vtmp2, TEMP vtmp3, TEMP vtmp4, TEMP vtmp5,
17286 TEMP vtmp6, TEMP vtmp7, KILL cr);
17287
17288 format %{ "Array Equals $ary1,ary2 -> $result # KILL $ary1 $ary2 $tmp $tmp1 $tmp2 $tmp3 V0-V7 cr" %}
17289 ins_encode %{
17290 address tpc = __ arrays_equals($ary1$$Register, $ary2$$Register,
17291 $tmp1$$Register, $tmp2$$Register, $tmp3$$Register,
17292 $result$$Register, $tmp$$Register, 2);
17293 if (tpc == NULL) {
17294 ciEnv::current()->record_failure("CodeCache is full");
17295 return;
17296 }
17297 %}
17298 ins_pipe(pipe_class_memory);
17299 %}
17300
17301 instruct has_negatives(iRegP_R1 ary1, iRegI_R2 len, iRegI_R0 result, rFlagsReg cr)
17302 %{
17303 match(Set result (HasNegatives ary1 len));
17304 effect(USE_KILL ary1, USE_KILL len, KILL cr);
17305 format %{ "has negatives byte[] $ary1,$len -> $result" %}
17306 ins_encode %{
17307 address tpc = __ has_negatives($ary1$$Register, $len$$Register, $result$$Register);
17308 if (tpc == NULL) {
17309 ciEnv::current()->record_failure("CodeCache is full");
17310 return;
17311 }
17312 %}
17313 ins_pipe( pipe_slow );
17314 %}
17315
17316 // fast char[] to byte[] compression
17317 instruct string_compress(iRegP_R2 src, iRegP_R1 dst, iRegI_R3 len,
17318 vRegD_V0 vtmp0, vRegD_V1 vtmp1, vRegD_V2 vtmp2,
17319 vRegD_V3 vtmp3, vRegD_V4 vtmp4, vRegD_V5 vtmp5,
17320 iRegI_R0 result, rFlagsReg cr)
17321 %{
17322 match(Set result (StrCompressedCopy src (Binary dst len)));
17323 effect(TEMP vtmp0, TEMP vtmp1, TEMP vtmp2, TEMP vtmp3, TEMP vtmp4, TEMP vtmp5,
17324 USE_KILL src, USE_KILL dst, USE len, KILL cr);
17325
17326 format %{ "String Compress $src,$dst,$len -> $result # KILL $src $dst V0-V5 cr" %}
17327 ins_encode %{
17328 __ char_array_compress($src$$Register, $dst$$Register, $len$$Register,
17329 $result$$Register, $vtmp0$$FloatRegister, $vtmp1$$FloatRegister,
17330 $vtmp2$$FloatRegister, $vtmp3$$FloatRegister,
17331 $vtmp4$$FloatRegister, $vtmp5$$FloatRegister);
17332 %}
17333 ins_pipe(pipe_slow);
17334 %}
17335
17336 // fast byte[] to char[] inflation
17337 instruct string_inflate(Universe dummy, iRegP_R0 src, iRegP_R1 dst, iRegI_R2 len, iRegP_R3 tmp,
17338 vRegD_V0 vtmp0, vRegD_V1 vtmp1, vRegD_V2 vtmp2, vRegD_V3 vtmp3,
17339 vRegD_V4 vtmp4, vRegD_V5 vtmp5, vRegD_V6 vtmp6, rFlagsReg cr)
17340 %{
17341 match(Set dummy (StrInflatedCopy src (Binary dst len)));
17342 effect(TEMP vtmp0, TEMP vtmp1, TEMP vtmp2, TEMP vtmp3,
17343 TEMP vtmp4, TEMP vtmp5, TEMP vtmp6, TEMP tmp,
17344 USE_KILL src, USE_KILL dst, USE_KILL len, KILL cr);
17345
17346 format %{ "String Inflate $src,$dst # KILL $tmp $src $dst $len V0-V6 cr" %}
17347 ins_encode %{
17348 address tpc = __ byte_array_inflate($src$$Register, $dst$$Register, $len$$Register,
17349 $vtmp0$$FloatRegister, $vtmp1$$FloatRegister,
17350 $vtmp2$$FloatRegister, $tmp$$Register);
17351 if (tpc == NULL) {
17352 ciEnv::current()->record_failure("CodeCache is full");
17353 return;
17354 }
17355 %}
17356 ins_pipe(pipe_class_memory);
17357 %}
17358
17359 // encode char[] to byte[] in ISO_8859_1
17360 instruct encode_iso_array(iRegP_R2 src, iRegP_R1 dst, iRegI_R3 len,
17361 vRegD_V0 vtmp0, vRegD_V1 vtmp1, vRegD_V2 vtmp2,
17362 vRegD_V3 vtmp3, vRegD_V4 vtmp4, vRegD_V5 vtmp5,
17363 iRegI_R0 result, rFlagsReg cr)
17364 %{
17365 predicate(!((EncodeISOArrayNode*)n)->is_ascii());
17366 match(Set result (EncodeISOArray src (Binary dst len)));
17367 effect(USE_KILL src, USE_KILL dst, USE len, KILL vtmp0, KILL vtmp1,
17368 KILL vtmp2, KILL vtmp3, KILL vtmp4, KILL vtmp5, KILL cr);
17369
17370 format %{ "Encode ISO array $src,$dst,$len -> $result # KILL $src $dst V0-V5 cr" %}
17371 ins_encode %{
17372 __ encode_iso_array($src$$Register, $dst$$Register, $len$$Register,
17373 $result$$Register, false,
17374 $vtmp0$$FloatRegister, $vtmp1$$FloatRegister,
17375 $vtmp2$$FloatRegister, $vtmp3$$FloatRegister,
17376 $vtmp4$$FloatRegister, $vtmp5$$FloatRegister);
17377 %}
17378 ins_pipe(pipe_class_memory);
17379 %}
17380
17381 instruct encode_ascii_array(iRegP_R2 src, iRegP_R1 dst, iRegI_R3 len,
17382 vRegD_V0 vtmp0, vRegD_V1 vtmp1, vRegD_V2 vtmp2,
17383 vRegD_V3 vtmp3, vRegD_V4 vtmp4, vRegD_V5 vtmp5,
17384 iRegI_R0 result, rFlagsReg cr)
17385 %{
17386 predicate(((EncodeISOArrayNode*)n)->is_ascii());
17387 match(Set result (EncodeISOArray src (Binary dst len)));
17388 effect(USE_KILL src, USE_KILL dst, USE len, KILL vtmp0, KILL vtmp1,
17389 KILL vtmp2, KILL vtmp3, KILL vtmp4, KILL vtmp5, KILL cr);
17390
17391 format %{ "Encode ASCII array $src,$dst,$len -> $result # KILL $src $dst V0-V5 cr" %}
17392 ins_encode %{
17393 __ encode_iso_array($src$$Register, $dst$$Register, $len$$Register,
17394 $result$$Register, true,
17395 $vtmp0$$FloatRegister, $vtmp1$$FloatRegister,
17396 $vtmp2$$FloatRegister, $vtmp3$$FloatRegister,
17397 $vtmp4$$FloatRegister, $vtmp5$$FloatRegister);
17398 %}
17399 ins_pipe(pipe_class_memory);
17400 %}
17401
17402 // ============================================================================
17403 // This name is KNOWN by the ADLC and cannot be changed.
17404 // The ADLC forces a 'TypeRawPtr::BOTTOM' output type
17405 // for this guy.
17406 instruct tlsLoadP(thread_RegP dst)
17407 %{
17408 match(Set dst (ThreadLocal));
17409
17410 ins_cost(0);
17411
17412 format %{ " -- \t// $dst=Thread::current(), empty" %}
17413
17414 size(0);
17415
17416 ins_encode( /*empty*/ );
17417
17418 ins_pipe(pipe_class_empty);
17419 %}
17420
17421 //----------PEEPHOLE RULES-----------------------------------------------------
17422 // These must follow all instruction definitions as they use the names
17423 // defined in the instructions definitions.
17424 //
17425 // peepmatch ( root_instr_name [preceding_instruction]* );
17426 //
17427 // peepconstraint %{
17428 // (instruction_number.operand_name relational_op instruction_number.operand_name
17429 // [, ...] );
17430 // // instruction numbers are zero-based using left to right order in peepmatch
17431 //
17432 // peepreplace ( instr_name ( [instruction_number.operand_name]* ) );
17433 // // provide an instruction_number.operand_name for each operand that appears
17434 // // in the replacement instruction's match rule
17435 //
17436 // ---------VM FLAGS---------------------------------------------------------
17437 //
17438 // All peephole optimizations can be turned off using -XX:-OptoPeephole
17439 //
17440 // Each peephole rule is given an identifying number starting with zero and
17441 // increasing by one in the order seen by the parser. An individual peephole
17442 // can be enabled, and all others disabled, by using -XX:OptoPeepholeAt=#
17443 // on the command-line.
17444 //
17445 // ---------CURRENT LIMITATIONS----------------------------------------------
17446 //
17447 // Only match adjacent instructions in same basic block
17448 // Only equality constraints
17449 // Only constraints between operands, not (0.dest_reg == RAX_enc)
17450 // Only one replacement instruction
17451 //
17452 // ---------EXAMPLE----------------------------------------------------------
17453 //
17454 // // pertinent parts of existing instructions in architecture description
17455 // instruct movI(iRegINoSp dst, iRegI src)
17456 // %{
17457 // match(Set dst (CopyI src));
17458 // %}
17459 //
17460 // instruct incI_iReg(iRegINoSp dst, immI1 src, rFlagsReg cr)
17461 // %{
17462 // match(Set dst (AddI dst src));
17463 // effect(KILL cr);
17464 // %}
17465 //
17466 // // Change (inc mov) to lea
17467 // peephole %{
17468 // // increment preceeded by register-register move
17469 // peepmatch ( incI_iReg movI );
17470 // // require that the destination register of the increment
17471 // // match the destination register of the move
17472 // peepconstraint ( 0.dst == 1.dst );
17473 // // construct a replacement instruction that sets
17474 // // the destination to ( move's source register + one )
17475 // peepreplace ( leaI_iReg_immI( 0.dst 1.src 0.src ) );
17476 // %}
17477 //
17478
17479 // Implementation no longer uses movX instructions since
17480 // machine-independent system no longer uses CopyX nodes.
17481 //
17482 // peephole
17483 // %{
17484 // peepmatch (incI_iReg movI);
17485 // peepconstraint (0.dst == 1.dst);
17486 // peepreplace (leaI_iReg_immI(0.dst 1.src 0.src));
17487 // %}
17488
17489 // peephole
17490 // %{
17491 // peepmatch (decI_iReg movI);
17492 // peepconstraint (0.dst == 1.dst);
17493 // peepreplace (leaI_iReg_immI(0.dst 1.src 0.src));
17494 // %}
17495
17496 // peephole
17497 // %{
17498 // peepmatch (addI_iReg_imm movI);
17499 // peepconstraint (0.dst == 1.dst);
17500 // peepreplace (leaI_iReg_immI(0.dst 1.src 0.src));
17501 // %}
17502
17503 // peephole
17504 // %{
17505 // peepmatch (incL_iReg movL);
17506 // peepconstraint (0.dst == 1.dst);
17507 // peepreplace (leaL_iReg_immL(0.dst 1.src 0.src));
17508 // %}
17509
17510 // peephole
17511 // %{
17512 // peepmatch (decL_iReg movL);
17513 // peepconstraint (0.dst == 1.dst);
17514 // peepreplace (leaL_iReg_immL(0.dst 1.src 0.src));
17515 // %}
17516
17517 // peephole
17518 // %{
17519 // peepmatch (addL_iReg_imm movL);
17520 // peepconstraint (0.dst == 1.dst);
17521 // peepreplace (leaL_iReg_immL(0.dst 1.src 0.src));
17522 // %}
17523
17524 // peephole
17525 // %{
17526 // peepmatch (addP_iReg_imm movP);
17527 // peepconstraint (0.dst == 1.dst);
17528 // peepreplace (leaP_iReg_imm(0.dst 1.src 0.src));
17529 // %}
17530
17531 // // Change load of spilled value to only a spill
17532 // instruct storeI(memory mem, iRegI src)
17533 // %{
17534 // match(Set mem (StoreI mem src));
17535 // %}
17536 //
17537 // instruct loadI(iRegINoSp dst, memory mem)
17538 // %{
17539 // match(Set dst (LoadI mem));
17540 // %}
17541 //
17542
17543 //----------SMARTSPILL RULES---------------------------------------------------
17544 // These must follow all instruction definitions as they use the names
17545 // defined in the instructions definitions.
17546
17547 // Local Variables:
17548 // mode: c++
17549 // End: