1 //
2 // Copyright (c) 2003, 2021, Oracle and/or its affiliates. All rights reserved.
3 // Copyright (c) 2014, 2021, Red Hat, Inc. All rights reserved.
4 // DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
5 //
6 // This code is free software; you can redistribute it and/or modify it
7 // under the terms of the GNU General Public License version 2 only, as
8 // published by the Free Software Foundation.
9 //
10 // This code is distributed in the hope that it will be useful, but WITHOUT
11 // ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
12 // FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
13 // version 2 for more details (a copy is included in the LICENSE file that
14 // accompanied this code).
15 //
16 // You should have received a copy of the GNU General Public License version
17 // 2 along with this work; if not, write to the Free Software Foundation,
18 // Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
19 //
20 // Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
21 // or visit www.oracle.com if you need additional information or have any
22 // questions.
23 //
24 //
25
26 // AArch64 Architecture Description File
27
28 //----------REGISTER DEFINITION BLOCK------------------------------------------
29 // This information is used by the matcher and the register allocator to
30 // describe individual registers and classes of registers within the target
31 // archtecture.
32
33 register %{
34 //----------Architecture Description Register Definitions----------------------
35 // General Registers
36 // "reg_def" name ( register save type, C convention save type,
37 // ideal register type, encoding );
38 // Register Save Types:
39 //
40 // NS = No-Save: The register allocator assumes that these registers
41 // can be used without saving upon entry to the method, &
42 // that they do not need to be saved at call sites.
43 //
44 // SOC = Save-On-Call: The register allocator assumes that these registers
45 // can be used without saving upon entry to the method,
46 // but that they must be saved at call sites.
47 //
48 // SOE = Save-On-Entry: The register allocator assumes that these registers
49 // must be saved before using them upon entry to the
50 // method, but they do not need to be saved at call
51 // sites.
52 //
53 // AS = Always-Save: The register allocator assumes that these registers
54 // must be saved before using them upon entry to the
55 // method, & that they must be saved at call sites.
56 //
57 // Ideal Register Type is used to determine how to save & restore a
58 // register. Op_RegI will get spilled with LoadI/StoreI, Op_RegP will get
59 // spilled with LoadP/StoreP. If the register supports both, use Op_RegI.
60 //
61 // The encoding number is the actual bit-pattern placed into the opcodes.
62
63 // We must define the 64 bit int registers in two 32 bit halves, the
64 // real lower register and a virtual upper half register. upper halves
65 // are used by the register allocator but are not actually supplied as
66 // operands to memory ops.
67 //
68 // follow the C1 compiler in making registers
69 //
70 // r0-r7,r10-r26 volatile (caller save)
71 // r27-r32 system (no save, no allocate)
72 // r8-r9 non-allocatable (so we can use them as scratch regs)
73 //
74 // as regards Java usage. we don't use any callee save registers
75 // because this makes it difficult to de-optimise a frame (see comment
76 // in x86 implementation of Deoptimization::unwind_callee_save_values)
77 //
78
79 // General Registers
80
81 reg_def R0 ( SOC, SOC, Op_RegI, 0, r0->as_VMReg() );
82 reg_def R0_H ( SOC, SOC, Op_RegI, 0, r0->as_VMReg()->next() );
83 reg_def R1 ( SOC, SOC, Op_RegI, 1, r1->as_VMReg() );
84 reg_def R1_H ( SOC, SOC, Op_RegI, 1, r1->as_VMReg()->next() );
85 reg_def R2 ( SOC, SOC, Op_RegI, 2, r2->as_VMReg() );
86 reg_def R2_H ( SOC, SOC, Op_RegI, 2, r2->as_VMReg()->next() );
87 reg_def R3 ( SOC, SOC, Op_RegI, 3, r3->as_VMReg() );
88 reg_def R3_H ( SOC, SOC, Op_RegI, 3, r3->as_VMReg()->next() );
89 reg_def R4 ( SOC, SOC, Op_RegI, 4, r4->as_VMReg() );
90 reg_def R4_H ( SOC, SOC, Op_RegI, 4, r4->as_VMReg()->next() );
91 reg_def R5 ( SOC, SOC, Op_RegI, 5, r5->as_VMReg() );
92 reg_def R5_H ( SOC, SOC, Op_RegI, 5, r5->as_VMReg()->next() );
93 reg_def R6 ( SOC, SOC, Op_RegI, 6, r6->as_VMReg() );
94 reg_def R6_H ( SOC, SOC, Op_RegI, 6, r6->as_VMReg()->next() );
95 reg_def R7 ( SOC, SOC, Op_RegI, 7, r7->as_VMReg() );
96 reg_def R7_H ( SOC, SOC, Op_RegI, 7, r7->as_VMReg()->next() );
97 reg_def R8 ( NS, SOC, Op_RegI, 8, r8->as_VMReg() ); // rscratch1, non-allocatable
98 reg_def R8_H ( NS, SOC, Op_RegI, 8, r8->as_VMReg()->next() );
99 reg_def R9 ( NS, SOC, Op_RegI, 9, r9->as_VMReg() ); // rscratch2, non-allocatable
100 reg_def R9_H ( NS, SOC, Op_RegI, 9, r9->as_VMReg()->next() );
101 reg_def R10 ( SOC, SOC, Op_RegI, 10, r10->as_VMReg() );
102 reg_def R10_H ( SOC, SOC, Op_RegI, 10, r10->as_VMReg()->next());
103 reg_def R11 ( SOC, SOC, Op_RegI, 11, r11->as_VMReg() );
104 reg_def R11_H ( SOC, SOC, Op_RegI, 11, r11->as_VMReg()->next());
105 reg_def R12 ( SOC, SOC, Op_RegI, 12, r12->as_VMReg() );
106 reg_def R12_H ( SOC, SOC, Op_RegI, 12, r12->as_VMReg()->next());
107 reg_def R13 ( SOC, SOC, Op_RegI, 13, r13->as_VMReg() );
108 reg_def R13_H ( SOC, SOC, Op_RegI, 13, r13->as_VMReg()->next());
109 reg_def R14 ( SOC, SOC, Op_RegI, 14, r14->as_VMReg() );
110 reg_def R14_H ( SOC, SOC, Op_RegI, 14, r14->as_VMReg()->next());
111 reg_def R15 ( SOC, SOC, Op_RegI, 15, r15->as_VMReg() );
112 reg_def R15_H ( SOC, SOC, Op_RegI, 15, r15->as_VMReg()->next());
113 reg_def R16 ( SOC, SOC, Op_RegI, 16, r16->as_VMReg() );
114 reg_def R16_H ( SOC, SOC, Op_RegI, 16, r16->as_VMReg()->next());
115 reg_def R17 ( SOC, SOC, Op_RegI, 17, r17->as_VMReg() );
116 reg_def R17_H ( SOC, SOC, Op_RegI, 17, r17->as_VMReg()->next());
117 reg_def R18 ( SOC, SOC, Op_RegI, 18, r18_tls->as_VMReg() );
118 reg_def R18_H ( SOC, SOC, Op_RegI, 18, r18_tls->as_VMReg()->next());
119 reg_def R19 ( SOC, SOE, Op_RegI, 19, r19->as_VMReg() );
120 reg_def R19_H ( SOC, SOE, Op_RegI, 19, r19->as_VMReg()->next());
121 reg_def R20 ( SOC, SOE, Op_RegI, 20, r20->as_VMReg() ); // caller esp
122 reg_def R20_H ( SOC, SOE, Op_RegI, 20, r20->as_VMReg()->next());
123 reg_def R21 ( SOC, SOE, Op_RegI, 21, r21->as_VMReg() );
124 reg_def R21_H ( SOC, SOE, Op_RegI, 21, r21->as_VMReg()->next());
125 reg_def R22 ( SOC, SOE, Op_RegI, 22, r22->as_VMReg() );
126 reg_def R22_H ( SOC, SOE, Op_RegI, 22, r22->as_VMReg()->next());
127 reg_def R23 ( SOC, SOE, Op_RegI, 23, r23->as_VMReg() );
128 reg_def R23_H ( SOC, SOE, Op_RegI, 23, r23->as_VMReg()->next());
129 reg_def R24 ( SOC, SOE, Op_RegI, 24, r24->as_VMReg() );
130 reg_def R24_H ( SOC, SOE, Op_RegI, 24, r24->as_VMReg()->next());
131 reg_def R25 ( SOC, SOE, Op_RegI, 25, r25->as_VMReg() );
132 reg_def R25_H ( SOC, SOE, Op_RegI, 25, r25->as_VMReg()->next());
133 reg_def R26 ( SOC, SOE, Op_RegI, 26, r26->as_VMReg() );
134 reg_def R26_H ( SOC, SOE, Op_RegI, 26, r26->as_VMReg()->next());
135 reg_def R27 ( SOC, SOE, Op_RegI, 27, r27->as_VMReg() ); // heapbase
136 reg_def R27_H ( SOC, SOE, Op_RegI, 27, r27->as_VMReg()->next());
137 reg_def R28 ( NS, SOE, Op_RegI, 28, r28->as_VMReg() ); // thread
138 reg_def R28_H ( NS, SOE, Op_RegI, 28, r28->as_VMReg()->next());
139 reg_def R29 ( NS, NS, Op_RegI, 29, r29->as_VMReg() ); // fp
140 reg_def R29_H ( NS, NS, Op_RegI, 29, r29->as_VMReg()->next());
141 reg_def R30 ( NS, NS, Op_RegI, 30, r30->as_VMReg() ); // lr
142 reg_def R30_H ( NS, NS, Op_RegI, 30, r30->as_VMReg()->next());
143 reg_def R31 ( NS, NS, Op_RegI, 31, r31_sp->as_VMReg() ); // sp
144 reg_def R31_H ( NS, NS, Op_RegI, 31, r31_sp->as_VMReg()->next());
145
146 // ----------------------------
147 // Float/Double/Vector Registers
148 // ----------------------------
149
150 // Double Registers
151
152 // The rules of ADL require that double registers be defined in pairs.
153 // Each pair must be two 32-bit values, but not necessarily a pair of
154 // single float registers. In each pair, ADLC-assigned register numbers
155 // must be adjacent, with the lower number even. Finally, when the
156 // CPU stores such a register pair to memory, the word associated with
157 // the lower ADLC-assigned number must be stored to the lower address.
158
159 // AArch64 has 32 floating-point registers. Each can store a vector of
160 // single or double precision floating-point values up to 8 * 32
161 // floats, 4 * 64 bit floats or 2 * 128 bit floats. We currently only
162 // use the first float or double element of the vector.
163
164 // for Java use float registers v0-v15 are always save on call whereas
165 // the platform ABI treats v8-v15 as callee save). float registers
166 // v16-v31 are SOC as per the platform spec
167
168 // For SVE vector registers, we simply extend vector register size to 8
169 // 'logical' slots. This is nominally 256 bits but it actually covers
170 // all possible 'physical' SVE vector register lengths from 128 ~ 2048
171 // bits. The 'physical' SVE vector register length is detected during
172 // startup, so the register allocator is able to identify the correct
173 // number of bytes needed for an SVE spill/unspill.
174 // Note that a vector register with 4 slots denotes a 128-bit NEON
175 // register allowing it to be distinguished from the corresponding SVE
176 // vector register when the SVE vector length is 128 bits.
177
178 reg_def V0 ( SOC, SOC, Op_RegF, 0, v0->as_VMReg() );
179 reg_def V0_H ( SOC, SOC, Op_RegF, 0, v0->as_VMReg()->next() );
180 reg_def V0_J ( SOC, SOC, Op_RegF, 0, v0->as_VMReg()->next(2) );
181 reg_def V0_K ( SOC, SOC, Op_RegF, 0, v0->as_VMReg()->next(3) );
182 reg_def V0_L ( SOC, SOC, Op_RegF, 0, v0->as_VMReg()->next(4) );
183 reg_def V0_M ( SOC, SOC, Op_RegF, 0, v0->as_VMReg()->next(5) );
184 reg_def V0_N ( SOC, SOC, Op_RegF, 0, v0->as_VMReg()->next(6) );
185 reg_def V0_O ( SOC, SOC, Op_RegF, 0, v0->as_VMReg()->next(7) );
186
187 reg_def V1 ( SOC, SOC, Op_RegF, 1, v1->as_VMReg() );
188 reg_def V1_H ( SOC, SOC, Op_RegF, 1, v1->as_VMReg()->next() );
189 reg_def V1_J ( SOC, SOC, Op_RegF, 1, v1->as_VMReg()->next(2) );
190 reg_def V1_K ( SOC, SOC, Op_RegF, 1, v1->as_VMReg()->next(3) );
191 reg_def V1_L ( SOC, SOC, Op_RegF, 1, v1->as_VMReg()->next(4) );
192 reg_def V1_M ( SOC, SOC, Op_RegF, 1, v1->as_VMReg()->next(5) );
193 reg_def V1_N ( SOC, SOC, Op_RegF, 1, v1->as_VMReg()->next(6) );
194 reg_def V1_O ( SOC, SOC, Op_RegF, 1, v1->as_VMReg()->next(7) );
195
196 reg_def V2 ( SOC, SOC, Op_RegF, 2, v2->as_VMReg() );
197 reg_def V2_H ( SOC, SOC, Op_RegF, 2, v2->as_VMReg()->next() );
198 reg_def V2_J ( SOC, SOC, Op_RegF, 2, v2->as_VMReg()->next(2) );
199 reg_def V2_K ( SOC, SOC, Op_RegF, 2, v2->as_VMReg()->next(3) );
200 reg_def V2_L ( SOC, SOC, Op_RegF, 2, v2->as_VMReg()->next(4) );
201 reg_def V2_M ( SOC, SOC, Op_RegF, 2, v2->as_VMReg()->next(5) );
202 reg_def V2_N ( SOC, SOC, Op_RegF, 2, v2->as_VMReg()->next(6) );
203 reg_def V2_O ( SOC, SOC, Op_RegF, 2, v2->as_VMReg()->next(7) );
204
205 reg_def V3 ( SOC, SOC, Op_RegF, 3, v3->as_VMReg() );
206 reg_def V3_H ( SOC, SOC, Op_RegF, 3, v3->as_VMReg()->next() );
207 reg_def V3_J ( SOC, SOC, Op_RegF, 3, v3->as_VMReg()->next(2) );
208 reg_def V3_K ( SOC, SOC, Op_RegF, 3, v3->as_VMReg()->next(3) );
209 reg_def V3_L ( SOC, SOC, Op_RegF, 3, v3->as_VMReg()->next(4) );
210 reg_def V3_M ( SOC, SOC, Op_RegF, 3, v3->as_VMReg()->next(5) );
211 reg_def V3_N ( SOC, SOC, Op_RegF, 3, v3->as_VMReg()->next(6) );
212 reg_def V3_O ( SOC, SOC, Op_RegF, 3, v3->as_VMReg()->next(7) );
213
214 reg_def V4 ( SOC, SOC, Op_RegF, 4, v4->as_VMReg() );
215 reg_def V4_H ( SOC, SOC, Op_RegF, 4, v4->as_VMReg()->next() );
216 reg_def V4_J ( SOC, SOC, Op_RegF, 4, v4->as_VMReg()->next(2) );
217 reg_def V4_K ( SOC, SOC, Op_RegF, 4, v4->as_VMReg()->next(3) );
218 reg_def V4_L ( SOC, SOC, Op_RegF, 4, v4->as_VMReg()->next(4) );
219 reg_def V4_M ( SOC, SOC, Op_RegF, 4, v4->as_VMReg()->next(5) );
220 reg_def V4_N ( SOC, SOC, Op_RegF, 4, v4->as_VMReg()->next(6) );
221 reg_def V4_O ( SOC, SOC, Op_RegF, 4, v4->as_VMReg()->next(7) );
222
223 reg_def V5 ( SOC, SOC, Op_RegF, 5, v5->as_VMReg() );
224 reg_def V5_H ( SOC, SOC, Op_RegF, 5, v5->as_VMReg()->next() );
225 reg_def V5_J ( SOC, SOC, Op_RegF, 5, v5->as_VMReg()->next(2) );
226 reg_def V5_K ( SOC, SOC, Op_RegF, 5, v5->as_VMReg()->next(3) );
227 reg_def V5_L ( SOC, SOC, Op_RegF, 5, v5->as_VMReg()->next(4) );
228 reg_def V5_M ( SOC, SOC, Op_RegF, 5, v5->as_VMReg()->next(5) );
229 reg_def V5_N ( SOC, SOC, Op_RegF, 5, v5->as_VMReg()->next(6) );
230 reg_def V5_O ( SOC, SOC, Op_RegF, 5, v5->as_VMReg()->next(7) );
231
232 reg_def V6 ( SOC, SOC, Op_RegF, 6, v6->as_VMReg() );
233 reg_def V6_H ( SOC, SOC, Op_RegF, 6, v6->as_VMReg()->next() );
234 reg_def V6_J ( SOC, SOC, Op_RegF, 6, v6->as_VMReg()->next(2) );
235 reg_def V6_K ( SOC, SOC, Op_RegF, 6, v6->as_VMReg()->next(3) );
236 reg_def V6_L ( SOC, SOC, Op_RegF, 6, v6->as_VMReg()->next(4) );
237 reg_def V6_M ( SOC, SOC, Op_RegF, 6, v6->as_VMReg()->next(5) );
238 reg_def V6_N ( SOC, SOC, Op_RegF, 6, v6->as_VMReg()->next(6) );
239 reg_def V6_O ( SOC, SOC, Op_RegF, 6, v6->as_VMReg()->next(7) );
240
241 reg_def V7 ( SOC, SOC, Op_RegF, 7, v7->as_VMReg() );
242 reg_def V7_H ( SOC, SOC, Op_RegF, 7, v7->as_VMReg()->next() );
243 reg_def V7_J ( SOC, SOC, Op_RegF, 7, v7->as_VMReg()->next(2) );
244 reg_def V7_K ( SOC, SOC, Op_RegF, 7, v7->as_VMReg()->next(3) );
245 reg_def V7_L ( SOC, SOC, Op_RegF, 7, v7->as_VMReg()->next(4) );
246 reg_def V7_M ( SOC, SOC, Op_RegF, 7, v7->as_VMReg()->next(5) );
247 reg_def V7_N ( SOC, SOC, Op_RegF, 7, v7->as_VMReg()->next(6) );
248 reg_def V7_O ( SOC, SOC, Op_RegF, 7, v7->as_VMReg()->next(7) );
249
250 reg_def V8 ( SOC, SOE, Op_RegF, 8, v8->as_VMReg() );
251 reg_def V8_H ( SOC, SOE, Op_RegF, 8, v8->as_VMReg()->next() );
252 reg_def V8_J ( SOC, SOC, Op_RegF, 8, v8->as_VMReg()->next(2) );
253 reg_def V8_K ( SOC, SOC, Op_RegF, 8, v8->as_VMReg()->next(3) );
254 reg_def V8_L ( SOC, SOC, Op_RegF, 8, v8->as_VMReg()->next(4) );
255 reg_def V8_M ( SOC, SOC, Op_RegF, 8, v8->as_VMReg()->next(5) );
256 reg_def V8_N ( SOC, SOC, Op_RegF, 8, v8->as_VMReg()->next(6) );
257 reg_def V8_O ( SOC, SOC, Op_RegF, 8, v8->as_VMReg()->next(7) );
258
259 reg_def V9 ( SOC, SOE, Op_RegF, 9, v9->as_VMReg() );
260 reg_def V9_H ( SOC, SOE, Op_RegF, 9, v9->as_VMReg()->next() );
261 reg_def V9_J ( SOC, SOC, Op_RegF, 9, v9->as_VMReg()->next(2) );
262 reg_def V9_K ( SOC, SOC, Op_RegF, 9, v9->as_VMReg()->next(3) );
263 reg_def V9_L ( SOC, SOC, Op_RegF, 9, v9->as_VMReg()->next(4) );
264 reg_def V9_M ( SOC, SOC, Op_RegF, 9, v9->as_VMReg()->next(5) );
265 reg_def V9_N ( SOC, SOC, Op_RegF, 9, v9->as_VMReg()->next(6) );
266 reg_def V9_O ( SOC, SOC, Op_RegF, 9, v9->as_VMReg()->next(7) );
267
268 reg_def V10 ( SOC, SOE, Op_RegF, 10, v10->as_VMReg() );
269 reg_def V10_H ( SOC, SOE, Op_RegF, 10, v10->as_VMReg()->next() );
270 reg_def V10_J ( SOC, SOC, Op_RegF, 10, v10->as_VMReg()->next(2) );
271 reg_def V10_K ( SOC, SOC, Op_RegF, 10, v10->as_VMReg()->next(3) );
272 reg_def V10_L ( SOC, SOC, Op_RegF, 10, v10->as_VMReg()->next(4) );
273 reg_def V10_M ( SOC, SOC, Op_RegF, 10, v10->as_VMReg()->next(5) );
274 reg_def V10_N ( SOC, SOC, Op_RegF, 10, v10->as_VMReg()->next(6) );
275 reg_def V10_O ( SOC, SOC, Op_RegF, 10, v10->as_VMReg()->next(7) );
276
277 reg_def V11 ( SOC, SOE, Op_RegF, 11, v11->as_VMReg() );
278 reg_def V11_H ( SOC, SOE, Op_RegF, 11, v11->as_VMReg()->next() );
279 reg_def V11_J ( SOC, SOC, Op_RegF, 11, v11->as_VMReg()->next(2) );
280 reg_def V11_K ( SOC, SOC, Op_RegF, 11, v11->as_VMReg()->next(3) );
281 reg_def V11_L ( SOC, SOC, Op_RegF, 11, v11->as_VMReg()->next(4) );
282 reg_def V11_M ( SOC, SOC, Op_RegF, 11, v11->as_VMReg()->next(5) );
283 reg_def V11_N ( SOC, SOC, Op_RegF, 11, v11->as_VMReg()->next(6) );
284 reg_def V11_O ( SOC, SOC, Op_RegF, 11, v11->as_VMReg()->next(7) );
285
286 reg_def V12 ( SOC, SOE, Op_RegF, 12, v12->as_VMReg() );
287 reg_def V12_H ( SOC, SOE, Op_RegF, 12, v12->as_VMReg()->next() );
288 reg_def V12_J ( SOC, SOC, Op_RegF, 12, v12->as_VMReg()->next(2) );
289 reg_def V12_K ( SOC, SOC, Op_RegF, 12, v12->as_VMReg()->next(3) );
290 reg_def V12_L ( SOC, SOC, Op_RegF, 12, v12->as_VMReg()->next(4) );
291 reg_def V12_M ( SOC, SOC, Op_RegF, 12, v12->as_VMReg()->next(5) );
292 reg_def V12_N ( SOC, SOC, Op_RegF, 12, v12->as_VMReg()->next(6) );
293 reg_def V12_O ( SOC, SOC, Op_RegF, 12, v12->as_VMReg()->next(7) );
294
295 reg_def V13 ( SOC, SOE, Op_RegF, 13, v13->as_VMReg() );
296 reg_def V13_H ( SOC, SOE, Op_RegF, 13, v13->as_VMReg()->next() );
297 reg_def V13_J ( SOC, SOC, Op_RegF, 13, v13->as_VMReg()->next(2) );
298 reg_def V13_K ( SOC, SOC, Op_RegF, 13, v13->as_VMReg()->next(3) );
299 reg_def V13_L ( SOC, SOC, Op_RegF, 13, v13->as_VMReg()->next(4) );
300 reg_def V13_M ( SOC, SOC, Op_RegF, 13, v13->as_VMReg()->next(5) );
301 reg_def V13_N ( SOC, SOC, Op_RegF, 13, v13->as_VMReg()->next(6) );
302 reg_def V13_O ( SOC, SOC, Op_RegF, 13, v13->as_VMReg()->next(7) );
303
304 reg_def V14 ( SOC, SOE, Op_RegF, 14, v14->as_VMReg() );
305 reg_def V14_H ( SOC, SOE, Op_RegF, 14, v14->as_VMReg()->next() );
306 reg_def V14_J ( SOC, SOC, Op_RegF, 14, v14->as_VMReg()->next(2) );
307 reg_def V14_K ( SOC, SOC, Op_RegF, 14, v14->as_VMReg()->next(3) );
308 reg_def V14_L ( SOC, SOC, Op_RegF, 14, v14->as_VMReg()->next(4) );
309 reg_def V14_M ( SOC, SOC, Op_RegF, 14, v14->as_VMReg()->next(5) );
310 reg_def V14_N ( SOC, SOC, Op_RegF, 14, v14->as_VMReg()->next(6) );
311 reg_def V14_O ( SOC, SOC, Op_RegF, 14, v14->as_VMReg()->next(7) );
312
313 reg_def V15 ( SOC, SOE, Op_RegF, 15, v15->as_VMReg() );
314 reg_def V15_H ( SOC, SOE, Op_RegF, 15, v15->as_VMReg()->next() );
315 reg_def V15_J ( SOC, SOC, Op_RegF, 15, v15->as_VMReg()->next(2) );
316 reg_def V15_K ( SOC, SOC, Op_RegF, 15, v15->as_VMReg()->next(3) );
317 reg_def V15_L ( SOC, SOC, Op_RegF, 15, v15->as_VMReg()->next(4) );
318 reg_def V15_M ( SOC, SOC, Op_RegF, 15, v15->as_VMReg()->next(5) );
319 reg_def V15_N ( SOC, SOC, Op_RegF, 15, v15->as_VMReg()->next(6) );
320 reg_def V15_O ( SOC, SOC, Op_RegF, 15, v15->as_VMReg()->next(7) );
321
322 reg_def V16 ( SOC, SOC, Op_RegF, 16, v16->as_VMReg() );
323 reg_def V16_H ( SOC, SOC, Op_RegF, 16, v16->as_VMReg()->next() );
324 reg_def V16_J ( SOC, SOC, Op_RegF, 16, v16->as_VMReg()->next(2) );
325 reg_def V16_K ( SOC, SOC, Op_RegF, 16, v16->as_VMReg()->next(3) );
326 reg_def V16_L ( SOC, SOC, Op_RegF, 16, v16->as_VMReg()->next(4) );
327 reg_def V16_M ( SOC, SOC, Op_RegF, 16, v16->as_VMReg()->next(5) );
328 reg_def V16_N ( SOC, SOC, Op_RegF, 16, v16->as_VMReg()->next(6) );
329 reg_def V16_O ( SOC, SOC, Op_RegF, 16, v16->as_VMReg()->next(7) );
330
331 reg_def V17 ( SOC, SOC, Op_RegF, 17, v17->as_VMReg() );
332 reg_def V17_H ( SOC, SOC, Op_RegF, 17, v17->as_VMReg()->next() );
333 reg_def V17_J ( SOC, SOC, Op_RegF, 17, v17->as_VMReg()->next(2) );
334 reg_def V17_K ( SOC, SOC, Op_RegF, 17, v17->as_VMReg()->next(3) );
335 reg_def V17_L ( SOC, SOC, Op_RegF, 17, v17->as_VMReg()->next(4) );
336 reg_def V17_M ( SOC, SOC, Op_RegF, 17, v17->as_VMReg()->next(5) );
337 reg_def V17_N ( SOC, SOC, Op_RegF, 17, v17->as_VMReg()->next(6) );
338 reg_def V17_O ( SOC, SOC, Op_RegF, 17, v17->as_VMReg()->next(7) );
339
340 reg_def V18 ( SOC, SOC, Op_RegF, 18, v18->as_VMReg() );
341 reg_def V18_H ( SOC, SOC, Op_RegF, 18, v18->as_VMReg()->next() );
342 reg_def V18_J ( SOC, SOC, Op_RegF, 18, v18->as_VMReg()->next(2) );
343 reg_def V18_K ( SOC, SOC, Op_RegF, 18, v18->as_VMReg()->next(3) );
344 reg_def V18_L ( SOC, SOC, Op_RegF, 18, v18->as_VMReg()->next(4) );
345 reg_def V18_M ( SOC, SOC, Op_RegF, 18, v18->as_VMReg()->next(5) );
346 reg_def V18_N ( SOC, SOC, Op_RegF, 18, v18->as_VMReg()->next(6) );
347 reg_def V18_O ( SOC, SOC, Op_RegF, 18, v18->as_VMReg()->next(7) );
348
349 reg_def V19 ( SOC, SOC, Op_RegF, 19, v19->as_VMReg() );
350 reg_def V19_H ( SOC, SOC, Op_RegF, 19, v19->as_VMReg()->next() );
351 reg_def V19_J ( SOC, SOC, Op_RegF, 19, v19->as_VMReg()->next(2) );
352 reg_def V19_K ( SOC, SOC, Op_RegF, 19, v19->as_VMReg()->next(3) );
353 reg_def V19_L ( SOC, SOC, Op_RegF, 19, v19->as_VMReg()->next(4) );
354 reg_def V19_M ( SOC, SOC, Op_RegF, 19, v19->as_VMReg()->next(5) );
355 reg_def V19_N ( SOC, SOC, Op_RegF, 19, v19->as_VMReg()->next(6) );
356 reg_def V19_O ( SOC, SOC, Op_RegF, 19, v19->as_VMReg()->next(7) );
357
358 reg_def V20 ( SOC, SOC, Op_RegF, 20, v20->as_VMReg() );
359 reg_def V20_H ( SOC, SOC, Op_RegF, 20, v20->as_VMReg()->next() );
360 reg_def V20_J ( SOC, SOC, Op_RegF, 20, v20->as_VMReg()->next(2) );
361 reg_def V20_K ( SOC, SOC, Op_RegF, 20, v20->as_VMReg()->next(3) );
362 reg_def V20_L ( SOC, SOC, Op_RegF, 20, v20->as_VMReg()->next(4) );
363 reg_def V20_M ( SOC, SOC, Op_RegF, 20, v20->as_VMReg()->next(5) );
364 reg_def V20_N ( SOC, SOC, Op_RegF, 20, v20->as_VMReg()->next(6) );
365 reg_def V20_O ( SOC, SOC, Op_RegF, 20, v20->as_VMReg()->next(7) );
366
367 reg_def V21 ( SOC, SOC, Op_RegF, 21, v21->as_VMReg() );
368 reg_def V21_H ( SOC, SOC, Op_RegF, 21, v21->as_VMReg()->next() );
369 reg_def V21_J ( SOC, SOC, Op_RegF, 21, v21->as_VMReg()->next(2) );
370 reg_def V21_K ( SOC, SOC, Op_RegF, 21, v21->as_VMReg()->next(3) );
371 reg_def V21_L ( SOC, SOC, Op_RegF, 21, v21->as_VMReg()->next(4) );
372 reg_def V21_M ( SOC, SOC, Op_RegF, 21, v21->as_VMReg()->next(5) );
373 reg_def V21_N ( SOC, SOC, Op_RegF, 21, v21->as_VMReg()->next(6) );
374 reg_def V21_O ( SOC, SOC, Op_RegF, 21, v21->as_VMReg()->next(7) );
375
376 reg_def V22 ( SOC, SOC, Op_RegF, 22, v22->as_VMReg() );
377 reg_def V22_H ( SOC, SOC, Op_RegF, 22, v22->as_VMReg()->next() );
378 reg_def V22_J ( SOC, SOC, Op_RegF, 22, v22->as_VMReg()->next(2) );
379 reg_def V22_K ( SOC, SOC, Op_RegF, 22, v22->as_VMReg()->next(3) );
380 reg_def V22_L ( SOC, SOC, Op_RegF, 22, v22->as_VMReg()->next(4) );
381 reg_def V22_M ( SOC, SOC, Op_RegF, 22, v22->as_VMReg()->next(5) );
382 reg_def V22_N ( SOC, SOC, Op_RegF, 22, v22->as_VMReg()->next(6) );
383 reg_def V22_O ( SOC, SOC, Op_RegF, 22, v22->as_VMReg()->next(7) );
384
385 reg_def V23 ( SOC, SOC, Op_RegF, 23, v23->as_VMReg() );
386 reg_def V23_H ( SOC, SOC, Op_RegF, 23, v23->as_VMReg()->next() );
387 reg_def V23_J ( SOC, SOC, Op_RegF, 23, v23->as_VMReg()->next(2) );
388 reg_def V23_K ( SOC, SOC, Op_RegF, 23, v23->as_VMReg()->next(3) );
389 reg_def V23_L ( SOC, SOC, Op_RegF, 23, v23->as_VMReg()->next(4) );
390 reg_def V23_M ( SOC, SOC, Op_RegF, 23, v23->as_VMReg()->next(5) );
391 reg_def V23_N ( SOC, SOC, Op_RegF, 23, v23->as_VMReg()->next(6) );
392 reg_def V23_O ( SOC, SOC, Op_RegF, 23, v23->as_VMReg()->next(7) );
393
394 reg_def V24 ( SOC, SOC, Op_RegF, 24, v24->as_VMReg() );
395 reg_def V24_H ( SOC, SOC, Op_RegF, 24, v24->as_VMReg()->next() );
396 reg_def V24_J ( SOC, SOC, Op_RegF, 24, v24->as_VMReg()->next(2) );
397 reg_def V24_K ( SOC, SOC, Op_RegF, 24, v24->as_VMReg()->next(3) );
398 reg_def V24_L ( SOC, SOC, Op_RegF, 24, v24->as_VMReg()->next(4) );
399 reg_def V24_M ( SOC, SOC, Op_RegF, 24, v24->as_VMReg()->next(5) );
400 reg_def V24_N ( SOC, SOC, Op_RegF, 24, v24->as_VMReg()->next(6) );
401 reg_def V24_O ( SOC, SOC, Op_RegF, 24, v24->as_VMReg()->next(7) );
402
403 reg_def V25 ( SOC, SOC, Op_RegF, 25, v25->as_VMReg() );
404 reg_def V25_H ( SOC, SOC, Op_RegF, 25, v25->as_VMReg()->next() );
405 reg_def V25_J ( SOC, SOC, Op_RegF, 25, v25->as_VMReg()->next(2) );
406 reg_def V25_K ( SOC, SOC, Op_RegF, 25, v25->as_VMReg()->next(3) );
407 reg_def V25_L ( SOC, SOC, Op_RegF, 25, v25->as_VMReg()->next(4) );
408 reg_def V25_M ( SOC, SOC, Op_RegF, 25, v25->as_VMReg()->next(5) );
409 reg_def V25_N ( SOC, SOC, Op_RegF, 25, v25->as_VMReg()->next(6) );
410 reg_def V25_O ( SOC, SOC, Op_RegF, 25, v25->as_VMReg()->next(7) );
411
412 reg_def V26 ( SOC, SOC, Op_RegF, 26, v26->as_VMReg() );
413 reg_def V26_H ( SOC, SOC, Op_RegF, 26, v26->as_VMReg()->next() );
414 reg_def V26_J ( SOC, SOC, Op_RegF, 26, v26->as_VMReg()->next(2) );
415 reg_def V26_K ( SOC, SOC, Op_RegF, 26, v26->as_VMReg()->next(3) );
416 reg_def V26_L ( SOC, SOC, Op_RegF, 26, v26->as_VMReg()->next(4) );
417 reg_def V26_M ( SOC, SOC, Op_RegF, 26, v26->as_VMReg()->next(5) );
418 reg_def V26_N ( SOC, SOC, Op_RegF, 26, v26->as_VMReg()->next(6) );
419 reg_def V26_O ( SOC, SOC, Op_RegF, 26, v26->as_VMReg()->next(7) );
420
421 reg_def V27 ( SOC, SOC, Op_RegF, 27, v27->as_VMReg() );
422 reg_def V27_H ( SOC, SOC, Op_RegF, 27, v27->as_VMReg()->next() );
423 reg_def V27_J ( SOC, SOC, Op_RegF, 27, v27->as_VMReg()->next(2) );
424 reg_def V27_K ( SOC, SOC, Op_RegF, 27, v27->as_VMReg()->next(3) );
425 reg_def V27_L ( SOC, SOC, Op_RegF, 27, v27->as_VMReg()->next(4) );
426 reg_def V27_M ( SOC, SOC, Op_RegF, 27, v27->as_VMReg()->next(5) );
427 reg_def V27_N ( SOC, SOC, Op_RegF, 27, v27->as_VMReg()->next(6) );
428 reg_def V27_O ( SOC, SOC, Op_RegF, 27, v27->as_VMReg()->next(7) );
429
430 reg_def V28 ( SOC, SOC, Op_RegF, 28, v28->as_VMReg() );
431 reg_def V28_H ( SOC, SOC, Op_RegF, 28, v28->as_VMReg()->next() );
432 reg_def V28_J ( SOC, SOC, Op_RegF, 28, v28->as_VMReg()->next(2) );
433 reg_def V28_K ( SOC, SOC, Op_RegF, 28, v28->as_VMReg()->next(3) );
434 reg_def V28_L ( SOC, SOC, Op_RegF, 28, v28->as_VMReg()->next(4) );
435 reg_def V28_M ( SOC, SOC, Op_RegF, 28, v28->as_VMReg()->next(5) );
436 reg_def V28_N ( SOC, SOC, Op_RegF, 28, v28->as_VMReg()->next(6) );
437 reg_def V28_O ( SOC, SOC, Op_RegF, 28, v28->as_VMReg()->next(7) );
438
439 reg_def V29 ( SOC, SOC, Op_RegF, 29, v29->as_VMReg() );
440 reg_def V29_H ( SOC, SOC, Op_RegF, 29, v29->as_VMReg()->next() );
441 reg_def V29_J ( SOC, SOC, Op_RegF, 29, v29->as_VMReg()->next(2) );
442 reg_def V29_K ( SOC, SOC, Op_RegF, 29, v29->as_VMReg()->next(3) );
443 reg_def V29_L ( SOC, SOC, Op_RegF, 29, v29->as_VMReg()->next(4) );
444 reg_def V29_M ( SOC, SOC, Op_RegF, 29, v29->as_VMReg()->next(5) );
445 reg_def V29_N ( SOC, SOC, Op_RegF, 29, v29->as_VMReg()->next(6) );
446 reg_def V29_O ( SOC, SOC, Op_RegF, 29, v29->as_VMReg()->next(7) );
447
448 reg_def V30 ( SOC, SOC, Op_RegF, 30, v30->as_VMReg() );
449 reg_def V30_H ( SOC, SOC, Op_RegF, 30, v30->as_VMReg()->next() );
450 reg_def V30_J ( SOC, SOC, Op_RegF, 30, v30->as_VMReg()->next(2) );
451 reg_def V30_K ( SOC, SOC, Op_RegF, 30, v30->as_VMReg()->next(3) );
452 reg_def V30_L ( SOC, SOC, Op_RegF, 30, v30->as_VMReg()->next(4) );
453 reg_def V30_M ( SOC, SOC, Op_RegF, 30, v30->as_VMReg()->next(5) );
454 reg_def V30_N ( SOC, SOC, Op_RegF, 30, v30->as_VMReg()->next(6) );
455 reg_def V30_O ( SOC, SOC, Op_RegF, 30, v30->as_VMReg()->next(7) );
456
457 reg_def V31 ( SOC, SOC, Op_RegF, 31, v31->as_VMReg() );
458 reg_def V31_H ( SOC, SOC, Op_RegF, 31, v31->as_VMReg()->next() );
459 reg_def V31_J ( SOC, SOC, Op_RegF, 31, v31->as_VMReg()->next(2) );
460 reg_def V31_K ( SOC, SOC, Op_RegF, 31, v31->as_VMReg()->next(3) );
461 reg_def V31_L ( SOC, SOC, Op_RegF, 31, v31->as_VMReg()->next(4) );
462 reg_def V31_M ( SOC, SOC, Op_RegF, 31, v31->as_VMReg()->next(5) );
463 reg_def V31_N ( SOC, SOC, Op_RegF, 31, v31->as_VMReg()->next(6) );
464 reg_def V31_O ( SOC, SOC, Op_RegF, 31, v31->as_VMReg()->next(7) );
465
466
467 // ----------------------------
468 // SVE Predicate Registers
469 // ----------------------------
470 reg_def P0 (SOC, SOC, Op_RegVectMask, 0, p0->as_VMReg());
471 reg_def P1 (SOC, SOC, Op_RegVectMask, 1, p1->as_VMReg());
472 reg_def P2 (SOC, SOC, Op_RegVectMask, 2, p2->as_VMReg());
473 reg_def P3 (SOC, SOC, Op_RegVectMask, 3, p3->as_VMReg());
474 reg_def P4 (SOC, SOC, Op_RegVectMask, 4, p4->as_VMReg());
475 reg_def P5 (SOC, SOC, Op_RegVectMask, 5, p5->as_VMReg());
476 reg_def P6 (SOC, SOC, Op_RegVectMask, 6, p6->as_VMReg());
477 reg_def P7 (SOC, SOC, Op_RegVectMask, 7, p7->as_VMReg());
478 reg_def P8 (SOC, SOC, Op_RegVectMask, 8, p8->as_VMReg());
479 reg_def P9 (SOC, SOC, Op_RegVectMask, 9, p9->as_VMReg());
480 reg_def P10 (SOC, SOC, Op_RegVectMask, 10, p10->as_VMReg());
481 reg_def P11 (SOC, SOC, Op_RegVectMask, 11, p11->as_VMReg());
482 reg_def P12 (SOC, SOC, Op_RegVectMask, 12, p12->as_VMReg());
483 reg_def P13 (SOC, SOC, Op_RegVectMask, 13, p13->as_VMReg());
484 reg_def P14 (SOC, SOC, Op_RegVectMask, 14, p14->as_VMReg());
485 reg_def P15 (SOC, SOC, Op_RegVectMask, 15, p15->as_VMReg());
486
487 // ----------------------------
488 // Special Registers
489 // ----------------------------
490
491 // the AArch64 CSPR status flag register is not directly acessible as
492 // instruction operand. the FPSR status flag register is a system
493 // register which can be written/read using MSR/MRS but again does not
494 // appear as an operand (a code identifying the FSPR occurs as an
495 // immediate value in the instruction).
496
497 reg_def RFLAGS(SOC, SOC, 0, 32, VMRegImpl::Bad());
498
499 // Specify priority of register selection within phases of register
500 // allocation. Highest priority is first. A useful heuristic is to
501 // give registers a low priority when they are required by machine
502 // instructions, like EAX and EDX on I486, and choose no-save registers
503 // before save-on-call, & save-on-call before save-on-entry. Registers
504 // which participate in fixed calling sequences should come last.
505 // Registers which are used as pairs must fall on an even boundary.
506
507 alloc_class chunk0(
508 // volatiles
509 R10, R10_H,
510 R11, R11_H,
511 R12, R12_H,
512 R13, R13_H,
513 R14, R14_H,
514 R15, R15_H,
515 R16, R16_H,
516 R17, R17_H,
517 R18, R18_H,
518
519 // arg registers
520 R0, R0_H,
521 R1, R1_H,
522 R2, R2_H,
523 R3, R3_H,
524 R4, R4_H,
525 R5, R5_H,
526 R6, R6_H,
527 R7, R7_H,
528
529 // non-volatiles
530 R19, R19_H,
531 R20, R20_H,
532 R21, R21_H,
533 R22, R22_H,
534 R23, R23_H,
535 R24, R24_H,
536 R25, R25_H,
537 R26, R26_H,
538
539 // non-allocatable registers
540
541 R27, R27_H, // heapbase
542 R28, R28_H, // thread
543 R29, R29_H, // fp
544 R30, R30_H, // lr
545 R31, R31_H, // sp
546 R8, R8_H, // rscratch1
547 R9, R9_H, // rscratch2
548 );
549
550 alloc_class chunk1(
551
552 // no save
553 V16, V16_H, V16_J, V16_K, V16_L, V16_M, V16_N, V16_O,
554 V17, V17_H, V17_J, V17_K, V17_L, V17_M, V17_N, V17_O,
555 V18, V18_H, V18_J, V18_K, V18_L, V18_M, V18_N, V18_O,
556 V19, V19_H, V19_J, V19_K, V19_L, V19_M, V19_N, V19_O,
557 V20, V20_H, V20_J, V20_K, V20_L, V20_M, V20_N, V20_O,
558 V21, V21_H, V21_J, V21_K, V21_L, V21_M, V21_N, V21_O,
559 V22, V22_H, V22_J, V22_K, V22_L, V22_M, V22_N, V22_O,
560 V23, V23_H, V23_J, V23_K, V23_L, V23_M, V23_N, V23_O,
561 V24, V24_H, V24_J, V24_K, V24_L, V24_M, V24_N, V24_O,
562 V25, V25_H, V25_J, V25_K, V25_L, V25_M, V25_N, V25_O,
563 V26, V26_H, V26_J, V26_K, V26_L, V26_M, V26_N, V26_O,
564 V27, V27_H, V27_J, V27_K, V27_L, V27_M, V27_N, V27_O,
565 V28, V28_H, V28_J, V28_K, V28_L, V28_M, V28_N, V28_O,
566 V29, V29_H, V29_J, V29_K, V29_L, V29_M, V29_N, V29_O,
567 V30, V30_H, V30_J, V30_K, V30_L, V30_M, V30_N, V30_O,
568 V31, V31_H, V31_J, V31_K, V31_L, V31_M, V31_N, V31_O,
569
570 // arg registers
571 V0, V0_H, V0_J, V0_K, V0_L, V0_M, V0_N, V0_O,
572 V1, V1_H, V1_J, V1_K, V1_L, V1_M, V1_N, V1_O,
573 V2, V2_H, V2_J, V2_K, V2_L, V2_M, V2_N, V2_O,
574 V3, V3_H, V3_J, V3_K, V3_L, V3_M, V3_N, V3_O,
575 V4, V4_H, V4_J, V4_K, V4_L, V4_M, V4_N, V4_O,
576 V5, V5_H, V5_J, V5_K, V5_L, V5_M, V5_N, V5_O,
577 V6, V6_H, V6_J, V6_K, V6_L, V6_M, V6_N, V6_O,
578 V7, V7_H, V7_J, V7_K, V7_L, V7_M, V7_N, V7_O,
579
580 // non-volatiles
581 V8, V8_H, V8_J, V8_K, V8_L, V8_M, V8_N, V8_O,
582 V9, V9_H, V9_J, V9_K, V9_L, V9_M, V9_N, V9_O,
583 V10, V10_H, V10_J, V10_K, V10_L, V10_M, V10_N, V10_O,
584 V11, V11_H, V11_J, V11_K, V11_L, V11_M, V11_N, V11_O,
585 V12, V12_H, V12_J, V12_K, V12_L, V12_M, V12_N, V12_O,
586 V13, V13_H, V13_J, V13_K, V13_L, V13_M, V13_N, V13_O,
587 V14, V14_H, V14_J, V14_K, V14_L, V14_M, V14_N, V14_O,
588 V15, V15_H, V15_J, V15_K, V15_L, V15_M, V15_N, V15_O,
589 );
590
591 alloc_class chunk2 (
592 P0,
593 P1,
594 P2,
595 P3,
596 P4,
597 P5,
598 P6,
599 P7,
600
601 P8,
602 P9,
603 P10,
604 P11,
605 P12,
606 P13,
607 P14,
608 P15,
609 );
610
611 alloc_class chunk3(RFLAGS);
612
613 //----------Architecture Description Register Classes--------------------------
614 // Several register classes are automatically defined based upon information in
615 // this architecture description.
616 // 1) reg_class inline_cache_reg ( /* as def'd in frame section */ )
617 // 2) reg_class stack_slots( /* one chunk of stack-based "registers" */ )
618 //
619
620 // Class for all 32 bit general purpose registers
621 reg_class all_reg32(
622 R0,
623 R1,
624 R2,
625 R3,
626 R4,
627 R5,
628 R6,
629 R7,
630 R10,
631 R11,
632 R12,
633 R13,
634 R14,
635 R15,
636 R16,
637 R17,
638 R18,
639 R19,
640 R20,
641 R21,
642 R22,
643 R23,
644 R24,
645 R25,
646 R26,
647 R27,
648 R28,
649 R29,
650 R30,
651 R31
652 );
653
654
655 // Class for all 32 bit integer registers (excluding SP which
656 // will never be used as an integer register)
657 reg_class any_reg32 %{
658 return _ANY_REG32_mask;
659 %}
660
661 // Singleton class for R0 int register
662 reg_class int_r0_reg(R0);
663
664 // Singleton class for R2 int register
665 reg_class int_r2_reg(R2);
666
667 // Singleton class for R3 int register
668 reg_class int_r3_reg(R3);
669
670 // Singleton class for R4 int register
671 reg_class int_r4_reg(R4);
672
673 // Singleton class for R31 int register
674 reg_class int_r31_reg(R31);
675
676 // Class for all 64 bit general purpose registers
677 reg_class all_reg(
678 R0, R0_H,
679 R1, R1_H,
680 R2, R2_H,
681 R3, R3_H,
682 R4, R4_H,
683 R5, R5_H,
684 R6, R6_H,
685 R7, R7_H,
686 R10, R10_H,
687 R11, R11_H,
688 R12, R12_H,
689 R13, R13_H,
690 R14, R14_H,
691 R15, R15_H,
692 R16, R16_H,
693 R17, R17_H,
694 R18, R18_H,
695 R19, R19_H,
696 R20, R20_H,
697 R21, R21_H,
698 R22, R22_H,
699 R23, R23_H,
700 R24, R24_H,
701 R25, R25_H,
702 R26, R26_H,
703 R27, R27_H,
704 R28, R28_H,
705 R29, R29_H,
706 R30, R30_H,
707 R31, R31_H
708 );
709
710 // Class for all long integer registers (including SP)
711 reg_class any_reg %{
712 return _ANY_REG_mask;
713 %}
714
715 // Class for non-allocatable 32 bit registers
716 reg_class non_allocatable_reg32(
717 #ifdef R18_RESERVED
718 // See comment in register_aarch64.hpp
719 R18, // tls on Windows
720 #endif
721 R28, // thread
722 R30, // lr
723 R31 // sp
724 );
725
726 // Class for non-allocatable 64 bit registers
727 reg_class non_allocatable_reg(
728 #ifdef R18_RESERVED
729 // See comment in register_aarch64.hpp
730 R18, R18_H, // tls on Windows, platform register on macOS
731 #endif
732 R28, R28_H, // thread
733 R30, R30_H, // lr
734 R31, R31_H // sp
735 );
736
737 // Class for all non-special integer registers
738 reg_class no_special_reg32 %{
739 return _NO_SPECIAL_REG32_mask;
740 %}
741
742 // Class for all non-special long integer registers
743 reg_class no_special_reg %{
744 return _NO_SPECIAL_REG_mask;
745 %}
746
747 // Class for 64 bit register r0
748 reg_class r0_reg(
749 R0, R0_H
750 );
751
752 // Class for 64 bit register r1
753 reg_class r1_reg(
754 R1, R1_H
755 );
756
757 // Class for 64 bit register r2
758 reg_class r2_reg(
759 R2, R2_H
760 );
761
762 // Class for 64 bit register r3
763 reg_class r3_reg(
764 R3, R3_H
765 );
766
767 // Class for 64 bit register r4
768 reg_class r4_reg(
769 R4, R4_H
770 );
771
772 // Class for 64 bit register r5
773 reg_class r5_reg(
774 R5, R5_H
775 );
776
777 // Class for 64 bit register r10
778 reg_class r10_reg(
779 R10, R10_H
780 );
781
782 // Class for 64 bit register r11
783 reg_class r11_reg(
784 R11, R11_H
785 );
786
787 // Class for method register
788 reg_class method_reg(
789 R12, R12_H
790 );
791
792 // Class for heapbase register
793 reg_class heapbase_reg(
794 R27, R27_H
795 );
796
797 // Class for thread register
798 reg_class thread_reg(
799 R28, R28_H
800 );
801
802 // Class for frame pointer register
803 reg_class fp_reg(
804 R29, R29_H
805 );
806
807 // Class for link register
808 reg_class lr_reg(
809 R30, R30_H
810 );
811
812 // Class for long sp register
813 reg_class sp_reg(
814 R31, R31_H
815 );
816
817 // Class for all pointer registers
818 reg_class ptr_reg %{
819 return _PTR_REG_mask;
820 %}
821
822 // Class for all non_special pointer registers
823 reg_class no_special_ptr_reg %{
824 return _NO_SPECIAL_PTR_REG_mask;
825 %}
826
827 // Class for all float registers
828 reg_class float_reg(
829 V0,
830 V1,
831 V2,
832 V3,
833 V4,
834 V5,
835 V6,
836 V7,
837 V8,
838 V9,
839 V10,
840 V11,
841 V12,
842 V13,
843 V14,
844 V15,
845 V16,
846 V17,
847 V18,
848 V19,
849 V20,
850 V21,
851 V22,
852 V23,
853 V24,
854 V25,
855 V26,
856 V27,
857 V28,
858 V29,
859 V30,
860 V31
861 );
862
863 // Double precision float registers have virtual `high halves' that
864 // are needed by the allocator.
865 // Class for all double registers
866 reg_class double_reg(
867 V0, V0_H,
868 V1, V1_H,
869 V2, V2_H,
870 V3, V3_H,
871 V4, V4_H,
872 V5, V5_H,
873 V6, V6_H,
874 V7, V7_H,
875 V8, V8_H,
876 V9, V9_H,
877 V10, V10_H,
878 V11, V11_H,
879 V12, V12_H,
880 V13, V13_H,
881 V14, V14_H,
882 V15, V15_H,
883 V16, V16_H,
884 V17, V17_H,
885 V18, V18_H,
886 V19, V19_H,
887 V20, V20_H,
888 V21, V21_H,
889 V22, V22_H,
890 V23, V23_H,
891 V24, V24_H,
892 V25, V25_H,
893 V26, V26_H,
894 V27, V27_H,
895 V28, V28_H,
896 V29, V29_H,
897 V30, V30_H,
898 V31, V31_H
899 );
900
901 // Class for all SVE vector registers.
902 reg_class vectora_reg (
903 V0, V0_H, V0_J, V0_K, V0_L, V0_M, V0_N, V0_O,
904 V1, V1_H, V1_J, V1_K, V1_L, V1_M, V1_N, V1_O,
905 V2, V2_H, V2_J, V2_K, V2_L, V2_M, V2_N, V2_O,
906 V3, V3_H, V3_J, V3_K, V3_L, V3_M, V3_N, V3_O,
907 V4, V4_H, V4_J, V4_K, V4_L, V4_M, V4_N, V4_O,
908 V5, V5_H, V5_J, V5_K, V5_L, V5_M, V5_N, V5_O,
909 V6, V6_H, V6_J, V6_K, V6_L, V6_M, V6_N, V6_O,
910 V7, V7_H, V7_J, V7_K, V7_L, V7_M, V7_N, V7_O,
911 V8, V8_H, V8_J, V8_K, V8_L, V8_M, V8_N, V8_O,
912 V9, V9_H, V9_J, V9_K, V9_L, V9_M, V9_N, V9_O,
913 V10, V10_H, V10_J, V10_K, V10_L, V10_M, V10_N, V10_O,
914 V11, V11_H, V11_J, V11_K, V11_L, V11_M, V11_N, V11_O,
915 V12, V12_H, V12_J, V12_K, V12_L, V12_M, V12_N, V12_O,
916 V13, V13_H, V13_J, V13_K, V13_L, V13_M, V13_N, V13_O,
917 V14, V14_H, V14_J, V14_K, V14_L, V14_M, V14_N, V14_O,
918 V15, V15_H, V15_J, V15_K, V15_L, V15_M, V15_N, V15_O,
919 V16, V16_H, V16_J, V16_K, V16_L, V16_M, V16_N, V16_O,
920 V17, V17_H, V17_J, V17_K, V17_L, V17_M, V17_N, V17_O,
921 V18, V18_H, V18_J, V18_K, V18_L, V18_M, V18_N, V18_O,
922 V19, V19_H, V19_J, V19_K, V19_L, V19_M, V19_N, V19_O,
923 V20, V20_H, V20_J, V20_K, V20_L, V20_M, V20_N, V20_O,
924 V21, V21_H, V21_J, V21_K, V21_L, V21_M, V21_N, V21_O,
925 V22, V22_H, V22_J, V22_K, V22_L, V22_M, V22_N, V22_O,
926 V23, V23_H, V23_J, V23_K, V23_L, V23_M, V23_N, V23_O,
927 V24, V24_H, V24_J, V24_K, V24_L, V24_M, V24_N, V24_O,
928 V25, V25_H, V25_J, V25_K, V25_L, V25_M, V25_N, V25_O,
929 V26, V26_H, V26_J, V26_K, V26_L, V26_M, V26_N, V26_O,
930 V27, V27_H, V27_J, V27_K, V27_L, V27_M, V27_N, V27_O,
931 V28, V28_H, V28_J, V28_K, V28_L, V28_M, V28_N, V28_O,
932 V29, V29_H, V29_J, V29_K, V29_L, V29_M, V29_N, V29_O,
933 V30, V30_H, V30_J, V30_K, V30_L, V30_M, V30_N, V30_O,
934 V31, V31_H, V31_J, V31_K, V31_L, V31_M, V31_N, V31_O,
935 );
936
937 // Class for all 64bit vector registers
938 reg_class vectord_reg(
939 V0, V0_H,
940 V1, V1_H,
941 V2, V2_H,
942 V3, V3_H,
943 V4, V4_H,
944 V5, V5_H,
945 V6, V6_H,
946 V7, V7_H,
947 V8, V8_H,
948 V9, V9_H,
949 V10, V10_H,
950 V11, V11_H,
951 V12, V12_H,
952 V13, V13_H,
953 V14, V14_H,
954 V15, V15_H,
955 V16, V16_H,
956 V17, V17_H,
957 V18, V18_H,
958 V19, V19_H,
959 V20, V20_H,
960 V21, V21_H,
961 V22, V22_H,
962 V23, V23_H,
963 V24, V24_H,
964 V25, V25_H,
965 V26, V26_H,
966 V27, V27_H,
967 V28, V28_H,
968 V29, V29_H,
969 V30, V30_H,
970 V31, V31_H
971 );
972
973 // Class for all 128bit vector registers
974 reg_class vectorx_reg(
975 V0, V0_H, V0_J, V0_K,
976 V1, V1_H, V1_J, V1_K,
977 V2, V2_H, V2_J, V2_K,
978 V3, V3_H, V3_J, V3_K,
979 V4, V4_H, V4_J, V4_K,
980 V5, V5_H, V5_J, V5_K,
981 V6, V6_H, V6_J, V6_K,
982 V7, V7_H, V7_J, V7_K,
983 V8, V8_H, V8_J, V8_K,
984 V9, V9_H, V9_J, V9_K,
985 V10, V10_H, V10_J, V10_K,
986 V11, V11_H, V11_J, V11_K,
987 V12, V12_H, V12_J, V12_K,
988 V13, V13_H, V13_J, V13_K,
989 V14, V14_H, V14_J, V14_K,
990 V15, V15_H, V15_J, V15_K,
991 V16, V16_H, V16_J, V16_K,
992 V17, V17_H, V17_J, V17_K,
993 V18, V18_H, V18_J, V18_K,
994 V19, V19_H, V19_J, V19_K,
995 V20, V20_H, V20_J, V20_K,
996 V21, V21_H, V21_J, V21_K,
997 V22, V22_H, V22_J, V22_K,
998 V23, V23_H, V23_J, V23_K,
999 V24, V24_H, V24_J, V24_K,
1000 V25, V25_H, V25_J, V25_K,
1001 V26, V26_H, V26_J, V26_K,
1002 V27, V27_H, V27_J, V27_K,
1003 V28, V28_H, V28_J, V28_K,
1004 V29, V29_H, V29_J, V29_K,
1005 V30, V30_H, V30_J, V30_K,
1006 V31, V31_H, V31_J, V31_K
1007 );
1008
1009 // Class for 128 bit register v0
1010 reg_class v0_reg(
1011 V0, V0_H
1012 );
1013
1014 // Class for 128 bit register v1
1015 reg_class v1_reg(
1016 V1, V1_H
1017 );
1018
1019 // Class for 128 bit register v2
1020 reg_class v2_reg(
1021 V2, V2_H
1022 );
1023
1024 // Class for 128 bit register v3
1025 reg_class v3_reg(
1026 V3, V3_H
1027 );
1028
1029 // Class for 128 bit register v4
1030 reg_class v4_reg(
1031 V4, V4_H
1032 );
1033
1034 // Class for 128 bit register v5
1035 reg_class v5_reg(
1036 V5, V5_H
1037 );
1038
1039 // Class for 128 bit register v6
1040 reg_class v6_reg(
1041 V6, V6_H
1042 );
1043
1044 // Class for 128 bit register v7
1045 reg_class v7_reg(
1046 V7, V7_H
1047 );
1048
1049 // Class for 128 bit register v8
1050 reg_class v8_reg(
1051 V8, V8_H
1052 );
1053
1054 // Class for 128 bit register v9
1055 reg_class v9_reg(
1056 V9, V9_H
1057 );
1058
1059 // Class for 128 bit register v10
1060 reg_class v10_reg(
1061 V10, V10_H
1062 );
1063
1064 // Class for 128 bit register v11
1065 reg_class v11_reg(
1066 V11, V11_H
1067 );
1068
1069 // Class for 128 bit register v12
1070 reg_class v12_reg(
1071 V12, V12_H
1072 );
1073
1074 // Class for 128 bit register v13
1075 reg_class v13_reg(
1076 V13, V13_H
1077 );
1078
1079 // Class for 128 bit register v14
1080 reg_class v14_reg(
1081 V14, V14_H
1082 );
1083
1084 // Class for 128 bit register v15
1085 reg_class v15_reg(
1086 V15, V15_H
1087 );
1088
1089 // Class for 128 bit register v16
1090 reg_class v16_reg(
1091 V16, V16_H
1092 );
1093
1094 // Class for 128 bit register v17
1095 reg_class v17_reg(
1096 V17, V17_H
1097 );
1098
1099 // Class for 128 bit register v18
1100 reg_class v18_reg(
1101 V18, V18_H
1102 );
1103
1104 // Class for 128 bit register v19
1105 reg_class v19_reg(
1106 V19, V19_H
1107 );
1108
1109 // Class for 128 bit register v20
1110 reg_class v20_reg(
1111 V20, V20_H
1112 );
1113
1114 // Class for 128 bit register v21
1115 reg_class v21_reg(
1116 V21, V21_H
1117 );
1118
1119 // Class for 128 bit register v22
1120 reg_class v22_reg(
1121 V22, V22_H
1122 );
1123
1124 // Class for 128 bit register v23
1125 reg_class v23_reg(
1126 V23, V23_H
1127 );
1128
1129 // Class for 128 bit register v24
1130 reg_class v24_reg(
1131 V24, V24_H
1132 );
1133
1134 // Class for 128 bit register v25
1135 reg_class v25_reg(
1136 V25, V25_H
1137 );
1138
1139 // Class for 128 bit register v26
1140 reg_class v26_reg(
1141 V26, V26_H
1142 );
1143
1144 // Class for 128 bit register v27
1145 reg_class v27_reg(
1146 V27, V27_H
1147 );
1148
1149 // Class for 128 bit register v28
1150 reg_class v28_reg(
1151 V28, V28_H
1152 );
1153
1154 // Class for 128 bit register v29
1155 reg_class v29_reg(
1156 V29, V29_H
1157 );
1158
1159 // Class for 128 bit register v30
1160 reg_class v30_reg(
1161 V30, V30_H
1162 );
1163
1164 // Class for 128 bit register v31
1165 reg_class v31_reg(
1166 V31, V31_H
1167 );
1168
1169 // Class for all SVE predicate registers.
1170 reg_class pr_reg (
1171 P0,
1172 P1,
1173 P2,
1174 P3,
1175 P4,
1176 P5,
1177 P6,
1178 // P7, non-allocatable, preserved with all elements preset to TRUE.
1179 P8,
1180 P9,
1181 P10,
1182 P11,
1183 P12,
1184 P13,
1185 P14,
1186 P15
1187 );
1188
1189 // Class for SVE governing predicate registers, which are used
1190 // to determine the active elements of a predicated instruction.
1191 reg_class gov_pr (
1192 P0,
1193 P1,
1194 P2,
1195 P3,
1196 P4,
1197 P5,
1198 P6,
1199 // P7, non-allocatable, preserved with all elements preset to TRUE.
1200 );
1201
1202 // Singleton class for condition codes
1203 reg_class int_flags(RFLAGS);
1204
1205 %}
1206
1207 //----------DEFINITION BLOCK---------------------------------------------------
1208 // Define name --> value mappings to inform the ADLC of an integer valued name
1209 // Current support includes integer values in the range [0, 0x7FFFFFFF]
1210 // Format:
1211 // int_def <name> ( <int_value>, <expression>);
1212 // Generated Code in ad_<arch>.hpp
1213 // #define <name> (<expression>)
1214 // // value == <int_value>
1215 // Generated code in ad_<arch>.cpp adlc_verification()
1216 // assert( <name> == <int_value>, "Expect (<expression>) to equal <int_value>");
1217 //
1218
1219 // we follow the ppc-aix port in using a simple cost model which ranks
1220 // register operations as cheap, memory ops as more expensive and
1221 // branches as most expensive. the first two have a low as well as a
1222 // normal cost. huge cost appears to be a way of saying don't do
1223 // something
1224
1225 definitions %{
1226 // The default cost (of a register move instruction).
1227 int_def INSN_COST ( 100, 100);
1228 int_def BRANCH_COST ( 200, 2 * INSN_COST);
1229 int_def CALL_COST ( 200, 2 * INSN_COST);
1230 int_def VOLATILE_REF_COST ( 1000, 10 * INSN_COST);
1231 %}
1232
1233
1234 //----------SOURCE BLOCK-------------------------------------------------------
1235 // This is a block of C++ code which provides values, functions, and
1236 // definitions necessary in the rest of the architecture description
1237
1238 source_hpp %{
1239
1240 #include "asm/macroAssembler.hpp"
1241 #include "gc/shared/barrierSetAssembler.hpp"
1242 #include "gc/shared/cardTable.hpp"
1243 #include "gc/shared/cardTableBarrierSet.hpp"
1244 #include "gc/shared/collectedHeap.hpp"
1245 #include "opto/addnode.hpp"
1246 #include "opto/convertnode.hpp"
1247 #include "runtime/objectMonitor.hpp"
1248
1249 extern RegMask _ANY_REG32_mask;
1250 extern RegMask _ANY_REG_mask;
1251 extern RegMask _PTR_REG_mask;
1252 extern RegMask _NO_SPECIAL_REG32_mask;
1253 extern RegMask _NO_SPECIAL_REG_mask;
1254 extern RegMask _NO_SPECIAL_PTR_REG_mask;
1255
1256 class CallStubImpl {
1257
1258 //--------------------------------------------------------------
1259 //---< Used for optimization in Compile::shorten_branches >---
1260 //--------------------------------------------------------------
1261
1262 public:
1263 // Size of call trampoline stub.
1264 static uint size_call_trampoline() {
1265 return 0; // no call trampolines on this platform
1266 }
1267
1268 // number of relocations needed by a call trampoline stub
1269 static uint reloc_call_trampoline() {
1270 return 0; // no call trampolines on this platform
1271 }
1272 };
1273
1274 class HandlerImpl {
1275
1276 public:
1277
1278 static int emit_exception_handler(CodeBuffer &cbuf);
1279 static int emit_deopt_handler(CodeBuffer& cbuf);
1280
1281 static uint size_exception_handler() {
1282 return MacroAssembler::far_codestub_branch_size();
1283 }
1284
1285 static uint size_deopt_handler() {
1286 // count one adr and one far branch instruction
1287 return NativeInstruction::instruction_size + MacroAssembler::far_codestub_branch_size();
1288 }
1289 };
1290
1291 class Node::PD {
1292 public:
1293 enum NodeFlags {
1294 _last_flag = Node::_last_flag
1295 };
1296 };
1297
1298 bool is_CAS(int opcode, bool maybe_volatile);
1299
1300 // predicates controlling emit of ldr<x>/ldar<x> and associated dmb
1301
1302 bool unnecessary_acquire(const Node *barrier);
1303 bool needs_acquiring_load(const Node *load);
1304
1305 // predicates controlling emit of str<x>/stlr<x> and associated dmbs
1306
1307 bool unnecessary_release(const Node *barrier);
1308 bool unnecessary_volatile(const Node *barrier);
1309 bool needs_releasing_store(const Node *store);
1310
1311 // predicate controlling translation of CompareAndSwapX
1312 bool needs_acquiring_load_exclusive(const Node *load);
1313
1314 // predicate controlling addressing modes
1315 bool size_fits_all_mem_uses(AddPNode* addp, int shift);
1316 %}
1317
1318 source %{
1319
1320 // Derived RegMask with conditionally allocatable registers
1321
1322 void PhaseOutput::pd_perform_mach_node_analysis() {
1323 }
1324
1325 int MachNode::pd_alignment_required() const {
1326 return 1;
1327 }
1328
1329 int MachNode::compute_padding(int current_offset) const {
1330 return 0;
1331 }
1332
1333 RegMask _ANY_REG32_mask;
1334 RegMask _ANY_REG_mask;
1335 RegMask _PTR_REG_mask;
1336 RegMask _NO_SPECIAL_REG32_mask;
1337 RegMask _NO_SPECIAL_REG_mask;
1338 RegMask _NO_SPECIAL_PTR_REG_mask;
1339
1340 void reg_mask_init() {
1341 // We derive below RegMask(s) from the ones which are auto-generated from
1342 // adlc register classes to make AArch64 rheapbase (r27) and rfp (r29)
1343 // registers conditionally reserved.
1344
1345 _ANY_REG32_mask = _ALL_REG32_mask;
1346 _ANY_REG32_mask.Remove(OptoReg::as_OptoReg(r31_sp->as_VMReg()));
1347
1348 _ANY_REG_mask = _ALL_REG_mask;
1349
1350 _PTR_REG_mask = _ALL_REG_mask;
1351
1352 _NO_SPECIAL_REG32_mask = _ALL_REG32_mask;
1353 _NO_SPECIAL_REG32_mask.SUBTRACT(_NON_ALLOCATABLE_REG32_mask);
1354
1355 _NO_SPECIAL_REG_mask = _ALL_REG_mask;
1356 _NO_SPECIAL_REG_mask.SUBTRACT(_NON_ALLOCATABLE_REG_mask);
1357
1358 _NO_SPECIAL_PTR_REG_mask = _ALL_REG_mask;
1359 _NO_SPECIAL_PTR_REG_mask.SUBTRACT(_NON_ALLOCATABLE_REG_mask);
1360
1361 // r27 is not allocatable when compressed oops is on and heapbase is not
1362 // zero, compressed klass pointers doesn't use r27 after JDK-8234794
1363 if (UseCompressedOops && (CompressedOops::ptrs_base() != NULL)) {
1364 _NO_SPECIAL_REG32_mask.Remove(OptoReg::as_OptoReg(r27->as_VMReg()));
1365 _NO_SPECIAL_REG_mask.SUBTRACT(_HEAPBASE_REG_mask);
1366 _NO_SPECIAL_PTR_REG_mask.SUBTRACT(_HEAPBASE_REG_mask);
1367 }
1368
1369 // r29 is not allocatable when PreserveFramePointer is on
1370 if (PreserveFramePointer) {
1371 _NO_SPECIAL_REG32_mask.Remove(OptoReg::as_OptoReg(r29->as_VMReg()));
1372 _NO_SPECIAL_REG_mask.SUBTRACT(_FP_REG_mask);
1373 _NO_SPECIAL_PTR_REG_mask.SUBTRACT(_FP_REG_mask);
1374 }
1375 }
1376
1377 // Optimizaton of volatile gets and puts
1378 // -------------------------------------
1379 //
1380 // AArch64 has ldar<x> and stlr<x> instructions which we can safely
1381 // use to implement volatile reads and writes. For a volatile read
1382 // we simply need
1383 //
1384 // ldar<x>
1385 //
1386 // and for a volatile write we need
1387 //
1388 // stlr<x>
1389 //
1390 // Alternatively, we can implement them by pairing a normal
1391 // load/store with a memory barrier. For a volatile read we need
1392 //
1393 // ldr<x>
1394 // dmb ishld
1395 //
1396 // for a volatile write
1397 //
1398 // dmb ish
1399 // str<x>
1400 // dmb ish
1401 //
1402 // We can also use ldaxr and stlxr to implement compare and swap CAS
1403 // sequences. These are normally translated to an instruction
1404 // sequence like the following
1405 //
1406 // dmb ish
1407 // retry:
1408 // ldxr<x> rval raddr
1409 // cmp rval rold
1410 // b.ne done
1411 // stlxr<x> rval, rnew, rold
1412 // cbnz rval retry
1413 // done:
1414 // cset r0, eq
1415 // dmb ishld
1416 //
1417 // Note that the exclusive store is already using an stlxr
1418 // instruction. That is required to ensure visibility to other
1419 // threads of the exclusive write (assuming it succeeds) before that
1420 // of any subsequent writes.
1421 //
1422 // The following instruction sequence is an improvement on the above
1423 //
1424 // retry:
1425 // ldaxr<x> rval raddr
1426 // cmp rval rold
1427 // b.ne done
1428 // stlxr<x> rval, rnew, rold
1429 // cbnz rval retry
1430 // done:
1431 // cset r0, eq
1432 //
1433 // We don't need the leading dmb ish since the stlxr guarantees
1434 // visibility of prior writes in the case that the swap is
1435 // successful. Crucially we don't have to worry about the case where
1436 // the swap is not successful since no valid program should be
1437 // relying on visibility of prior changes by the attempting thread
1438 // in the case where the CAS fails.
1439 //
1440 // Similarly, we don't need the trailing dmb ishld if we substitute
1441 // an ldaxr instruction since that will provide all the guarantees we
1442 // require regarding observation of changes made by other threads
1443 // before any change to the CAS address observed by the load.
1444 //
1445 // In order to generate the desired instruction sequence we need to
1446 // be able to identify specific 'signature' ideal graph node
1447 // sequences which i) occur as a translation of a volatile reads or
1448 // writes or CAS operations and ii) do not occur through any other
1449 // translation or graph transformation. We can then provide
1450 // alternative aldc matching rules which translate these node
1451 // sequences to the desired machine code sequences. Selection of the
1452 // alternative rules can be implemented by predicates which identify
1453 // the relevant node sequences.
1454 //
1455 // The ideal graph generator translates a volatile read to the node
1456 // sequence
1457 //
1458 // LoadX[mo_acquire]
1459 // MemBarAcquire
1460 //
1461 // As a special case when using the compressed oops optimization we
1462 // may also see this variant
1463 //
1464 // LoadN[mo_acquire]
1465 // DecodeN
1466 // MemBarAcquire
1467 //
1468 // A volatile write is translated to the node sequence
1469 //
1470 // MemBarRelease
1471 // StoreX[mo_release] {CardMark}-optional
1472 // MemBarVolatile
1473 //
1474 // n.b. the above node patterns are generated with a strict
1475 // 'signature' configuration of input and output dependencies (see
1476 // the predicates below for exact details). The card mark may be as
1477 // simple as a few extra nodes or, in a few GC configurations, may
1478 // include more complex control flow between the leading and
1479 // trailing memory barriers. However, whatever the card mark
1480 // configuration these signatures are unique to translated volatile
1481 // reads/stores -- they will not appear as a result of any other
1482 // bytecode translation or inlining nor as a consequence of
1483 // optimizing transforms.
1484 //
1485 // We also want to catch inlined unsafe volatile gets and puts and
1486 // be able to implement them using either ldar<x>/stlr<x> or some
1487 // combination of ldr<x>/stlr<x> and dmb instructions.
1488 //
1489 // Inlined unsafe volatiles puts manifest as a minor variant of the
1490 // normal volatile put node sequence containing an extra cpuorder
1491 // membar
1492 //
1493 // MemBarRelease
1494 // MemBarCPUOrder
1495 // StoreX[mo_release] {CardMark}-optional
1496 // MemBarCPUOrder
1497 // MemBarVolatile
1498 //
1499 // n.b. as an aside, a cpuorder membar is not itself subject to
1500 // matching and translation by adlc rules. However, the rule
1501 // predicates need to detect its presence in order to correctly
1502 // select the desired adlc rules.
1503 //
1504 // Inlined unsafe volatile gets manifest as a slightly different
1505 // node sequence to a normal volatile get because of the
1506 // introduction of some CPUOrder memory barriers to bracket the
1507 // Load. However, but the same basic skeleton of a LoadX feeding a
1508 // MemBarAcquire, possibly thorugh an optional DecodeN, is still
1509 // present
1510 //
1511 // MemBarCPUOrder
1512 // || \\
1513 // MemBarCPUOrder LoadX[mo_acquire]
1514 // || |
1515 // || {DecodeN} optional
1516 // || /
1517 // MemBarAcquire
1518 //
1519 // In this case the acquire membar does not directly depend on the
1520 // load. However, we can be sure that the load is generated from an
1521 // inlined unsafe volatile get if we see it dependent on this unique
1522 // sequence of membar nodes. Similarly, given an acquire membar we
1523 // can know that it was added because of an inlined unsafe volatile
1524 // get if it is fed and feeds a cpuorder membar and if its feed
1525 // membar also feeds an acquiring load.
1526 //
1527 // Finally an inlined (Unsafe) CAS operation is translated to the
1528 // following ideal graph
1529 //
1530 // MemBarRelease
1531 // MemBarCPUOrder
1532 // CompareAndSwapX {CardMark}-optional
1533 // MemBarCPUOrder
1534 // MemBarAcquire
1535 //
1536 // So, where we can identify these volatile read and write
1537 // signatures we can choose to plant either of the above two code
1538 // sequences. For a volatile read we can simply plant a normal
1539 // ldr<x> and translate the MemBarAcquire to a dmb. However, we can
1540 // also choose to inhibit translation of the MemBarAcquire and
1541 // inhibit planting of the ldr<x>, instead planting an ldar<x>.
1542 //
1543 // When we recognise a volatile store signature we can choose to
1544 // plant at a dmb ish as a translation for the MemBarRelease, a
1545 // normal str<x> and then a dmb ish for the MemBarVolatile.
1546 // Alternatively, we can inhibit translation of the MemBarRelease
1547 // and MemBarVolatile and instead plant a simple stlr<x>
1548 // instruction.
1549 //
1550 // when we recognise a CAS signature we can choose to plant a dmb
1551 // ish as a translation for the MemBarRelease, the conventional
1552 // macro-instruction sequence for the CompareAndSwap node (which
1553 // uses ldxr<x>) and then a dmb ishld for the MemBarAcquire.
1554 // Alternatively, we can elide generation of the dmb instructions
1555 // and plant the alternative CompareAndSwap macro-instruction
1556 // sequence (which uses ldaxr<x>).
1557 //
1558 // Of course, the above only applies when we see these signature
1559 // configurations. We still want to plant dmb instructions in any
1560 // other cases where we may see a MemBarAcquire, MemBarRelease or
1561 // MemBarVolatile. For example, at the end of a constructor which
1562 // writes final/volatile fields we will see a MemBarRelease
1563 // instruction and this needs a 'dmb ish' lest we risk the
1564 // constructed object being visible without making the
1565 // final/volatile field writes visible.
1566 //
1567 // n.b. the translation rules below which rely on detection of the
1568 // volatile signatures and insert ldar<x> or stlr<x> are failsafe.
1569 // If we see anything other than the signature configurations we
1570 // always just translate the loads and stores to ldr<x> and str<x>
1571 // and translate acquire, release and volatile membars to the
1572 // relevant dmb instructions.
1573 //
1574
1575 // is_CAS(int opcode, bool maybe_volatile)
1576 //
1577 // return true if opcode is one of the possible CompareAndSwapX
1578 // values otherwise false.
1579
1580 bool is_CAS(int opcode, bool maybe_volatile)
1581 {
1582 switch(opcode) {
1583 // We handle these
1584 case Op_CompareAndSwapI:
1585 case Op_CompareAndSwapL:
1586 case Op_CompareAndSwapP:
1587 case Op_CompareAndSwapN:
1588 case Op_ShenandoahCompareAndSwapP:
1589 case Op_ShenandoahCompareAndSwapN:
1590 case Op_CompareAndSwapB:
1591 case Op_CompareAndSwapS:
1592 case Op_GetAndSetI:
1593 case Op_GetAndSetL:
1594 case Op_GetAndSetP:
1595 case Op_GetAndSetN:
1596 case Op_GetAndAddI:
1597 case Op_GetAndAddL:
1598 return true;
1599 case Op_CompareAndExchangeI:
1600 case Op_CompareAndExchangeN:
1601 case Op_CompareAndExchangeB:
1602 case Op_CompareAndExchangeS:
1603 case Op_CompareAndExchangeL:
1604 case Op_CompareAndExchangeP:
1605 case Op_WeakCompareAndSwapB:
1606 case Op_WeakCompareAndSwapS:
1607 case Op_WeakCompareAndSwapI:
1608 case Op_WeakCompareAndSwapL:
1609 case Op_WeakCompareAndSwapP:
1610 case Op_WeakCompareAndSwapN:
1611 case Op_ShenandoahWeakCompareAndSwapP:
1612 case Op_ShenandoahWeakCompareAndSwapN:
1613 case Op_ShenandoahCompareAndExchangeP:
1614 case Op_ShenandoahCompareAndExchangeN:
1615 return maybe_volatile;
1616 default:
1617 return false;
1618 }
1619 }
1620
1621 // helper to determine the maximum number of Phi nodes we may need to
1622 // traverse when searching from a card mark membar for the merge mem
1623 // feeding a trailing membar or vice versa
1624
1625 // predicates controlling emit of ldr<x>/ldar<x>
1626
1627 bool unnecessary_acquire(const Node *barrier)
1628 {
1629 assert(barrier->is_MemBar(), "expecting a membar");
1630
1631 MemBarNode* mb = barrier->as_MemBar();
1632
1633 if (mb->trailing_load()) {
1634 return true;
1635 }
1636
1637 if (mb->trailing_load_store()) {
1638 Node* load_store = mb->in(MemBarNode::Precedent);
1639 assert(load_store->is_LoadStore(), "unexpected graph shape");
1640 return is_CAS(load_store->Opcode(), true);
1641 }
1642
1643 return false;
1644 }
1645
1646 bool needs_acquiring_load(const Node *n)
1647 {
1648 assert(n->is_Load(), "expecting a load");
1649 LoadNode *ld = n->as_Load();
1650 return ld->is_acquire();
1651 }
1652
1653 bool unnecessary_release(const Node *n)
1654 {
1655 assert((n->is_MemBar() &&
1656 n->Opcode() == Op_MemBarRelease),
1657 "expecting a release membar");
1658
1659 MemBarNode *barrier = n->as_MemBar();
1660 if (!barrier->leading()) {
1661 return false;
1662 } else {
1663 Node* trailing = barrier->trailing_membar();
1664 MemBarNode* trailing_mb = trailing->as_MemBar();
1665 assert(trailing_mb->trailing(), "Not a trailing membar?");
1666 assert(trailing_mb->leading_membar() == n, "inconsistent leading/trailing membars");
1667
1668 Node* mem = trailing_mb->in(MemBarNode::Precedent);
1669 if (mem->is_Store()) {
1670 assert(mem->as_Store()->is_release(), "");
1671 assert(trailing_mb->Opcode() == Op_MemBarVolatile, "");
1672 return true;
1673 } else {
1674 assert(mem->is_LoadStore(), "");
1675 assert(trailing_mb->Opcode() == Op_MemBarAcquire, "");
1676 return is_CAS(mem->Opcode(), true);
1677 }
1678 }
1679 return false;
1680 }
1681
1682 bool unnecessary_volatile(const Node *n)
1683 {
1684 // assert n->is_MemBar();
1685 MemBarNode *mbvol = n->as_MemBar();
1686
1687 bool release = mbvol->trailing_store();
1688 assert(!release || (mbvol->in(MemBarNode::Precedent)->is_Store() && mbvol->in(MemBarNode::Precedent)->as_Store()->is_release()), "");
1689 #ifdef ASSERT
1690 if (release) {
1691 Node* leading = mbvol->leading_membar();
1692 assert(leading->Opcode() == Op_MemBarRelease, "");
1693 assert(leading->as_MemBar()->leading_store(), "");
1694 assert(leading->as_MemBar()->trailing_membar() == mbvol, "");
1695 }
1696 #endif
1697
1698 return release;
1699 }
1700
1701 // predicates controlling emit of str<x>/stlr<x>
1702
1703 bool needs_releasing_store(const Node *n)
1704 {
1705 // assert n->is_Store();
1706 StoreNode *st = n->as_Store();
1707 return st->trailing_membar() != NULL;
1708 }
1709
1710 // predicate controlling translation of CAS
1711 //
1712 // returns true if CAS needs to use an acquiring load otherwise false
1713
1714 bool needs_acquiring_load_exclusive(const Node *n)
1715 {
1716 assert(is_CAS(n->Opcode(), true), "expecting a compare and swap");
1717 LoadStoreNode* ldst = n->as_LoadStore();
1718 if (is_CAS(n->Opcode(), false)) {
1719 assert(ldst->trailing_membar() != NULL, "expected trailing membar");
1720 } else {
1721 return ldst->trailing_membar() != NULL;
1722 }
1723
1724 // so we can just return true here
1725 return true;
1726 }
1727
1728 #define __ _masm.
1729
1730 // advance declarations for helper functions to convert register
1731 // indices to register objects
1732
1733 // the ad file has to provide implementations of certain methods
1734 // expected by the generic code
1735 //
1736 // REQUIRED FUNCTIONALITY
1737
1738 //=============================================================================
1739
1740 // !!!!! Special hack to get all types of calls to specify the byte offset
1741 // from the start of the call to the point where the return address
1742 // will point.
1743
1744 int MachCallStaticJavaNode::ret_addr_offset()
1745 {
1746 // call should be a simple bl
1747 int off = 4;
1748 return off;
1749 }
1750
1751 int MachCallDynamicJavaNode::ret_addr_offset()
1752 {
1753 return 16; // movz, movk, movk, bl
1754 }
1755
1756 int MachCallRuntimeNode::ret_addr_offset() {
1757 // for generated stubs the call will be
1758 // bl(addr)
1759 // or with far branches
1760 // bl(trampoline_stub)
1761 // for real runtime callouts it will be six instructions
1762 // see aarch64_enc_java_to_runtime
1763 // adr(rscratch2, retaddr)
1764 // lea(rscratch1, RuntimeAddress(addr)
1765 // stp(zr, rscratch2, Address(__ pre(sp, -2 * wordSize)))
1766 // blr(rscratch1)
1767 CodeBlob *cb = CodeCache::find_blob(_entry_point);
1768 if (cb) {
1769 return 1 * NativeInstruction::instruction_size;
1770 } else {
1771 return 6 * NativeInstruction::instruction_size;
1772 }
1773 }
1774
1775 int MachCallNativeNode::ret_addr_offset() {
1776 // This is implemented using aarch64_enc_java_to_runtime as above.
1777 CodeBlob *cb = CodeCache::find_blob(_entry_point);
1778 if (cb) {
1779 return 1 * NativeInstruction::instruction_size;
1780 } else {
1781 return 6 * NativeInstruction::instruction_size;
1782 }
1783 }
1784
1785 //=============================================================================
1786
1787 #ifndef PRODUCT
1788 void MachBreakpointNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
1789 st->print("BREAKPOINT");
1790 }
1791 #endif
1792
1793 void MachBreakpointNode::emit(CodeBuffer &cbuf, PhaseRegAlloc *ra_) const {
1794 C2_MacroAssembler _masm(&cbuf);
1795 __ brk(0);
1796 }
1797
1798 uint MachBreakpointNode::size(PhaseRegAlloc *ra_) const {
1799 return MachNode::size(ra_);
1800 }
1801
1802 //=============================================================================
1803
1804 #ifndef PRODUCT
1805 void MachNopNode::format(PhaseRegAlloc*, outputStream* st) const {
1806 st->print("nop \t# %d bytes pad for loops and calls", _count);
1807 }
1808 #endif
1809
1810 void MachNopNode::emit(CodeBuffer &cbuf, PhaseRegAlloc*) const {
1811 C2_MacroAssembler _masm(&cbuf);
1812 for (int i = 0; i < _count; i++) {
1813 __ nop();
1814 }
1815 }
1816
1817 uint MachNopNode::size(PhaseRegAlloc*) const {
1818 return _count * NativeInstruction::instruction_size;
1819 }
1820
1821 //=============================================================================
1822 const RegMask& MachConstantBaseNode::_out_RegMask = RegMask::Empty;
1823
1824 int ConstantTable::calculate_table_base_offset() const {
1825 return 0; // absolute addressing, no offset
1826 }
1827
1828 bool MachConstantBaseNode::requires_postalloc_expand() const { return false; }
1829 void MachConstantBaseNode::postalloc_expand(GrowableArray <Node *> *nodes, PhaseRegAlloc *ra_) {
1830 ShouldNotReachHere();
1831 }
1832
1833 void MachConstantBaseNode::emit(CodeBuffer& cbuf, PhaseRegAlloc* ra_) const {
1834 // Empty encoding
1835 }
1836
1837 uint MachConstantBaseNode::size(PhaseRegAlloc* ra_) const {
1838 return 0;
1839 }
1840
1841 #ifndef PRODUCT
1842 void MachConstantBaseNode::format(PhaseRegAlloc* ra_, outputStream* st) const {
1843 st->print("-- \t// MachConstantBaseNode (empty encoding)");
1844 }
1845 #endif
1846
1847 #ifndef PRODUCT
1848 void MachPrologNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
1849 Compile* C = ra_->C;
1850
1851 int framesize = C->output()->frame_slots() << LogBytesPerInt;
1852
1853 if (C->output()->need_stack_bang(framesize))
1854 st->print("# stack bang size=%d\n\t", framesize);
1855
1856 if (framesize < ((1 << 9) + 2 * wordSize)) {
1857 st->print("sub sp, sp, #%d\n\t", framesize);
1858 st->print("stp rfp, lr, [sp, #%d]", framesize - 2 * wordSize);
1859 if (PreserveFramePointer) st->print("\n\tadd rfp, sp, #%d", framesize - 2 * wordSize);
1860 } else {
1861 st->print("stp lr, rfp, [sp, #%d]!\n\t", -(2 * wordSize));
1862 if (PreserveFramePointer) st->print("mov rfp, sp\n\t");
1863 st->print("mov rscratch1, #%d\n\t", framesize - 2 * wordSize);
1864 st->print("sub sp, sp, rscratch1");
1865 }
1866 if (C->stub_function() == NULL && BarrierSet::barrier_set()->barrier_set_nmethod() != NULL) {
1867 st->print("\n\t");
1868 st->print("ldr rscratch1, [guard]\n\t");
1869 st->print("dmb ishld\n\t");
1870 st->print("ldr rscratch2, [rthread, #thread_disarmed_offset]\n\t");
1871 st->print("cmp rscratch1, rscratch2\n\t");
1872 st->print("b.eq skip");
1873 st->print("\n\t");
1874 st->print("blr #nmethod_entry_barrier_stub\n\t");
1875 st->print("b skip\n\t");
1876 st->print("guard: int\n\t");
1877 st->print("\n\t");
1878 st->print("skip:\n\t");
1879 }
1880 }
1881 #endif
1882
1883 void MachPrologNode::emit(CodeBuffer &cbuf, PhaseRegAlloc *ra_) const {
1884 Compile* C = ra_->C;
1885 C2_MacroAssembler _masm(&cbuf);
1886
1887 // n.b. frame size includes space for return pc and rfp
1888 const int framesize = C->output()->frame_size_in_bytes();
1889
1890 // insert a nop at the start of the prolog so we can patch in a
1891 // branch if we need to invalidate the method later
1892 __ nop();
1893
1894 if (C->clinit_barrier_on_entry()) {
1895 assert(!C->method()->holder()->is_not_initialized(), "initialization should have been started");
1896
1897 Label L_skip_barrier;
1898
1899 __ mov_metadata(rscratch2, C->method()->holder()->constant_encoding());
1900 __ clinit_barrier(rscratch2, rscratch1, &L_skip_barrier);
1901 __ far_jump(RuntimeAddress(SharedRuntime::get_handle_wrong_method_stub()));
1902 __ bind(L_skip_barrier);
1903 }
1904
1905 if (C->max_vector_size() >= 16) {
1906 __ reinitialize_ptrue();
1907 }
1908
1909 int bangsize = C->output()->bang_size_in_bytes();
1910 if (C->output()->need_stack_bang(bangsize))
1911 __ generate_stack_overflow_check(bangsize);
1912
1913 __ build_frame(framesize);
1914
1915 if (C->stub_function() == NULL) {
1916 BarrierSetAssembler* bs = BarrierSet::barrier_set()->barrier_set_assembler();
1917 bs->nmethod_entry_barrier(&_masm);
1918 }
1919
1920 if (VerifyStackAtCalls) {
1921 Unimplemented();
1922 }
1923
1924 C->output()->set_frame_complete(cbuf.insts_size());
1925
1926 if (C->has_mach_constant_base_node()) {
1927 // NOTE: We set the table base offset here because users might be
1928 // emitted before MachConstantBaseNode.
1929 ConstantTable& constant_table = C->output()->constant_table();
1930 constant_table.set_table_base_offset(constant_table.calculate_table_base_offset());
1931 }
1932 }
1933
1934 uint MachPrologNode::size(PhaseRegAlloc* ra_) const
1935 {
1936 return MachNode::size(ra_); // too many variables; just compute it
1937 // the hard way
1938 }
1939
1940 int MachPrologNode::reloc() const
1941 {
1942 return 0;
1943 }
1944
1945 //=============================================================================
1946
1947 #ifndef PRODUCT
1948 void MachEpilogNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
1949 Compile* C = ra_->C;
1950 int framesize = C->output()->frame_slots() << LogBytesPerInt;
1951
1952 st->print("# pop frame %d\n\t",framesize);
1953
1954 if (framesize == 0) {
1955 st->print("ldp lr, rfp, [sp],#%d\n\t", (2 * wordSize));
1956 } else if (framesize < ((1 << 9) + 2 * wordSize)) {
1957 st->print("ldp lr, rfp, [sp,#%d]\n\t", framesize - 2 * wordSize);
1958 st->print("add sp, sp, #%d\n\t", framesize);
1959 } else {
1960 st->print("mov rscratch1, #%d\n\t", framesize - 2 * wordSize);
1961 st->print("add sp, sp, rscratch1\n\t");
1962 st->print("ldp lr, rfp, [sp],#%d\n\t", (2 * wordSize));
1963 }
1964
1965 if (do_polling() && C->is_method_compilation()) {
1966 st->print("# test polling word\n\t");
1967 st->print("ldr rscratch1, [rthread],#%d\n\t", in_bytes(JavaThread::polling_word_offset()));
1968 st->print("cmp sp, rscratch1\n\t");
1969 st->print("bhi #slow_path");
1970 }
1971 }
1972 #endif
1973
1974 void MachEpilogNode::emit(CodeBuffer &cbuf, PhaseRegAlloc *ra_) const {
1975 Compile* C = ra_->C;
1976 C2_MacroAssembler _masm(&cbuf);
1977 int framesize = C->output()->frame_slots() << LogBytesPerInt;
1978
1979 __ remove_frame(framesize);
1980
1981 if (StackReservedPages > 0 && C->has_reserved_stack_access()) {
1982 __ reserved_stack_check();
1983 }
1984
1985 if (do_polling() && C->is_method_compilation()) {
1986 Label dummy_label;
1987 Label* code_stub = &dummy_label;
1988 if (!C->output()->in_scratch_emit_size()) {
1989 C2SafepointPollStub* stub = new (C->comp_arena()) C2SafepointPollStub(__ offset());
1990 C->output()->add_stub(stub);
1991 code_stub = &stub->entry();
1992 }
1993 __ relocate(relocInfo::poll_return_type);
1994 __ safepoint_poll(*code_stub, true /* at_return */, false /* acquire */, true /* in_nmethod */);
1995 }
1996 }
1997
1998 uint MachEpilogNode::size(PhaseRegAlloc *ra_) const {
1999 // Variable size. Determine dynamically.
2000 return MachNode::size(ra_);
2001 }
2002
2003 int MachEpilogNode::reloc() const {
2004 // Return number of relocatable values contained in this instruction.
2005 return 1; // 1 for polling page.
2006 }
2007
2008 const Pipeline * MachEpilogNode::pipeline() const {
2009 return MachNode::pipeline_class();
2010 }
2011
2012 //=============================================================================
2013
2014 // Figure out which register class each belongs in: rc_int, rc_float or
2015 // rc_stack.
2016 enum RC { rc_bad, rc_int, rc_float, rc_predicate, rc_stack };
2017
2018 static enum RC rc_class(OptoReg::Name reg) {
2019
2020 if (reg == OptoReg::Bad) {
2021 return rc_bad;
2022 }
2023
2024 // we have 32 int registers * 2 halves
2025 int slots_of_int_registers = RegisterImpl::max_slots_per_register * RegisterImpl::number_of_registers;
2026
2027 if (reg < slots_of_int_registers) {
2028 return rc_int;
2029 }
2030
2031 // we have 32 float register * 8 halves
2032 int slots_of_float_registers = FloatRegisterImpl::max_slots_per_register * FloatRegisterImpl::number_of_registers;
2033 if (reg < slots_of_int_registers + slots_of_float_registers) {
2034 return rc_float;
2035 }
2036
2037 int slots_of_predicate_registers = PRegisterImpl::max_slots_per_register * PRegisterImpl::number_of_registers;
2038 if (reg < slots_of_int_registers + slots_of_float_registers + slots_of_predicate_registers) {
2039 return rc_predicate;
2040 }
2041
2042 // Between predicate regs & stack is the flags.
2043 assert(OptoReg::is_stack(reg), "blow up if spilling flags");
2044
2045 return rc_stack;
2046 }
2047
2048 uint MachSpillCopyNode::implementation(CodeBuffer *cbuf, PhaseRegAlloc *ra_, bool do_size, outputStream *st) const {
2049 Compile* C = ra_->C;
2050
2051 // Get registers to move.
2052 OptoReg::Name src_hi = ra_->get_reg_second(in(1));
2053 OptoReg::Name src_lo = ra_->get_reg_first(in(1));
2054 OptoReg::Name dst_hi = ra_->get_reg_second(this);
2055 OptoReg::Name dst_lo = ra_->get_reg_first(this);
2056
2057 enum RC src_hi_rc = rc_class(src_hi);
2058 enum RC src_lo_rc = rc_class(src_lo);
2059 enum RC dst_hi_rc = rc_class(dst_hi);
2060 enum RC dst_lo_rc = rc_class(dst_lo);
2061
2062 assert(src_lo != OptoReg::Bad && dst_lo != OptoReg::Bad, "must move at least 1 register");
2063
2064 if (src_hi != OptoReg::Bad) {
2065 assert((src_lo&1)==0 && src_lo+1==src_hi &&
2066 (dst_lo&1)==0 && dst_lo+1==dst_hi,
2067 "expected aligned-adjacent pairs");
2068 }
2069
2070 if (src_lo == dst_lo && src_hi == dst_hi) {
2071 return 0; // Self copy, no move.
2072 }
2073
2074 bool is64 = (src_lo & 1) == 0 && src_lo + 1 == src_hi &&
2075 (dst_lo & 1) == 0 && dst_lo + 1 == dst_hi;
2076 int src_offset = ra_->reg2offset(src_lo);
2077 int dst_offset = ra_->reg2offset(dst_lo);
2078
2079 if (bottom_type()->isa_vect() != NULL) {
2080 uint ireg = ideal_reg();
2081 if (ireg == Op_VecA && cbuf) {
2082 C2_MacroAssembler _masm(cbuf);
2083 int sve_vector_reg_size_in_bytes = Matcher::scalable_vector_reg_size(T_BYTE);
2084 if (src_lo_rc == rc_stack && dst_lo_rc == rc_stack) {
2085 // stack->stack
2086 __ spill_copy_sve_vector_stack_to_stack(src_offset, dst_offset,
2087 sve_vector_reg_size_in_bytes);
2088 } else if (src_lo_rc == rc_float && dst_lo_rc == rc_stack) {
2089 __ spill_sve_vector(as_FloatRegister(Matcher::_regEncode[src_lo]), ra_->reg2offset(dst_lo),
2090 sve_vector_reg_size_in_bytes);
2091 } else if (src_lo_rc == rc_stack && dst_lo_rc == rc_float) {
2092 __ unspill_sve_vector(as_FloatRegister(Matcher::_regEncode[dst_lo]), ra_->reg2offset(src_lo),
2093 sve_vector_reg_size_in_bytes);
2094 } else if (src_lo_rc == rc_float && dst_lo_rc == rc_float) {
2095 __ sve_orr(as_FloatRegister(Matcher::_regEncode[dst_lo]),
2096 as_FloatRegister(Matcher::_regEncode[src_lo]),
2097 as_FloatRegister(Matcher::_regEncode[src_lo]));
2098 } else {
2099 ShouldNotReachHere();
2100 }
2101 } else if (cbuf) {
2102 assert(ireg == Op_VecD || ireg == Op_VecX, "must be 64 bit or 128 bit vector");
2103 C2_MacroAssembler _masm(cbuf);
2104 assert((src_lo_rc != rc_int && dst_lo_rc != rc_int), "sanity");
2105 if (src_lo_rc == rc_stack && dst_lo_rc == rc_stack) {
2106 // stack->stack
2107 assert((src_offset & 7) == 0 && (dst_offset & 7) == 0, "unaligned stack offset");
2108 if (ireg == Op_VecD) {
2109 __ unspill(rscratch1, true, src_offset);
2110 __ spill(rscratch1, true, dst_offset);
2111 } else {
2112 __ spill_copy128(src_offset, dst_offset);
2113 }
2114 } else if (src_lo_rc == rc_float && dst_lo_rc == rc_float) {
2115 __ mov(as_FloatRegister(Matcher::_regEncode[dst_lo]),
2116 ireg == Op_VecD ? __ T8B : __ T16B,
2117 as_FloatRegister(Matcher::_regEncode[src_lo]));
2118 } else if (src_lo_rc == rc_float && dst_lo_rc == rc_stack) {
2119 __ spill(as_FloatRegister(Matcher::_regEncode[src_lo]),
2120 ireg == Op_VecD ? __ D : __ Q,
2121 ra_->reg2offset(dst_lo));
2122 } else if (src_lo_rc == rc_stack && dst_lo_rc == rc_float) {
2123 __ unspill(as_FloatRegister(Matcher::_regEncode[dst_lo]),
2124 ireg == Op_VecD ? __ D : __ Q,
2125 ra_->reg2offset(src_lo));
2126 } else {
2127 ShouldNotReachHere();
2128 }
2129 }
2130 } else if (cbuf) {
2131 C2_MacroAssembler _masm(cbuf);
2132 switch (src_lo_rc) {
2133 case rc_int:
2134 if (dst_lo_rc == rc_int) { // gpr --> gpr copy
2135 if (is64) {
2136 __ mov(as_Register(Matcher::_regEncode[dst_lo]),
2137 as_Register(Matcher::_regEncode[src_lo]));
2138 } else {
2139 C2_MacroAssembler _masm(cbuf);
2140 __ movw(as_Register(Matcher::_regEncode[dst_lo]),
2141 as_Register(Matcher::_regEncode[src_lo]));
2142 }
2143 } else if (dst_lo_rc == rc_float) { // gpr --> fpr copy
2144 if (is64) {
2145 __ fmovd(as_FloatRegister(Matcher::_regEncode[dst_lo]),
2146 as_Register(Matcher::_regEncode[src_lo]));
2147 } else {
2148 __ fmovs(as_FloatRegister(Matcher::_regEncode[dst_lo]),
2149 as_Register(Matcher::_regEncode[src_lo]));
2150 }
2151 } else { // gpr --> stack spill
2152 assert(dst_lo_rc == rc_stack, "spill to bad register class");
2153 __ spill(as_Register(Matcher::_regEncode[src_lo]), is64, dst_offset);
2154 }
2155 break;
2156 case rc_float:
2157 if (dst_lo_rc == rc_int) { // fpr --> gpr copy
2158 if (is64) {
2159 __ fmovd(as_Register(Matcher::_regEncode[dst_lo]),
2160 as_FloatRegister(Matcher::_regEncode[src_lo]));
2161 } else {
2162 __ fmovs(as_Register(Matcher::_regEncode[dst_lo]),
2163 as_FloatRegister(Matcher::_regEncode[src_lo]));
2164 }
2165 } else if (dst_lo_rc == rc_float) { // fpr --> fpr copy
2166 if (cbuf) {
2167 __ fmovd(as_FloatRegister(Matcher::_regEncode[dst_lo]),
2168 as_FloatRegister(Matcher::_regEncode[src_lo]));
2169 } else {
2170 __ fmovs(as_FloatRegister(Matcher::_regEncode[dst_lo]),
2171 as_FloatRegister(Matcher::_regEncode[src_lo]));
2172 }
2173 } else { // fpr --> stack spill
2174 assert(dst_lo_rc == rc_stack, "spill to bad register class");
2175 __ spill(as_FloatRegister(Matcher::_regEncode[src_lo]),
2176 is64 ? __ D : __ S, dst_offset);
2177 }
2178 break;
2179 case rc_stack:
2180 if (dst_lo_rc == rc_int) { // stack --> gpr load
2181 __ unspill(as_Register(Matcher::_regEncode[dst_lo]), is64, src_offset);
2182 } else if (dst_lo_rc == rc_float) { // stack --> fpr load
2183 __ unspill(as_FloatRegister(Matcher::_regEncode[dst_lo]),
2184 is64 ? __ D : __ S, src_offset);
2185 } else { // stack --> stack copy
2186 assert(dst_lo_rc == rc_stack, "spill to bad register class");
2187 __ unspill(rscratch1, is64, src_offset);
2188 __ spill(rscratch1, is64, dst_offset);
2189 }
2190 break;
2191 default:
2192 assert(false, "bad rc_class for spill");
2193 ShouldNotReachHere();
2194 }
2195 }
2196
2197 if (st) {
2198 st->print("spill ");
2199 if (src_lo_rc == rc_stack) {
2200 st->print("[sp, #%d] -> ", ra_->reg2offset(src_lo));
2201 } else {
2202 st->print("%s -> ", Matcher::regName[src_lo]);
2203 }
2204 if (dst_lo_rc == rc_stack) {
2205 st->print("[sp, #%d]", ra_->reg2offset(dst_lo));
2206 } else {
2207 st->print("%s", Matcher::regName[dst_lo]);
2208 }
2209 if (bottom_type()->isa_vect() != NULL) {
2210 int vsize = 0;
2211 switch (ideal_reg()) {
2212 case Op_VecD:
2213 vsize = 64;
2214 break;
2215 case Op_VecX:
2216 vsize = 128;
2217 break;
2218 case Op_VecA:
2219 vsize = Matcher::scalable_vector_reg_size(T_BYTE) * 8;
2220 break;
2221 default:
2222 assert(false, "bad register type for spill");
2223 ShouldNotReachHere();
2224 }
2225 st->print("\t# vector spill size = %d", vsize);
2226 } else {
2227 st->print("\t# spill size = %d", is64 ? 64 : 32);
2228 }
2229 }
2230
2231 return 0;
2232
2233 }
2234
2235 #ifndef PRODUCT
2236 void MachSpillCopyNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
2237 if (!ra_)
2238 st->print("N%d = SpillCopy(N%d)", _idx, in(1)->_idx);
2239 else
2240 implementation(NULL, ra_, false, st);
2241 }
2242 #endif
2243
2244 void MachSpillCopyNode::emit(CodeBuffer &cbuf, PhaseRegAlloc *ra_) const {
2245 implementation(&cbuf, ra_, false, NULL);
2246 }
2247
2248 uint MachSpillCopyNode::size(PhaseRegAlloc *ra_) const {
2249 return MachNode::size(ra_);
2250 }
2251
2252 //=============================================================================
2253
2254 #ifndef PRODUCT
2255 void BoxLockNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
2256 int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
2257 int reg = ra_->get_reg_first(this);
2258 st->print("add %s, rsp, #%d]\t# box lock",
2259 Matcher::regName[reg], offset);
2260 }
2261 #endif
2262
2263 void BoxLockNode::emit(CodeBuffer &cbuf, PhaseRegAlloc *ra_) const {
2264 C2_MacroAssembler _masm(&cbuf);
2265
2266 int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
2267 int reg = ra_->get_encode(this);
2268
2269 // This add will handle any 24-bit signed offset. 24 bits allows an
2270 // 8 megabyte stack frame.
2271 __ add(as_Register(reg), sp, offset);
2272 }
2273
2274 uint BoxLockNode::size(PhaseRegAlloc *ra_) const {
2275 // BoxLockNode is not a MachNode, so we can't just call MachNode::size(ra_).
2276 int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
2277
2278 if (Assembler::operand_valid_for_add_sub_immediate(offset)) {
2279 return NativeInstruction::instruction_size;
2280 } else {
2281 return 2 * NativeInstruction::instruction_size;
2282 }
2283 }
2284
2285 //=============================================================================
2286
2287 #ifndef PRODUCT
2288 void MachUEPNode::format(PhaseRegAlloc* ra_, outputStream* st) const
2289 {
2290 st->print_cr("# MachUEPNode");
2291 if (UseCompressedClassPointers) {
2292 st->print_cr("\tldrw rscratch1, j_rarg0 + oopDesc::klass_offset_in_bytes()]\t# compressed klass");
2293 if (CompressedKlassPointers::shift() != 0) {
2294 st->print_cr("\tdecode_klass_not_null rscratch1, rscratch1");
2295 }
2296 } else {
2297 st->print_cr("\tldr rscratch1, j_rarg0 + oopDesc::klass_offset_in_bytes()]\t# compressed klass");
2298 }
2299 st->print_cr("\tcmp r0, rscratch1\t # Inline cache check");
2300 st->print_cr("\tbne, SharedRuntime::_ic_miss_stub");
2301 }
2302 #endif
2303
2304 void MachUEPNode::emit(CodeBuffer& cbuf, PhaseRegAlloc* ra_) const
2305 {
2306 // This is the unverified entry point.
2307 C2_MacroAssembler _masm(&cbuf);
2308
2309 __ cmp_klass(j_rarg0, rscratch2, rscratch1);
2310 Label skip;
2311 // TODO
2312 // can we avoid this skip and still use a reloc?
2313 __ br(Assembler::EQ, skip);
2314 __ far_jump(RuntimeAddress(SharedRuntime::get_ic_miss_stub()));
2315 __ bind(skip);
2316 }
2317
2318 uint MachUEPNode::size(PhaseRegAlloc* ra_) const
2319 {
2320 return MachNode::size(ra_);
2321 }
2322
2323 // REQUIRED EMIT CODE
2324
2325 //=============================================================================
2326
2327 // Emit exception handler code.
2328 int HandlerImpl::emit_exception_handler(CodeBuffer& cbuf)
2329 {
2330 // mov rscratch1 #exception_blob_entry_point
2331 // br rscratch1
2332 // Note that the code buffer's insts_mark is always relative to insts.
2333 // That's why we must use the macroassembler to generate a handler.
2334 C2_MacroAssembler _masm(&cbuf);
2335 address base = __ start_a_stub(size_exception_handler());
2336 if (base == NULL) {
2337 ciEnv::current()->record_failure("CodeCache is full");
2338 return 0; // CodeBuffer::expand failed
2339 }
2340 int offset = __ offset();
2341 __ far_jump(RuntimeAddress(OptoRuntime::exception_blob()->entry_point()));
2342 assert(__ offset() - offset <= (int) size_exception_handler(), "overflow");
2343 __ end_a_stub();
2344 return offset;
2345 }
2346
2347 // Emit deopt handler code.
2348 int HandlerImpl::emit_deopt_handler(CodeBuffer& cbuf)
2349 {
2350 // Note that the code buffer's insts_mark is always relative to insts.
2351 // That's why we must use the macroassembler to generate a handler.
2352 C2_MacroAssembler _masm(&cbuf);
2353 address base = __ start_a_stub(size_deopt_handler());
2354 if (base == NULL) {
2355 ciEnv::current()->record_failure("CodeCache is full");
2356 return 0; // CodeBuffer::expand failed
2357 }
2358 int offset = __ offset();
2359
2360 __ adr(lr, __ pc());
2361 __ far_jump(RuntimeAddress(SharedRuntime::deopt_blob()->unpack()));
2362
2363 assert(__ offset() - offset == (int) size_deopt_handler(), "overflow");
2364 __ end_a_stub();
2365 return offset;
2366 }
2367
2368 // REQUIRED MATCHER CODE
2369
2370 //=============================================================================
2371
2372 const bool Matcher::match_rule_supported(int opcode) {
2373 if (!has_match_rule(opcode))
2374 return false;
2375
2376 bool ret_value = true;
2377 switch (opcode) {
2378 case Op_OnSpinWait:
2379 return VM_Version::supports_on_spin_wait();
2380 case Op_CacheWB:
2381 case Op_CacheWBPreSync:
2382 case Op_CacheWBPostSync:
2383 if (!VM_Version::supports_data_cache_line_flush()) {
2384 ret_value = false;
2385 }
2386 break;
2387 }
2388
2389 return ret_value; // Per default match rules are supported.
2390 }
2391
2392 // Identify extra cases that we might want to provide match rules for vector nodes and
2393 // other intrinsics guarded with vector length (vlen) and element type (bt).
2394 const bool Matcher::match_rule_supported_vector(int opcode, int vlen, BasicType bt) {
2395 if (!match_rule_supported(opcode) || !vector_size_supported(bt, vlen)) {
2396 return false;
2397 }
2398 int bit_size = vlen * type2aelembytes(bt) * 8;
2399 if (UseSVE == 0 && bit_size > 128) {
2400 return false;
2401 }
2402 if (UseSVE > 0) {
2403 return op_sve_supported(opcode);
2404 } else { // NEON
2405 // Special cases
2406 switch (opcode) {
2407 case Op_VectorMaskCmp:
2408 // We don't have VectorReinterpret with bit_size less than 64 support for
2409 // now, even for byte type. To be refined with fully VectorCast support.
2410 case Op_VectorReinterpret:
2411 if (vlen < 2 || bit_size < 64) {
2412 return false;
2413 }
2414 break;
2415 case Op_MulAddVS2VI:
2416 if (bit_size < 128) {
2417 return false;
2418 }
2419 break;
2420 case Op_MulVL:
2421 return false;
2422 case Op_VectorLoadShuffle:
2423 case Op_VectorRearrange:
2424 if (vlen < 4) {
2425 return false;
2426 }
2427 break;
2428 // Some types of VectorCast are not implemented for now.
2429 case Op_VectorCastI2X:
2430 if (bt == T_BYTE) {
2431 return false;
2432 }
2433 break;
2434 case Op_VectorCastS2X:
2435 if (vlen < 4 || bit_size < 64) {
2436 return false;
2437 }
2438 break;
2439 case Op_VectorCastF2X:
2440 case Op_VectorCastD2X:
2441 if (bt == T_INT || bt == T_SHORT || bt == T_BYTE || bt == T_LONG) {
2442 return false;
2443 }
2444 break;
2445 default:
2446 break;
2447 }
2448 }
2449 return true; // Per default match rules are supported.
2450 }
2451
2452 const RegMask* Matcher::predicate_reg_mask(void) {
2453 return &_PR_REG_mask;
2454 }
2455
2456 const TypeVect* Matcher::predicate_reg_type(const Type* elemTy, int length) {
2457 return new TypeVectMask(elemTy, length);
2458 }
2459
2460 // Vector calling convention not yet implemented.
2461 const bool Matcher::supports_vector_calling_convention(void) {
2462 return false;
2463 }
2464
2465 OptoRegPair Matcher::vector_return_value(uint ideal_reg) {
2466 Unimplemented();
2467 return OptoRegPair(0, 0);
2468 }
2469
2470 const int Matcher::float_pressure(int default_pressure_threshold) {
2471 return default_pressure_threshold;
2472 }
2473
2474 // Is this branch offset short enough that a short branch can be used?
2475 //
2476 // NOTE: If the platform does not provide any short branch variants, then
2477 // this method should return false for offset 0.
2478 bool Matcher::is_short_branch_offset(int rule, int br_size, int offset) {
2479 // The passed offset is relative to address of the branch.
2480
2481 return (-32768 <= offset && offset < 32768);
2482 }
2483
2484 // Vector width in bytes.
2485 const int Matcher::vector_width_in_bytes(BasicType bt) {
2486 // The MaxVectorSize should have been set by detecting SVE max vector register size.
2487 int size = MIN2((UseSVE > 0) ? 256 : 16, (int)MaxVectorSize);
2488 // Minimum 2 values in vector
2489 if (size < 2*type2aelembytes(bt)) size = 0;
2490 // But never < 4
2491 if (size < 4) size = 0;
2492 return size;
2493 }
2494
2495 // Limits on vector size (number of elements) loaded into vector.
2496 const int Matcher::max_vector_size(const BasicType bt) {
2497 return vector_width_in_bytes(bt)/type2aelembytes(bt);
2498 }
2499 const int Matcher::min_vector_size(const BasicType bt) {
2500 int max_size = max_vector_size(bt);
2501 if ((UseSVE > 0) && (MaxVectorSize >= 16)) {
2502 // Currently vector length less than SVE vector register size is not supported.
2503 return max_size;
2504 } else { // NEON
2505 // Limit the vector size to 8 bytes
2506 int size = 8 / type2aelembytes(bt);
2507 if (bt == T_BYTE) {
2508 // To support vector api shuffle/rearrange.
2509 size = 4;
2510 } else if (bt == T_BOOLEAN) {
2511 // To support vector api load/store mask.
2512 size = 2;
2513 }
2514 if (size < 2) size = 2;
2515 return MIN2(size,max_size);
2516 }
2517 }
2518
2519 // Actual max scalable vector register length.
2520 const int Matcher::scalable_vector_reg_size(const BasicType bt) {
2521 return Matcher::max_vector_size(bt);
2522 }
2523
2524 // Vector ideal reg.
2525 const uint Matcher::vector_ideal_reg(int len) {
2526 if (UseSVE > 0 && 16 <= len && len <= 256) {
2527 return Op_VecA;
2528 }
2529 switch(len) {
2530 // For 16-bit/32-bit mask vector, reuse VecD.
2531 case 2:
2532 case 4:
2533 case 8: return Op_VecD;
2534 case 16: return Op_VecX;
2535 }
2536 ShouldNotReachHere();
2537 return 0;
2538 }
2539
2540 MachOper* Matcher::pd_specialize_generic_vector_operand(MachOper* original_opnd, uint ideal_reg, bool is_temp) {
2541 ShouldNotReachHere(); // generic vector operands not supported
2542 return NULL;
2543 }
2544
2545 bool Matcher::is_generic_reg2reg_move(MachNode* m) {
2546 ShouldNotReachHere(); // generic vector operands not supported
2547 return false;
2548 }
2549
2550 bool Matcher::is_generic_vector(MachOper* opnd) {
2551 ShouldNotReachHere(); // generic vector operands not supported
2552 return false;
2553 }
2554
2555 // Return whether or not this register is ever used as an argument.
2556 // This function is used on startup to build the trampoline stubs in
2557 // generateOptoStub. Registers not mentioned will be killed by the VM
2558 // call in the trampoline, and arguments in those registers not be
2559 // available to the callee.
2560 bool Matcher::can_be_java_arg(int reg)
2561 {
2562 return
2563 reg == R0_num || reg == R0_H_num ||
2564 reg == R1_num || reg == R1_H_num ||
2565 reg == R2_num || reg == R2_H_num ||
2566 reg == R3_num || reg == R3_H_num ||
2567 reg == R4_num || reg == R4_H_num ||
2568 reg == R5_num || reg == R5_H_num ||
2569 reg == R6_num || reg == R6_H_num ||
2570 reg == R7_num || reg == R7_H_num ||
2571 reg == V0_num || reg == V0_H_num ||
2572 reg == V1_num || reg == V1_H_num ||
2573 reg == V2_num || reg == V2_H_num ||
2574 reg == V3_num || reg == V3_H_num ||
2575 reg == V4_num || reg == V4_H_num ||
2576 reg == V5_num || reg == V5_H_num ||
2577 reg == V6_num || reg == V6_H_num ||
2578 reg == V7_num || reg == V7_H_num;
2579 }
2580
2581 bool Matcher::is_spillable_arg(int reg)
2582 {
2583 return can_be_java_arg(reg);
2584 }
2585
2586 bool Matcher::use_asm_for_ldiv_by_con(jlong divisor) {
2587 return false;
2588 }
2589
2590 RegMask Matcher::divI_proj_mask() {
2591 ShouldNotReachHere();
2592 return RegMask();
2593 }
2594
2595 // Register for MODI projection of divmodI.
2596 RegMask Matcher::modI_proj_mask() {
2597 ShouldNotReachHere();
2598 return RegMask();
2599 }
2600
2601 // Register for DIVL projection of divmodL.
2602 RegMask Matcher::divL_proj_mask() {
2603 ShouldNotReachHere();
2604 return RegMask();
2605 }
2606
2607 // Register for MODL projection of divmodL.
2608 RegMask Matcher::modL_proj_mask() {
2609 ShouldNotReachHere();
2610 return RegMask();
2611 }
2612
2613 const RegMask Matcher::method_handle_invoke_SP_save_mask() {
2614 return FP_REG_mask();
2615 }
2616
2617 bool size_fits_all_mem_uses(AddPNode* addp, int shift) {
2618 for (DUIterator_Fast imax, i = addp->fast_outs(imax); i < imax; i++) {
2619 Node* u = addp->fast_out(i);
2620 if (u->is_LoadStore()) {
2621 // On AArch64, LoadStoreNodes (i.e. compare and swap
2622 // instructions) only take register indirect as an operand, so
2623 // any attempt to use an AddPNode as an input to a LoadStoreNode
2624 // must fail.
2625 return false;
2626 }
2627 if (u->is_Mem()) {
2628 int opsize = u->as_Mem()->memory_size();
2629 assert(opsize > 0, "unexpected memory operand size");
2630 if (u->as_Mem()->memory_size() != (1<<shift)) {
2631 return false;
2632 }
2633 }
2634 }
2635 return true;
2636 }
2637
2638 // Should the matcher clone input 'm' of node 'n'?
2639 bool Matcher::pd_clone_node(Node* n, Node* m, Matcher::MStack& mstack) {
2640 if (is_vshift_con_pattern(n, m)) { // ShiftV src (ShiftCntV con)
2641 mstack.push(m, Visit); // m = ShiftCntV
2642 return true;
2643 }
2644 return false;
2645 }
2646
2647 // Should the Matcher clone shifts on addressing modes, expecting them
2648 // to be subsumed into complex addressing expressions or compute them
2649 // into registers?
2650 bool Matcher::pd_clone_address_expressions(AddPNode* m, Matcher::MStack& mstack, VectorSet& address_visited) {
2651 if (clone_base_plus_offset_address(m, mstack, address_visited)) {
2652 return true;
2653 }
2654
2655 Node *off = m->in(AddPNode::Offset);
2656 if (off->Opcode() == Op_LShiftL && off->in(2)->is_Con() &&
2657 size_fits_all_mem_uses(m, off->in(2)->get_int()) &&
2658 // Are there other uses besides address expressions?
2659 !is_visited(off)) {
2660 address_visited.set(off->_idx); // Flag as address_visited
2661 mstack.push(off->in(2), Visit);
2662 Node *conv = off->in(1);
2663 if (conv->Opcode() == Op_ConvI2L &&
2664 // Are there other uses besides address expressions?
2665 !is_visited(conv)) {
2666 address_visited.set(conv->_idx); // Flag as address_visited
2667 mstack.push(conv->in(1), Pre_Visit);
2668 } else {
2669 mstack.push(conv, Pre_Visit);
2670 }
2671 address_visited.test_set(m->_idx); // Flag as address_visited
2672 mstack.push(m->in(AddPNode::Address), Pre_Visit);
2673 mstack.push(m->in(AddPNode::Base), Pre_Visit);
2674 return true;
2675 } else if (off->Opcode() == Op_ConvI2L &&
2676 // Are there other uses besides address expressions?
2677 !is_visited(off)) {
2678 address_visited.test_set(m->_idx); // Flag as address_visited
2679 address_visited.set(off->_idx); // Flag as address_visited
2680 mstack.push(off->in(1), Pre_Visit);
2681 mstack.push(m->in(AddPNode::Address), Pre_Visit);
2682 mstack.push(m->in(AddPNode::Base), Pre_Visit);
2683 return true;
2684 }
2685 return false;
2686 }
2687
2688 #define MOV_VOLATILE(REG, BASE, INDEX, SCALE, DISP, SCRATCH, INSN) \
2689 C2_MacroAssembler _masm(&cbuf); \
2690 { \
2691 guarantee(INDEX == -1, "mode not permitted for volatile"); \
2692 guarantee(DISP == 0, "mode not permitted for volatile"); \
2693 guarantee(SCALE == 0, "mode not permitted for volatile"); \
2694 __ INSN(REG, as_Register(BASE)); \
2695 }
2696
2697
2698 static Address mem2address(int opcode, Register base, int index, int size, int disp)
2699 {
2700 Address::extend scale;
2701
2702 // Hooboy, this is fugly. We need a way to communicate to the
2703 // encoder that the index needs to be sign extended, so we have to
2704 // enumerate all the cases.
2705 switch (opcode) {
2706 case INDINDEXSCALEDI2L:
2707 case INDINDEXSCALEDI2LN:
2708 case INDINDEXI2L:
2709 case INDINDEXI2LN:
2710 scale = Address::sxtw(size);
2711 break;
2712 default:
2713 scale = Address::lsl(size);
2714 }
2715
2716 if (index == -1) {
2717 return Address(base, disp);
2718 } else {
2719 assert(disp == 0, "unsupported address mode: disp = %d", disp);
2720 return Address(base, as_Register(index), scale);
2721 }
2722 }
2723
2724
2725 typedef void (MacroAssembler::* mem_insn)(Register Rt, const Address &adr);
2726 typedef void (MacroAssembler::* mem_insn2)(Register Rt, Register adr);
2727 typedef void (MacroAssembler::* mem_float_insn)(FloatRegister Rt, const Address &adr);
2728 typedef void (MacroAssembler::* mem_vector_insn)(FloatRegister Rt,
2729 MacroAssembler::SIMD_RegVariant T, const Address &adr);
2730
2731 // Used for all non-volatile memory accesses. The use of
2732 // $mem->opcode() to discover whether this pattern uses sign-extended
2733 // offsets is something of a kludge.
2734 static void loadStore(C2_MacroAssembler masm, mem_insn insn,
2735 Register reg, int opcode,
2736 Register base, int index, int scale, int disp,
2737 int size_in_memory)
2738 {
2739 Address addr = mem2address(opcode, base, index, scale, disp);
2740 if (addr.getMode() == Address::base_plus_offset) {
2741 /* If we get an out-of-range offset it is a bug in the compiler,
2742 so we assert here. */
2743 assert(Address::offset_ok_for_immed(addr.offset(), exact_log2(size_in_memory)),
2744 "c2 compiler bug");
2745 /* Fix up any out-of-range offsets. */
2746 assert_different_registers(rscratch1, base);
2747 assert_different_registers(rscratch1, reg);
2748 addr = masm.legitimize_address(addr, size_in_memory, rscratch1);
2749 }
2750 (masm.*insn)(reg, addr);
2751 }
2752
2753 static void loadStore(C2_MacroAssembler masm, mem_float_insn insn,
2754 FloatRegister reg, int opcode,
2755 Register base, int index, int size, int disp,
2756 int size_in_memory)
2757 {
2758 Address::extend scale;
2759
2760 switch (opcode) {
2761 case INDINDEXSCALEDI2L:
2762 case INDINDEXSCALEDI2LN:
2763 scale = Address::sxtw(size);
2764 break;
2765 default:
2766 scale = Address::lsl(size);
2767 }
2768
2769 if (index == -1) {
2770 /* If we get an out-of-range offset it is a bug in the compiler,
2771 so we assert here. */
2772 assert(Address::offset_ok_for_immed(disp, exact_log2(size_in_memory)), "c2 compiler bug");
2773 /* Fix up any out-of-range offsets. */
2774 assert_different_registers(rscratch1, base);
2775 Address addr = Address(base, disp);
2776 addr = masm.legitimize_address(addr, size_in_memory, rscratch1);
2777 (masm.*insn)(reg, addr);
2778 } else {
2779 assert(disp == 0, "unsupported address mode: disp = %d", disp);
2780 (masm.*insn)(reg, Address(base, as_Register(index), scale));
2781 }
2782 }
2783
2784 static void loadStore(C2_MacroAssembler masm, mem_vector_insn insn,
2785 FloatRegister reg, MacroAssembler::SIMD_RegVariant T,
2786 int opcode, Register base, int index, int size, int disp)
2787 {
2788 if (index == -1) {
2789 (masm.*insn)(reg, T, Address(base, disp));
2790 } else {
2791 assert(disp == 0, "unsupported address mode");
2792 (masm.*insn)(reg, T, Address(base, as_Register(index), Address::lsl(size)));
2793 }
2794 }
2795
2796 %}
2797
2798
2799
2800 //----------ENCODING BLOCK-----------------------------------------------------
2801 // This block specifies the encoding classes used by the compiler to
2802 // output byte streams. Encoding classes are parameterized macros
2803 // used by Machine Instruction Nodes in order to generate the bit
2804 // encoding of the instruction. Operands specify their base encoding
2805 // interface with the interface keyword. There are currently
2806 // supported four interfaces, REG_INTER, CONST_INTER, MEMORY_INTER, &
2807 // COND_INTER. REG_INTER causes an operand to generate a function
2808 // which returns its register number when queried. CONST_INTER causes
2809 // an operand to generate a function which returns the value of the
2810 // constant when queried. MEMORY_INTER causes an operand to generate
2811 // four functions which return the Base Register, the Index Register,
2812 // the Scale Value, and the Offset Value of the operand when queried.
2813 // COND_INTER causes an operand to generate six functions which return
2814 // the encoding code (ie - encoding bits for the instruction)
2815 // associated with each basic boolean condition for a conditional
2816 // instruction.
2817 //
2818 // Instructions specify two basic values for encoding. Again, a
2819 // function is available to check if the constant displacement is an
2820 // oop. They use the ins_encode keyword to specify their encoding
2821 // classes (which must be a sequence of enc_class names, and their
2822 // parameters, specified in the encoding block), and they use the
2823 // opcode keyword to specify, in order, their primary, secondary, and
2824 // tertiary opcode. Only the opcode sections which a particular
2825 // instruction needs for encoding need to be specified.
2826 encode %{
2827 // Build emit functions for each basic byte or larger field in the
2828 // intel encoding scheme (opcode, rm, sib, immediate), and call them
2829 // from C++ code in the enc_class source block. Emit functions will
2830 // live in the main source block for now. In future, we can
2831 // generalize this by adding a syntax that specifies the sizes of
2832 // fields in an order, so that the adlc can build the emit functions
2833 // automagically
2834
2835 // catch all for unimplemented encodings
2836 enc_class enc_unimplemented %{
2837 C2_MacroAssembler _masm(&cbuf);
2838 __ unimplemented("C2 catch all");
2839 %}
2840
2841 // BEGIN Non-volatile memory access
2842
2843 // This encoding class is generated automatically from ad_encode.m4.
2844 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2845 enc_class aarch64_enc_ldrsbw(iRegI dst, memory1 mem) %{
2846 Register dst_reg = as_Register($dst$$reg);
2847 loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrsbw, dst_reg, $mem->opcode(),
2848 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
2849 %}
2850
2851 // This encoding class is generated automatically from ad_encode.m4.
2852 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2853 enc_class aarch64_enc_ldrsb(iRegI dst, memory1 mem) %{
2854 Register dst_reg = as_Register($dst$$reg);
2855 loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrsb, dst_reg, $mem->opcode(),
2856 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
2857 %}
2858
2859 // This encoding class is generated automatically from ad_encode.m4.
2860 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2861 enc_class aarch64_enc_ldrb(iRegI dst, memory1 mem) %{
2862 Register dst_reg = as_Register($dst$$reg);
2863 loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrb, dst_reg, $mem->opcode(),
2864 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
2865 %}
2866
2867 // This encoding class is generated automatically from ad_encode.m4.
2868 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2869 enc_class aarch64_enc_ldrb(iRegL dst, memory1 mem) %{
2870 Register dst_reg = as_Register($dst$$reg);
2871 loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrb, dst_reg, $mem->opcode(),
2872 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
2873 %}
2874
2875 // This encoding class is generated automatically from ad_encode.m4.
2876 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2877 enc_class aarch64_enc_ldrshw(iRegI dst, memory2 mem) %{
2878 Register dst_reg = as_Register($dst$$reg);
2879 loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrshw, dst_reg, $mem->opcode(),
2880 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
2881 %}
2882
2883 // This encoding class is generated automatically from ad_encode.m4.
2884 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2885 enc_class aarch64_enc_ldrsh(iRegI dst, memory2 mem) %{
2886 Register dst_reg = as_Register($dst$$reg);
2887 loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrsh, dst_reg, $mem->opcode(),
2888 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
2889 %}
2890
2891 // This encoding class is generated automatically from ad_encode.m4.
2892 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2893 enc_class aarch64_enc_ldrh(iRegI dst, memory2 mem) %{
2894 Register dst_reg = as_Register($dst$$reg);
2895 loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrh, dst_reg, $mem->opcode(),
2896 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
2897 %}
2898
2899 // This encoding class is generated automatically from ad_encode.m4.
2900 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2901 enc_class aarch64_enc_ldrh(iRegL dst, memory2 mem) %{
2902 Register dst_reg = as_Register($dst$$reg);
2903 loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrh, dst_reg, $mem->opcode(),
2904 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
2905 %}
2906
2907 // This encoding class is generated automatically from ad_encode.m4.
2908 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2909 enc_class aarch64_enc_ldrw(iRegI dst, memory4 mem) %{
2910 Register dst_reg = as_Register($dst$$reg);
2911 loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrw, dst_reg, $mem->opcode(),
2912 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
2913 %}
2914
2915 // This encoding class is generated automatically from ad_encode.m4.
2916 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2917 enc_class aarch64_enc_ldrw(iRegL dst, memory4 mem) %{
2918 Register dst_reg = as_Register($dst$$reg);
2919 loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrw, dst_reg, $mem->opcode(),
2920 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
2921 %}
2922
2923 // This encoding class is generated automatically from ad_encode.m4.
2924 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2925 enc_class aarch64_enc_ldrsw(iRegL dst, memory4 mem) %{
2926 Register dst_reg = as_Register($dst$$reg);
2927 loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrsw, dst_reg, $mem->opcode(),
2928 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
2929 %}
2930
2931 // This encoding class is generated automatically from ad_encode.m4.
2932 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2933 enc_class aarch64_enc_ldr(iRegL dst, memory8 mem) %{
2934 Register dst_reg = as_Register($dst$$reg);
2935 loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldr, dst_reg, $mem->opcode(),
2936 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
2937 %}
2938
2939 // This encoding class is generated automatically from ad_encode.m4.
2940 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2941 enc_class aarch64_enc_ldrs(vRegF dst, memory4 mem) %{
2942 FloatRegister dst_reg = as_FloatRegister($dst$$reg);
2943 loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrs, dst_reg, $mem->opcode(),
2944 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
2945 %}
2946
2947 // This encoding class is generated automatically from ad_encode.m4.
2948 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2949 enc_class aarch64_enc_ldrd(vRegD dst, memory8 mem) %{
2950 FloatRegister dst_reg = as_FloatRegister($dst$$reg);
2951 loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrd, dst_reg, $mem->opcode(),
2952 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
2953 %}
2954
2955 // This encoding class is generated automatically from ad_encode.m4.
2956 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2957 enc_class aarch64_enc_strb(iRegI src, memory1 mem) %{
2958 Register src_reg = as_Register($src$$reg);
2959 loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::strb, src_reg, $mem->opcode(),
2960 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
2961 %}
2962
2963 // This encoding class is generated automatically from ad_encode.m4.
2964 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2965 enc_class aarch64_enc_strb0(memory1 mem) %{
2966 C2_MacroAssembler _masm(&cbuf);
2967 loadStore(_masm, &MacroAssembler::strb, zr, $mem->opcode(),
2968 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
2969 %}
2970
2971 // This encoding class is generated automatically from ad_encode.m4.
2972 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2973 enc_class aarch64_enc_strh(iRegI src, memory2 mem) %{
2974 Register src_reg = as_Register($src$$reg);
2975 loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::strh, src_reg, $mem->opcode(),
2976 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
2977 %}
2978
2979 // This encoding class is generated automatically from ad_encode.m4.
2980 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2981 enc_class aarch64_enc_strh0(memory2 mem) %{
2982 C2_MacroAssembler _masm(&cbuf);
2983 loadStore(_masm, &MacroAssembler::strh, zr, $mem->opcode(),
2984 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
2985 %}
2986
2987 // This encoding class is generated automatically from ad_encode.m4.
2988 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2989 enc_class aarch64_enc_strw(iRegI src, memory4 mem) %{
2990 Register src_reg = as_Register($src$$reg);
2991 loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::strw, src_reg, $mem->opcode(),
2992 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
2993 %}
2994
2995 // This encoding class is generated automatically from ad_encode.m4.
2996 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2997 enc_class aarch64_enc_strw0(memory4 mem) %{
2998 C2_MacroAssembler _masm(&cbuf);
2999 loadStore(_masm, &MacroAssembler::strw, zr, $mem->opcode(),
3000 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
3001 %}
3002
3003 // This encoding class is generated automatically from ad_encode.m4.
3004 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
3005 enc_class aarch64_enc_str(iRegL src, memory8 mem) %{
3006 Register src_reg = as_Register($src$$reg);
3007 // we sometimes get asked to store the stack pointer into the
3008 // current thread -- we cannot do that directly on AArch64
3009 if (src_reg == r31_sp) {
3010 C2_MacroAssembler _masm(&cbuf);
3011 assert(as_Register($mem$$base) == rthread, "unexpected store for sp");
3012 __ mov(rscratch2, sp);
3013 src_reg = rscratch2;
3014 }
3015 loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::str, src_reg, $mem->opcode(),
3016 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
3017 %}
3018
3019 // This encoding class is generated automatically from ad_encode.m4.
3020 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
3021 enc_class aarch64_enc_str0(memory8 mem) %{
3022 C2_MacroAssembler _masm(&cbuf);
3023 loadStore(_masm, &MacroAssembler::str, zr, $mem->opcode(),
3024 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
3025 %}
3026
3027 // This encoding class is generated automatically from ad_encode.m4.
3028 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
3029 enc_class aarch64_enc_strs(vRegF src, memory4 mem) %{
3030 FloatRegister src_reg = as_FloatRegister($src$$reg);
3031 loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::strs, src_reg, $mem->opcode(),
3032 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
3033 %}
3034
3035 // This encoding class is generated automatically from ad_encode.m4.
3036 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
3037 enc_class aarch64_enc_strd(vRegD src, memory8 mem) %{
3038 FloatRegister src_reg = as_FloatRegister($src$$reg);
3039 loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::strd, src_reg, $mem->opcode(),
3040 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
3041 %}
3042
3043 // This encoding class is generated automatically from ad_encode.m4.
3044 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
3045 enc_class aarch64_enc_strb0_ordered(memory4 mem) %{
3046 C2_MacroAssembler _masm(&cbuf);
3047 __ membar(Assembler::StoreStore);
3048 loadStore(_masm, &MacroAssembler::strb, zr, $mem->opcode(),
3049 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
3050 %}
3051
3052 // END Non-volatile memory access
3053
3054 // Vector loads and stores
3055 enc_class aarch64_enc_ldrvH(vecD dst, memory mem) %{
3056 FloatRegister dst_reg = as_FloatRegister($dst$$reg);
3057 loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldr, dst_reg, MacroAssembler::H,
3058 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3059 %}
3060
3061 enc_class aarch64_enc_ldrvS(vecD dst, memory mem) %{
3062 FloatRegister dst_reg = as_FloatRegister($dst$$reg);
3063 loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldr, dst_reg, MacroAssembler::S,
3064 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3065 %}
3066
3067 enc_class aarch64_enc_ldrvD(vecD dst, memory mem) %{
3068 FloatRegister dst_reg = as_FloatRegister($dst$$reg);
3069 loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldr, dst_reg, MacroAssembler::D,
3070 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3071 %}
3072
3073 enc_class aarch64_enc_ldrvQ(vecX dst, memory mem) %{
3074 FloatRegister dst_reg = as_FloatRegister($dst$$reg);
3075 loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldr, dst_reg, MacroAssembler::Q,
3076 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3077 %}
3078
3079 enc_class aarch64_enc_strvH(vecD src, memory mem) %{
3080 FloatRegister src_reg = as_FloatRegister($src$$reg);
3081 loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::str, src_reg, MacroAssembler::H,
3082 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3083 %}
3084
3085 enc_class aarch64_enc_strvS(vecD src, memory mem) %{
3086 FloatRegister src_reg = as_FloatRegister($src$$reg);
3087 loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::str, src_reg, MacroAssembler::S,
3088 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3089 %}
3090
3091 enc_class aarch64_enc_strvD(vecD src, memory mem) %{
3092 FloatRegister src_reg = as_FloatRegister($src$$reg);
3093 loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::str, src_reg, MacroAssembler::D,
3094 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3095 %}
3096
3097 enc_class aarch64_enc_strvQ(vecX src, memory mem) %{
3098 FloatRegister src_reg = as_FloatRegister($src$$reg);
3099 loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::str, src_reg, MacroAssembler::Q,
3100 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3101 %}
3102
3103 // volatile loads and stores
3104
3105 enc_class aarch64_enc_stlrb(iRegI src, memory mem) %{
3106 MOV_VOLATILE(as_Register($src$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3107 rscratch1, stlrb);
3108 %}
3109
3110 enc_class aarch64_enc_stlrb0(memory mem) %{
3111 MOV_VOLATILE(zr, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3112 rscratch1, stlrb);
3113 %}
3114
3115 enc_class aarch64_enc_stlrh(iRegI src, memory mem) %{
3116 MOV_VOLATILE(as_Register($src$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3117 rscratch1, stlrh);
3118 %}
3119
3120 enc_class aarch64_enc_stlrh0(memory mem) %{
3121 MOV_VOLATILE(zr, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3122 rscratch1, stlrh);
3123 %}
3124
3125 enc_class aarch64_enc_stlrw(iRegI src, memory mem) %{
3126 MOV_VOLATILE(as_Register($src$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3127 rscratch1, stlrw);
3128 %}
3129
3130 enc_class aarch64_enc_stlrw0(memory mem) %{
3131 MOV_VOLATILE(zr, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3132 rscratch1, stlrw);
3133 %}
3134
3135 enc_class aarch64_enc_ldarsbw(iRegI dst, memory mem) %{
3136 Register dst_reg = as_Register($dst$$reg);
3137 MOV_VOLATILE(dst_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3138 rscratch1, ldarb);
3139 __ sxtbw(dst_reg, dst_reg);
3140 %}
3141
3142 enc_class aarch64_enc_ldarsb(iRegL dst, memory mem) %{
3143 Register dst_reg = as_Register($dst$$reg);
3144 MOV_VOLATILE(dst_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3145 rscratch1, ldarb);
3146 __ sxtb(dst_reg, dst_reg);
3147 %}
3148
3149 enc_class aarch64_enc_ldarbw(iRegI dst, memory mem) %{
3150 MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3151 rscratch1, ldarb);
3152 %}
3153
3154 enc_class aarch64_enc_ldarb(iRegL dst, memory mem) %{
3155 MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3156 rscratch1, ldarb);
3157 %}
3158
3159 enc_class aarch64_enc_ldarshw(iRegI dst, memory mem) %{
3160 Register dst_reg = as_Register($dst$$reg);
3161 MOV_VOLATILE(dst_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3162 rscratch1, ldarh);
3163 __ sxthw(dst_reg, dst_reg);
3164 %}
3165
3166 enc_class aarch64_enc_ldarsh(iRegL dst, memory mem) %{
3167 Register dst_reg = as_Register($dst$$reg);
3168 MOV_VOLATILE(dst_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3169 rscratch1, ldarh);
3170 __ sxth(dst_reg, dst_reg);
3171 %}
3172
3173 enc_class aarch64_enc_ldarhw(iRegI dst, memory mem) %{
3174 MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3175 rscratch1, ldarh);
3176 %}
3177
3178 enc_class aarch64_enc_ldarh(iRegL dst, memory mem) %{
3179 MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3180 rscratch1, ldarh);
3181 %}
3182
3183 enc_class aarch64_enc_ldarw(iRegI dst, memory mem) %{
3184 MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3185 rscratch1, ldarw);
3186 %}
3187
3188 enc_class aarch64_enc_ldarw(iRegL dst, memory mem) %{
3189 MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3190 rscratch1, ldarw);
3191 %}
3192
3193 enc_class aarch64_enc_ldar(iRegL dst, memory mem) %{
3194 MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3195 rscratch1, ldar);
3196 %}
3197
3198 enc_class aarch64_enc_fldars(vRegF dst, memory mem) %{
3199 MOV_VOLATILE(rscratch1, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3200 rscratch1, ldarw);
3201 __ fmovs(as_FloatRegister($dst$$reg), rscratch1);
3202 %}
3203
3204 enc_class aarch64_enc_fldard(vRegD dst, memory mem) %{
3205 MOV_VOLATILE(rscratch1, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3206 rscratch1, ldar);
3207 __ fmovd(as_FloatRegister($dst$$reg), rscratch1);
3208 %}
3209
3210 enc_class aarch64_enc_stlr(iRegL src, memory mem) %{
3211 Register src_reg = as_Register($src$$reg);
3212 // we sometimes get asked to store the stack pointer into the
3213 // current thread -- we cannot do that directly on AArch64
3214 if (src_reg == r31_sp) {
3215 C2_MacroAssembler _masm(&cbuf);
3216 assert(as_Register($mem$$base) == rthread, "unexpected store for sp");
3217 __ mov(rscratch2, sp);
3218 src_reg = rscratch2;
3219 }
3220 MOV_VOLATILE(src_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3221 rscratch1, stlr);
3222 %}
3223
3224 enc_class aarch64_enc_stlr0(memory mem) %{
3225 MOV_VOLATILE(zr, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3226 rscratch1, stlr);
3227 %}
3228
3229 enc_class aarch64_enc_fstlrs(vRegF src, memory mem) %{
3230 {
3231 C2_MacroAssembler _masm(&cbuf);
3232 FloatRegister src_reg = as_FloatRegister($src$$reg);
3233 __ fmovs(rscratch2, src_reg);
3234 }
3235 MOV_VOLATILE(rscratch2, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3236 rscratch1, stlrw);
3237 %}
3238
3239 enc_class aarch64_enc_fstlrd(vRegD src, memory mem) %{
3240 {
3241 C2_MacroAssembler _masm(&cbuf);
3242 FloatRegister src_reg = as_FloatRegister($src$$reg);
3243 __ fmovd(rscratch2, src_reg);
3244 }
3245 MOV_VOLATILE(rscratch2, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3246 rscratch1, stlr);
3247 %}
3248
3249 // synchronized read/update encodings
3250
3251 enc_class aarch64_enc_ldaxr(iRegL dst, memory8 mem) %{
3252 C2_MacroAssembler _masm(&cbuf);
3253 Register dst_reg = as_Register($dst$$reg);
3254 Register base = as_Register($mem$$base);
3255 int index = $mem$$index;
3256 int scale = $mem$$scale;
3257 int disp = $mem$$disp;
3258 if (index == -1) {
3259 if (disp != 0) {
3260 __ lea(rscratch1, Address(base, disp));
3261 __ ldaxr(dst_reg, rscratch1);
3262 } else {
3263 // TODO
3264 // should we ever get anything other than this case?
3265 __ ldaxr(dst_reg, base);
3266 }
3267 } else {
3268 Register index_reg = as_Register(index);
3269 if (disp == 0) {
3270 __ lea(rscratch1, Address(base, index_reg, Address::lsl(scale)));
3271 __ ldaxr(dst_reg, rscratch1);
3272 } else {
3273 __ lea(rscratch1, Address(base, disp));
3274 __ lea(rscratch1, Address(rscratch1, index_reg, Address::lsl(scale)));
3275 __ ldaxr(dst_reg, rscratch1);
3276 }
3277 }
3278 %}
3279
3280 enc_class aarch64_enc_stlxr(iRegLNoSp src, memory8 mem) %{
3281 C2_MacroAssembler _masm(&cbuf);
3282 Register src_reg = as_Register($src$$reg);
3283 Register base = as_Register($mem$$base);
3284 int index = $mem$$index;
3285 int scale = $mem$$scale;
3286 int disp = $mem$$disp;
3287 if (index == -1) {
3288 if (disp != 0) {
3289 __ lea(rscratch2, Address(base, disp));
3290 __ stlxr(rscratch1, src_reg, rscratch2);
3291 } else {
3292 // TODO
3293 // should we ever get anything other than this case?
3294 __ stlxr(rscratch1, src_reg, base);
3295 }
3296 } else {
3297 Register index_reg = as_Register(index);
3298 if (disp == 0) {
3299 __ lea(rscratch2, Address(base, index_reg, Address::lsl(scale)));
3300 __ stlxr(rscratch1, src_reg, rscratch2);
3301 } else {
3302 __ lea(rscratch2, Address(base, disp));
3303 __ lea(rscratch2, Address(rscratch2, index_reg, Address::lsl(scale)));
3304 __ stlxr(rscratch1, src_reg, rscratch2);
3305 }
3306 }
3307 __ cmpw(rscratch1, zr);
3308 %}
3309
3310 enc_class aarch64_enc_cmpxchg(memory mem, iRegLNoSp oldval, iRegLNoSp newval) %{
3311 C2_MacroAssembler _masm(&cbuf);
3312 guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
3313 __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
3314 Assembler::xword, /*acquire*/ false, /*release*/ true,
3315 /*weak*/ false, noreg);
3316 %}
3317
3318 enc_class aarch64_enc_cmpxchgw(memory mem, iRegINoSp oldval, iRegINoSp newval) %{
3319 C2_MacroAssembler _masm(&cbuf);
3320 guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
3321 __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
3322 Assembler::word, /*acquire*/ false, /*release*/ true,
3323 /*weak*/ false, noreg);
3324 %}
3325
3326 enc_class aarch64_enc_cmpxchgs(memory mem, iRegINoSp oldval, iRegINoSp newval) %{
3327 C2_MacroAssembler _masm(&cbuf);
3328 guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
3329 __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
3330 Assembler::halfword, /*acquire*/ false, /*release*/ true,
3331 /*weak*/ false, noreg);
3332 %}
3333
3334 enc_class aarch64_enc_cmpxchgb(memory mem, iRegINoSp oldval, iRegINoSp newval) %{
3335 C2_MacroAssembler _masm(&cbuf);
3336 guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
3337 __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
3338 Assembler::byte, /*acquire*/ false, /*release*/ true,
3339 /*weak*/ false, noreg);
3340 %}
3341
3342
3343 // The only difference between aarch64_enc_cmpxchg and
3344 // aarch64_enc_cmpxchg_acq is that we use load-acquire in the
3345 // CompareAndSwap sequence to serve as a barrier on acquiring a
3346 // lock.
3347 enc_class aarch64_enc_cmpxchg_acq(memory mem, iRegLNoSp oldval, iRegLNoSp newval) %{
3348 C2_MacroAssembler _masm(&cbuf);
3349 guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
3350 __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
3351 Assembler::xword, /*acquire*/ true, /*release*/ true,
3352 /*weak*/ false, noreg);
3353 %}
3354
3355 enc_class aarch64_enc_cmpxchgw_acq(memory mem, iRegINoSp oldval, iRegINoSp newval) %{
3356 C2_MacroAssembler _masm(&cbuf);
3357 guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
3358 __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
3359 Assembler::word, /*acquire*/ true, /*release*/ true,
3360 /*weak*/ false, noreg);
3361 %}
3362
3363 enc_class aarch64_enc_cmpxchgs_acq(memory mem, iRegINoSp oldval, iRegINoSp newval) %{
3364 C2_MacroAssembler _masm(&cbuf);
3365 guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
3366 __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
3367 Assembler::halfword, /*acquire*/ true, /*release*/ true,
3368 /*weak*/ false, noreg);
3369 %}
3370
3371 enc_class aarch64_enc_cmpxchgb_acq(memory mem, iRegINoSp oldval, iRegINoSp newval) %{
3372 C2_MacroAssembler _masm(&cbuf);
3373 guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
3374 __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
3375 Assembler::byte, /*acquire*/ true, /*release*/ true,
3376 /*weak*/ false, noreg);
3377 %}
3378
3379 // auxiliary used for CompareAndSwapX to set result register
3380 enc_class aarch64_enc_cset_eq(iRegINoSp res) %{
3381 C2_MacroAssembler _masm(&cbuf);
3382 Register res_reg = as_Register($res$$reg);
3383 __ cset(res_reg, Assembler::EQ);
3384 %}
3385
3386 // prefetch encodings
3387
3388 enc_class aarch64_enc_prefetchw(memory mem) %{
3389 C2_MacroAssembler _masm(&cbuf);
3390 Register base = as_Register($mem$$base);
3391 int index = $mem$$index;
3392 int scale = $mem$$scale;
3393 int disp = $mem$$disp;
3394 if (index == -1) {
3395 __ prfm(Address(base, disp), PSTL1KEEP);
3396 } else {
3397 Register index_reg = as_Register(index);
3398 if (disp == 0) {
3399 __ prfm(Address(base, index_reg, Address::lsl(scale)), PSTL1KEEP);
3400 } else {
3401 __ lea(rscratch1, Address(base, disp));
3402 __ prfm(Address(rscratch1, index_reg, Address::lsl(scale)), PSTL1KEEP);
3403 }
3404 }
3405 %}
3406
3407 /// mov envcodings
3408
3409 enc_class aarch64_enc_movw_imm(iRegI dst, immI src) %{
3410 C2_MacroAssembler _masm(&cbuf);
3411 uint32_t con = (uint32_t)$src$$constant;
3412 Register dst_reg = as_Register($dst$$reg);
3413 if (con == 0) {
3414 __ movw(dst_reg, zr);
3415 } else {
3416 __ movw(dst_reg, con);
3417 }
3418 %}
3419
3420 enc_class aarch64_enc_mov_imm(iRegL dst, immL src) %{
3421 C2_MacroAssembler _masm(&cbuf);
3422 Register dst_reg = as_Register($dst$$reg);
3423 uint64_t con = (uint64_t)$src$$constant;
3424 if (con == 0) {
3425 __ mov(dst_reg, zr);
3426 } else {
3427 __ mov(dst_reg, con);
3428 }
3429 %}
3430
3431 enc_class aarch64_enc_mov_p(iRegP dst, immP src) %{
3432 C2_MacroAssembler _masm(&cbuf);
3433 Register dst_reg = as_Register($dst$$reg);
3434 address con = (address)$src$$constant;
3435 if (con == NULL || con == (address)1) {
3436 ShouldNotReachHere();
3437 } else {
3438 relocInfo::relocType rtype = $src->constant_reloc();
3439 if (rtype == relocInfo::oop_type) {
3440 __ movoop(dst_reg, (jobject)con, /*immediate*/true);
3441 } else if (rtype == relocInfo::metadata_type) {
3442 __ mov_metadata(dst_reg, (Metadata*)con);
3443 } else {
3444 assert(rtype == relocInfo::none, "unexpected reloc type");
3445 if (! __ is_valid_AArch64_address(con) ||
3446 con < (address)(uintptr_t)os::vm_page_size()) {
3447 __ mov(dst_reg, con);
3448 } else {
3449 uint64_t offset;
3450 __ adrp(dst_reg, con, offset);
3451 __ add(dst_reg, dst_reg, offset);
3452 }
3453 }
3454 }
3455 %}
3456
3457 enc_class aarch64_enc_mov_p0(iRegP dst, immP0 src) %{
3458 C2_MacroAssembler _masm(&cbuf);
3459 Register dst_reg = as_Register($dst$$reg);
3460 __ mov(dst_reg, zr);
3461 %}
3462
3463 enc_class aarch64_enc_mov_p1(iRegP dst, immP_1 src) %{
3464 C2_MacroAssembler _masm(&cbuf);
3465 Register dst_reg = as_Register($dst$$reg);
3466 __ mov(dst_reg, (uint64_t)1);
3467 %}
3468
3469 enc_class aarch64_enc_mov_byte_map_base(iRegP dst, immByteMapBase src) %{
3470 C2_MacroAssembler _masm(&cbuf);
3471 __ load_byte_map_base($dst$$Register);
3472 %}
3473
3474 enc_class aarch64_enc_mov_n(iRegN dst, immN src) %{
3475 C2_MacroAssembler _masm(&cbuf);
3476 Register dst_reg = as_Register($dst$$reg);
3477 address con = (address)$src$$constant;
3478 if (con == NULL) {
3479 ShouldNotReachHere();
3480 } else {
3481 relocInfo::relocType rtype = $src->constant_reloc();
3482 assert(rtype == relocInfo::oop_type, "unexpected reloc type");
3483 __ set_narrow_oop(dst_reg, (jobject)con);
3484 }
3485 %}
3486
3487 enc_class aarch64_enc_mov_n0(iRegN dst, immN0 src) %{
3488 C2_MacroAssembler _masm(&cbuf);
3489 Register dst_reg = as_Register($dst$$reg);
3490 __ mov(dst_reg, zr);
3491 %}
3492
3493 enc_class aarch64_enc_mov_nk(iRegN dst, immNKlass src) %{
3494 C2_MacroAssembler _masm(&cbuf);
3495 Register dst_reg = as_Register($dst$$reg);
3496 address con = (address)$src$$constant;
3497 if (con == NULL) {
3498 ShouldNotReachHere();
3499 } else {
3500 relocInfo::relocType rtype = $src->constant_reloc();
3501 assert(rtype == relocInfo::metadata_type, "unexpected reloc type");
3502 __ set_narrow_klass(dst_reg, (Klass *)con);
3503 }
3504 %}
3505
3506 // arithmetic encodings
3507
3508 enc_class aarch64_enc_addsubw_imm(iRegI dst, iRegI src1, immIAddSub src2) %{
3509 C2_MacroAssembler _masm(&cbuf);
3510 Register dst_reg = as_Register($dst$$reg);
3511 Register src_reg = as_Register($src1$$reg);
3512 int32_t con = (int32_t)$src2$$constant;
3513 // add has primary == 0, subtract has primary == 1
3514 if ($primary) { con = -con; }
3515 if (con < 0) {
3516 __ subw(dst_reg, src_reg, -con);
3517 } else {
3518 __ addw(dst_reg, src_reg, con);
3519 }
3520 %}
3521
3522 enc_class aarch64_enc_addsub_imm(iRegL dst, iRegL src1, immLAddSub src2) %{
3523 C2_MacroAssembler _masm(&cbuf);
3524 Register dst_reg = as_Register($dst$$reg);
3525 Register src_reg = as_Register($src1$$reg);
3526 int32_t con = (int32_t)$src2$$constant;
3527 // add has primary == 0, subtract has primary == 1
3528 if ($primary) { con = -con; }
3529 if (con < 0) {
3530 __ sub(dst_reg, src_reg, -con);
3531 } else {
3532 __ add(dst_reg, src_reg, con);
3533 }
3534 %}
3535
3536 enc_class aarch64_enc_divw(iRegI dst, iRegI src1, iRegI src2) %{
3537 C2_MacroAssembler _masm(&cbuf);
3538 Register dst_reg = as_Register($dst$$reg);
3539 Register src1_reg = as_Register($src1$$reg);
3540 Register src2_reg = as_Register($src2$$reg);
3541 __ corrected_idivl(dst_reg, src1_reg, src2_reg, false, rscratch1);
3542 %}
3543
3544 enc_class aarch64_enc_div(iRegI dst, iRegI src1, iRegI src2) %{
3545 C2_MacroAssembler _masm(&cbuf);
3546 Register dst_reg = as_Register($dst$$reg);
3547 Register src1_reg = as_Register($src1$$reg);
3548 Register src2_reg = as_Register($src2$$reg);
3549 __ corrected_idivq(dst_reg, src1_reg, src2_reg, false, rscratch1);
3550 %}
3551
3552 enc_class aarch64_enc_modw(iRegI dst, iRegI src1, iRegI src2) %{
3553 C2_MacroAssembler _masm(&cbuf);
3554 Register dst_reg = as_Register($dst$$reg);
3555 Register src1_reg = as_Register($src1$$reg);
3556 Register src2_reg = as_Register($src2$$reg);
3557 __ corrected_idivl(dst_reg, src1_reg, src2_reg, true, rscratch1);
3558 %}
3559
3560 enc_class aarch64_enc_mod(iRegI dst, iRegI src1, iRegI src2) %{
3561 C2_MacroAssembler _masm(&cbuf);
3562 Register dst_reg = as_Register($dst$$reg);
3563 Register src1_reg = as_Register($src1$$reg);
3564 Register src2_reg = as_Register($src2$$reg);
3565 __ corrected_idivq(dst_reg, src1_reg, src2_reg, true, rscratch1);
3566 %}
3567
3568 // compare instruction encodings
3569
3570 enc_class aarch64_enc_cmpw(iRegI src1, iRegI src2) %{
3571 C2_MacroAssembler _masm(&cbuf);
3572 Register reg1 = as_Register($src1$$reg);
3573 Register reg2 = as_Register($src2$$reg);
3574 __ cmpw(reg1, reg2);
3575 %}
3576
3577 enc_class aarch64_enc_cmpw_imm_addsub(iRegI src1, immIAddSub src2) %{
3578 C2_MacroAssembler _masm(&cbuf);
3579 Register reg = as_Register($src1$$reg);
3580 int32_t val = $src2$$constant;
3581 if (val >= 0) {
3582 __ subsw(zr, reg, val);
3583 } else {
3584 __ addsw(zr, reg, -val);
3585 }
3586 %}
3587
3588 enc_class aarch64_enc_cmpw_imm(iRegI src1, immI src2) %{
3589 C2_MacroAssembler _masm(&cbuf);
3590 Register reg1 = as_Register($src1$$reg);
3591 uint32_t val = (uint32_t)$src2$$constant;
3592 __ movw(rscratch1, val);
3593 __ cmpw(reg1, rscratch1);
3594 %}
3595
3596 enc_class aarch64_enc_cmp(iRegL src1, iRegL src2) %{
3597 C2_MacroAssembler _masm(&cbuf);
3598 Register reg1 = as_Register($src1$$reg);
3599 Register reg2 = as_Register($src2$$reg);
3600 __ cmp(reg1, reg2);
3601 %}
3602
3603 enc_class aarch64_enc_cmp_imm_addsub(iRegL src1, immL12 src2) %{
3604 C2_MacroAssembler _masm(&cbuf);
3605 Register reg = as_Register($src1$$reg);
3606 int64_t val = $src2$$constant;
3607 if (val >= 0) {
3608 __ subs(zr, reg, val);
3609 } else if (val != -val) {
3610 __ adds(zr, reg, -val);
3611 } else {
3612 // aargh, Long.MIN_VALUE is a special case
3613 __ orr(rscratch1, zr, (uint64_t)val);
3614 __ subs(zr, reg, rscratch1);
3615 }
3616 %}
3617
3618 enc_class aarch64_enc_cmp_imm(iRegL src1, immL src2) %{
3619 C2_MacroAssembler _masm(&cbuf);
3620 Register reg1 = as_Register($src1$$reg);
3621 uint64_t val = (uint64_t)$src2$$constant;
3622 __ mov(rscratch1, val);
3623 __ cmp(reg1, rscratch1);
3624 %}
3625
3626 enc_class aarch64_enc_cmpp(iRegP src1, iRegP src2) %{
3627 C2_MacroAssembler _masm(&cbuf);
3628 Register reg1 = as_Register($src1$$reg);
3629 Register reg2 = as_Register($src2$$reg);
3630 __ cmp(reg1, reg2);
3631 %}
3632
3633 enc_class aarch64_enc_cmpn(iRegN src1, iRegN src2) %{
3634 C2_MacroAssembler _masm(&cbuf);
3635 Register reg1 = as_Register($src1$$reg);
3636 Register reg2 = as_Register($src2$$reg);
3637 __ cmpw(reg1, reg2);
3638 %}
3639
3640 enc_class aarch64_enc_testp(iRegP src) %{
3641 C2_MacroAssembler _masm(&cbuf);
3642 Register reg = as_Register($src$$reg);
3643 __ cmp(reg, zr);
3644 %}
3645
3646 enc_class aarch64_enc_testn(iRegN src) %{
3647 C2_MacroAssembler _masm(&cbuf);
3648 Register reg = as_Register($src$$reg);
3649 __ cmpw(reg, zr);
3650 %}
3651
3652 enc_class aarch64_enc_b(label lbl) %{
3653 C2_MacroAssembler _masm(&cbuf);
3654 Label *L = $lbl$$label;
3655 __ b(*L);
3656 %}
3657
3658 enc_class aarch64_enc_br_con(cmpOp cmp, label lbl) %{
3659 C2_MacroAssembler _masm(&cbuf);
3660 Label *L = $lbl$$label;
3661 __ br ((Assembler::Condition)$cmp$$cmpcode, *L);
3662 %}
3663
3664 enc_class aarch64_enc_br_conU(cmpOpU cmp, label lbl) %{
3665 C2_MacroAssembler _masm(&cbuf);
3666 Label *L = $lbl$$label;
3667 __ br ((Assembler::Condition)$cmp$$cmpcode, *L);
3668 %}
3669
3670 enc_class aarch64_enc_partial_subtype_check(iRegP sub, iRegP super, iRegP temp, iRegP result)
3671 %{
3672 Register sub_reg = as_Register($sub$$reg);
3673 Register super_reg = as_Register($super$$reg);
3674 Register temp_reg = as_Register($temp$$reg);
3675 Register result_reg = as_Register($result$$reg);
3676
3677 Label miss;
3678 C2_MacroAssembler _masm(&cbuf);
3679 __ check_klass_subtype_slow_path(sub_reg, super_reg, temp_reg, result_reg,
3680 NULL, &miss,
3681 /*set_cond_codes:*/ true);
3682 if ($primary) {
3683 __ mov(result_reg, zr);
3684 }
3685 __ bind(miss);
3686 %}
3687
3688 enc_class aarch64_enc_java_static_call(method meth) %{
3689 C2_MacroAssembler _masm(&cbuf);
3690
3691 address addr = (address)$meth$$method;
3692 address call;
3693 if (!_method) {
3694 // A call to a runtime wrapper, e.g. new, new_typeArray_Java, uncommon_trap.
3695 call = __ trampoline_call(Address(addr, relocInfo::runtime_call_type), &cbuf);
3696 if (call == NULL) {
3697 ciEnv::current()->record_failure("CodeCache is full");
3698 return;
3699 }
3700 } else {
3701 int method_index = resolved_method_index(cbuf);
3702 RelocationHolder rspec = _optimized_virtual ? opt_virtual_call_Relocation::spec(method_index)
3703 : static_call_Relocation::spec(method_index);
3704 call = __ trampoline_call(Address(addr, rspec), &cbuf);
3705 if (call == NULL) {
3706 ciEnv::current()->record_failure("CodeCache is full");
3707 return;
3708 }
3709 // Emit stub for static call
3710 address stub = CompiledStaticCall::emit_to_interp_stub(cbuf);
3711 if (stub == NULL) {
3712 ciEnv::current()->record_failure("CodeCache is full");
3713 return;
3714 }
3715 }
3716
3717 // Only non uncommon_trap calls need to reinitialize ptrue.
3718 if (Compile::current()->max_vector_size() >= 16 && uncommon_trap_request() == 0) {
3719 __ reinitialize_ptrue();
3720 }
3721 %}
3722
3723 enc_class aarch64_enc_java_dynamic_call(method meth) %{
3724 C2_MacroAssembler _masm(&cbuf);
3725 int method_index = resolved_method_index(cbuf);
3726 address call = __ ic_call((address)$meth$$method, method_index);
3727 if (call == NULL) {
3728 ciEnv::current()->record_failure("CodeCache is full");
3729 return;
3730 } else if (Compile::current()->max_vector_size() >= 16) {
3731 __ reinitialize_ptrue();
3732 }
3733 %}
3734
3735 enc_class aarch64_enc_call_epilog() %{
3736 C2_MacroAssembler _masm(&cbuf);
3737 if (VerifyStackAtCalls) {
3738 // Check that stack depth is unchanged: find majik cookie on stack
3739 __ call_Unimplemented();
3740 }
3741 %}
3742
3743 enc_class aarch64_enc_java_to_runtime(method meth) %{
3744 C2_MacroAssembler _masm(&cbuf);
3745
3746 // some calls to generated routines (arraycopy code) are scheduled
3747 // by C2 as runtime calls. if so we can call them using a br (they
3748 // will be in a reachable segment) otherwise we have to use a blr
3749 // which loads the absolute address into a register.
3750 address entry = (address)$meth$$method;
3751 CodeBlob *cb = CodeCache::find_blob(entry);
3752 if (cb) {
3753 address call = __ trampoline_call(Address(entry, relocInfo::runtime_call_type));
3754 if (call == NULL) {
3755 ciEnv::current()->record_failure("CodeCache is full");
3756 return;
3757 }
3758 } else {
3759 Label retaddr;
3760 __ adr(rscratch2, retaddr);
3761 __ lea(rscratch1, RuntimeAddress(entry));
3762 // Leave a breadcrumb for JavaFrameAnchor::capture_last_Java_pc()
3763 __ stp(zr, rscratch2, Address(__ pre(sp, -2 * wordSize)));
3764 __ blr(rscratch1);
3765 __ bind(retaddr);
3766 __ add(sp, sp, 2 * wordSize);
3767 }
3768 if (Compile::current()->max_vector_size() >= 16) {
3769 __ reinitialize_ptrue();
3770 }
3771 %}
3772
3773 enc_class aarch64_enc_rethrow() %{
3774 C2_MacroAssembler _masm(&cbuf);
3775 __ far_jump(RuntimeAddress(OptoRuntime::rethrow_stub()));
3776 %}
3777
3778 enc_class aarch64_enc_ret() %{
3779 C2_MacroAssembler _masm(&cbuf);
3780 #ifdef ASSERT
3781 if (Compile::current()->max_vector_size() >= 16) {
3782 __ verify_ptrue();
3783 }
3784 #endif
3785 __ ret(lr);
3786 %}
3787
3788 enc_class aarch64_enc_tail_call(iRegP jump_target) %{
3789 C2_MacroAssembler _masm(&cbuf);
3790 Register target_reg = as_Register($jump_target$$reg);
3791 __ br(target_reg);
3792 %}
3793
3794 enc_class aarch64_enc_tail_jmp(iRegP jump_target) %{
3795 C2_MacroAssembler _masm(&cbuf);
3796 Register target_reg = as_Register($jump_target$$reg);
3797 // exception oop should be in r0
3798 // ret addr has been popped into lr
3799 // callee expects it in r3
3800 __ mov(r3, lr);
3801 __ br(target_reg);
3802 %}
3803
3804 enc_class aarch64_enc_fast_lock(iRegP object, iRegP box, iRegP tmp, iRegP tmp2) %{
3805 C2_MacroAssembler _masm(&cbuf);
3806 Register oop = as_Register($object$$reg);
3807 Register box = as_Register($box$$reg);
3808 Register disp_hdr = as_Register($tmp$$reg);
3809 Register tmp = as_Register($tmp2$$reg);
3810 Label cont;
3811 Label object_has_monitor;
3812 Label cas_failed;
3813
3814 assert_different_registers(oop, box, tmp, disp_hdr);
3815
3816 // Load markWord from object into displaced_header.
3817 __ ldr(disp_hdr, Address(oop, oopDesc::mark_offset_in_bytes()));
3818
3819 if (DiagnoseSyncOnValueBasedClasses != 0) {
3820 __ load_klass(tmp, oop);
3821 __ ldrw(tmp, Address(tmp, Klass::access_flags_offset()));
3822 __ tstw(tmp, JVM_ACC_IS_VALUE_BASED_CLASS);
3823 __ br(Assembler::NE, cont);
3824 }
3825
3826 if (UseBiasedLocking && !UseOptoBiasInlining) {
3827 __ biased_locking_enter(box, oop, disp_hdr, tmp, true, cont);
3828 }
3829
3830 // Check for existing monitor
3831 __ tbnz(disp_hdr, exact_log2(markWord::monitor_value), object_has_monitor);
3832
3833 if (LockingMode == LM_MONITOR) {
3834 __ tst(oop, oop); // Set NE to indicate 'failure' -> take slow-path. We know that oop != 0.
3835 __ b(cont);
3836 } else if (LockingMode == LM_LEGACY) {
3837 // Set tmp to be (markWord of object | UNLOCK_VALUE).
3838 __ orr(tmp, disp_hdr, markWord::unlocked_value);
3839
3840 // Initialize the box. (Must happen before we update the object mark!)
3841 __ str(tmp, Address(box, BasicLock::displaced_header_offset_in_bytes()));
3842
3843 // Compare object markWord with an unlocked value (tmp) and if
3844 // equal exchange the stack address of our box with object markWord.
3845 // On failure disp_hdr contains the possibly locked markWord.
3846 __ cmpxchg(oop, tmp, box, Assembler::xword, /*acquire*/ true,
3847 /*release*/ true, /*weak*/ false, disp_hdr);
3848 __ br(Assembler::EQ, cont);
3849
3850 assert(oopDesc::mark_offset_in_bytes() == 0, "offset of _mark is not 0");
3851
3852 // If the compare-and-exchange succeeded, then we found an unlocked
3853 // object, will have now locked it will continue at label cont
3854
3855 __ bind(cas_failed);
3856 // We did not see an unlocked object so try the fast recursive case.
3857
3858 // Check if the owner is self by comparing the value in the
3859 // markWord of object (disp_hdr) with the stack pointer.
3860 __ mov(rscratch1, sp);
3861 __ sub(disp_hdr, disp_hdr, rscratch1);
3862 __ mov(tmp, (address) (~(os::vm_page_size()-1) | markWord::lock_mask_in_place));
3863 // If condition is true we are cont and hence we can store 0 as the
3864 // displaced header in the box, which indicates that it is a recursive lock.
3865 __ ands(tmp/*==0?*/, disp_hdr, tmp); // Sets flags for result
3866 __ str(tmp/*==0, perhaps*/, Address(box, BasicLock::displaced_header_offset_in_bytes()));
3867 __ b(cont);
3868 } else {
3869 assert(LockingMode == LM_LIGHTWEIGHT, "must be");
3870 __ fast_lock(oop, disp_hdr, tmp, rscratch1, cont);
3871 __ b(cont);
3872 }
3873
3874 // Handle existing monitor.
3875 __ bind(object_has_monitor);
3876
3877 // The object's monitor m is unlocked iff m->owner == NULL,
3878 // otherwise m->owner may contain a thread or a stack address.
3879 //
3880 // Try to CAS m->owner from NULL to current thread.
3881 __ add(tmp, disp_hdr, (ObjectMonitor::owner_offset_in_bytes()-markWord::monitor_value));
3882 __ cmpxchg(tmp, zr, rthread, Assembler::xword, /*acquire*/ true,
3883 /*release*/ true, /*weak*/ false, rscratch1); // Sets flags for result
3884
3885 if (LockingMode != LM_LIGHTWEIGHT) {
3886 // Store a non-null value into the box to avoid looking like a re-entrant
3887 // lock. The fast-path monitor unlock code checks for
3888 // markWord::monitor_value so use markWord::unused_mark which has the
3889 // relevant bit set, and also matches ObjectSynchronizer::enter.
3890 __ mov(tmp, (address)markWord::unused_mark().value());
3891 __ str(tmp, Address(box, BasicLock::displaced_header_offset_in_bytes()));
3892 }
3893 __ br(Assembler::EQ, cont); // CAS success means locking succeeded
3894
3895 __ cmp(rscratch1, rthread);
3896 __ br(Assembler::NE, cont); // Check for recursive locking
3897
3898 // Recursive lock case
3899 __ increment(Address(disp_hdr, ObjectMonitor::recursions_offset_in_bytes() - markWord::monitor_value), 1);
3900 // flag == EQ still from the cmp above, checking if this is a reentrant lock
3901
3902 __ bind(cont);
3903 // flag == EQ indicates success
3904 // flag == NE indicates failure
3905 %}
3906
3907 enc_class aarch64_enc_fast_unlock(iRegP object, iRegP box, iRegP tmp, iRegP tmp2) %{
3908 C2_MacroAssembler _masm(&cbuf);
3909 Register oop = as_Register($object$$reg);
3910 Register box = as_Register($box$$reg);
3911 Register disp_hdr = as_Register($tmp$$reg);
3912 Register tmp = as_Register($tmp2$$reg);
3913 Label cont;
3914 Label object_has_monitor;
3915
3916 assert_different_registers(oop, box, tmp, disp_hdr);
3917
3918 if (UseBiasedLocking && !UseOptoBiasInlining) {
3919 __ biased_locking_exit(oop, tmp, cont);
3920 }
3921
3922 if (LockingMode == LM_LEGACY) {
3923 // Find the lock address and load the displaced header from the stack.
3924 __ ldr(disp_hdr, Address(box, BasicLock::displaced_header_offset_in_bytes()));
3925
3926 // If the displaced header is 0, we have a recursive unlock.
3927 __ cmp(disp_hdr, zr);
3928 __ br(Assembler::EQ, cont);
3929 }
3930
3931 // Handle existing monitor.
3932 __ ldr(tmp, Address(oop, oopDesc::mark_offset_in_bytes()));
3933 __ tbnz(tmp, exact_log2(markWord::monitor_value), object_has_monitor);
3934
3935 if (LockingMode == LM_MONITOR) {
3936 __ tst(oop, oop); // Set NE to indicate 'failure' -> take slow-path. We know that oop != 0.
3937 __ b(cont);
3938 } else if (LockingMode == LM_LEGACY) {
3939 // Check if it is still a light weight lock, this is is true if we
3940 // see the stack address of the basicLock in the markWord of the
3941 // object.
3942
3943 __ cmpxchg(oop, box, disp_hdr, Assembler::xword, /*acquire*/ false,
3944 /*release*/ true, /*weak*/ false, tmp);
3945 __ b(cont);
3946 } else {
3947 assert(LockingMode == LM_LIGHTWEIGHT, "must be");
3948 __ fast_unlock(oop, tmp, box, disp_hdr, cont);
3949 __ b(cont);
3950 }
3951
3952 assert(oopDesc::mark_offset_in_bytes() == 0, "offset of _mark is not 0");
3953
3954 // Handle existing monitor.
3955 __ bind(object_has_monitor);
3956 STATIC_ASSERT(markWord::monitor_value <= INT_MAX);
3957 __ add(tmp, tmp, -(int)markWord::monitor_value); // monitor
3958
3959 if (LockingMode == LM_LIGHTWEIGHT) {
3960 // If the owner is anonymous, we need to fix it -- in an outline stub.
3961 Register tmp2 = disp_hdr;
3962 __ ldr(tmp2, Address(tmp, ObjectMonitor::owner_offset_in_bytes()));
3963 // We cannot use tbnz here, the target might be too far away and cannot
3964 // be encoded.
3965 __ tst(tmp2, (uint64_t)ObjectMonitor::ANONYMOUS_OWNER);
3966 C2HandleAnonOMOwnerStub* stub = new (Compile::current()->comp_arena()) C2HandleAnonOMOwnerStub(tmp, tmp2);
3967 Compile::current()->output()->add_stub(stub);
3968 __ br(Assembler::NE, stub->entry());
3969 __ bind(stub->continuation());
3970 }
3971
3972 __ ldr(disp_hdr, Address(tmp, ObjectMonitor::recursions_offset_in_bytes()));
3973
3974 Label notRecursive;
3975 __ cbz(disp_hdr, notRecursive);
3976
3977 // Recursive lock
3978 __ sub(disp_hdr, disp_hdr, 1u);
3979 __ str(disp_hdr, Address(tmp, ObjectMonitor::recursions_offset_in_bytes()));
3980 __ cmp(disp_hdr, disp_hdr); // Sets flags for result
3981 __ b(cont);
3982
3983 __ bind(notRecursive);
3984 __ ldr(rscratch1, Address(tmp, ObjectMonitor::EntryList_offset_in_bytes()));
3985 __ ldr(disp_hdr, Address(tmp, ObjectMonitor::cxq_offset_in_bytes()));
3986 __ orr(rscratch1, rscratch1, disp_hdr); // Will be 0 if both are 0.
3987 __ cmp(rscratch1, zr); // Sets flags for result
3988 __ cbnz(rscratch1, cont);
3989 // need a release store here
3990 __ lea(tmp, Address(tmp, ObjectMonitor::owner_offset_in_bytes()));
3991 __ stlr(zr, tmp); // set unowned
3992
3993 __ bind(cont);
3994 // flag == EQ indicates success
3995 // flag == NE indicates failure
3996 %}
3997
3998 %}
3999
4000 //----------FRAME--------------------------------------------------------------
4001 // Definition of frame structure and management information.
4002 //
4003 // S T A C K L A Y O U T Allocators stack-slot number
4004 // | (to get allocators register number
4005 // G Owned by | | v add OptoReg::stack0())
4006 // r CALLER | |
4007 // o | +--------+ pad to even-align allocators stack-slot
4008 // w V | pad0 | numbers; owned by CALLER
4009 // t -----------+--------+----> Matcher::_in_arg_limit, unaligned
4010 // h ^ | in | 5
4011 // | | args | 4 Holes in incoming args owned by SELF
4012 // | | | | 3
4013 // | | +--------+
4014 // V | | old out| Empty on Intel, window on Sparc
4015 // | old |preserve| Must be even aligned.
4016 // | SP-+--------+----> Matcher::_old_SP, even aligned
4017 // | | in | 3 area for Intel ret address
4018 // Owned by |preserve| Empty on Sparc.
4019 // SELF +--------+
4020 // | | pad2 | 2 pad to align old SP
4021 // | +--------+ 1
4022 // | | locks | 0
4023 // | +--------+----> OptoReg::stack0(), even aligned
4024 // | | pad1 | 11 pad to align new SP
4025 // | +--------+
4026 // | | | 10
4027 // | | spills | 9 spills
4028 // V | | 8 (pad0 slot for callee)
4029 // -----------+--------+----> Matcher::_out_arg_limit, unaligned
4030 // ^ | out | 7
4031 // | | args | 6 Holes in outgoing args owned by CALLEE
4032 // Owned by +--------+
4033 // CALLEE | new out| 6 Empty on Intel, window on Sparc
4034 // | new |preserve| Must be even-aligned.
4035 // | SP-+--------+----> Matcher::_new_SP, even aligned
4036 // | | |
4037 //
4038 // Note 1: Only region 8-11 is determined by the allocator. Region 0-5 is
4039 // known from SELF's arguments and the Java calling convention.
4040 // Region 6-7 is determined per call site.
4041 // Note 2: If the calling convention leaves holes in the incoming argument
4042 // area, those holes are owned by SELF. Holes in the outgoing area
4043 // are owned by the CALLEE. Holes should not be nessecary in the
4044 // incoming area, as the Java calling convention is completely under
4045 // the control of the AD file. Doubles can be sorted and packed to
4046 // avoid holes. Holes in the outgoing arguments may be nessecary for
4047 // varargs C calling conventions.
4048 // Note 3: Region 0-3 is even aligned, with pad2 as needed. Region 3-5 is
4049 // even aligned with pad0 as needed.
4050 // Region 6 is even aligned. Region 6-7 is NOT even aligned;
4051 // (the latter is true on Intel but is it false on AArch64?)
4052 // region 6-11 is even aligned; it may be padded out more so that
4053 // the region from SP to FP meets the minimum stack alignment.
4054 // Note 4: For I2C adapters, the incoming FP may not meet the minimum stack
4055 // alignment. Region 11, pad1, may be dynamically extended so that
4056 // SP meets the minimum alignment.
4057
4058 frame %{
4059 // These three registers define part of the calling convention
4060 // between compiled code and the interpreter.
4061
4062 // Inline Cache Register or Method for I2C.
4063 inline_cache_reg(R12);
4064
4065 // Number of stack slots consumed by locking an object
4066 sync_stack_slots(2);
4067
4068 // Compiled code's Frame Pointer
4069 frame_pointer(R31);
4070
4071 // Interpreter stores its frame pointer in a register which is
4072 // stored to the stack by I2CAdaptors.
4073 // I2CAdaptors convert from interpreted java to compiled java.
4074 interpreter_frame_pointer(R29);
4075
4076 // Stack alignment requirement
4077 stack_alignment(StackAlignmentInBytes); // Alignment size in bytes (128-bit -> 16 bytes)
4078
4079 // Number of outgoing stack slots killed above the out_preserve_stack_slots
4080 // for calls to C. Supports the var-args backing area for register parms.
4081 varargs_C_out_slots_killed(frame::arg_reg_save_area_bytes/BytesPerInt);
4082
4083 // The after-PROLOG location of the return address. Location of
4084 // return address specifies a type (REG or STACK) and a number
4085 // representing the register number (i.e. - use a register name) or
4086 // stack slot.
4087 // Ret Addr is on stack in slot 0 if no locks or verification or alignment.
4088 // Otherwise, it is above the locks and verification slot and alignment word
4089 // TODO this may well be correct but need to check why that - 2 is there
4090 // ppc port uses 0 but we definitely need to allow for fixed_slots
4091 // which folds in the space used for monitors
4092 return_addr(STACK - 2 +
4093 align_up((Compile::current()->in_preserve_stack_slots() +
4094 Compile::current()->fixed_slots()),
4095 stack_alignment_in_slots()));
4096
4097 // Location of compiled Java return values. Same as C for now.
4098 return_value
4099 %{
4100 // TODO do we allow ideal_reg == Op_RegN???
4101 assert(ideal_reg >= Op_RegI && ideal_reg <= Op_RegL,
4102 "only return normal values");
4103
4104 static const int lo[Op_RegL + 1] = { // enum name
4105 0, // Op_Node
4106 0, // Op_Set
4107 R0_num, // Op_RegN
4108 R0_num, // Op_RegI
4109 R0_num, // Op_RegP
4110 V0_num, // Op_RegF
4111 V0_num, // Op_RegD
4112 R0_num // Op_RegL
4113 };
4114
4115 static const int hi[Op_RegL + 1] = { // enum name
4116 0, // Op_Node
4117 0, // Op_Set
4118 OptoReg::Bad, // Op_RegN
4119 OptoReg::Bad, // Op_RegI
4120 R0_H_num, // Op_RegP
4121 OptoReg::Bad, // Op_RegF
4122 V0_H_num, // Op_RegD
4123 R0_H_num // Op_RegL
4124 };
4125
4126 return OptoRegPair(hi[ideal_reg], lo[ideal_reg]);
4127 %}
4128 %}
4129
4130 //----------ATTRIBUTES---------------------------------------------------------
4131 //----------Operand Attributes-------------------------------------------------
4132 op_attrib op_cost(1); // Required cost attribute
4133
4134 //----------Instruction Attributes---------------------------------------------
4135 ins_attrib ins_cost(INSN_COST); // Required cost attribute
4136 ins_attrib ins_size(32); // Required size attribute (in bits)
4137 ins_attrib ins_short_branch(0); // Required flag: is this instruction
4138 // a non-matching short branch variant
4139 // of some long branch?
4140 ins_attrib ins_alignment(4); // Required alignment attribute (must
4141 // be a power of 2) specifies the
4142 // alignment that some part of the
4143 // instruction (not necessarily the
4144 // start) requires. If > 1, a
4145 // compute_padding() function must be
4146 // provided for the instruction
4147
4148 //----------OPERANDS-----------------------------------------------------------
4149 // Operand definitions must precede instruction definitions for correct parsing
4150 // in the ADLC because operands constitute user defined types which are used in
4151 // instruction definitions.
4152
4153 //----------Simple Operands----------------------------------------------------
4154
4155 // Integer operands 32 bit
4156 // 32 bit immediate
4157 operand immI()
4158 %{
4159 match(ConI);
4160
4161 op_cost(0);
4162 format %{ %}
4163 interface(CONST_INTER);
4164 %}
4165
4166 // 32 bit zero
4167 operand immI0()
4168 %{
4169 predicate(n->get_int() == 0);
4170 match(ConI);
4171
4172 op_cost(0);
4173 format %{ %}
4174 interface(CONST_INTER);
4175 %}
4176
4177 // 32 bit unit increment
4178 operand immI_1()
4179 %{
4180 predicate(n->get_int() == 1);
4181 match(ConI);
4182
4183 op_cost(0);
4184 format %{ %}
4185 interface(CONST_INTER);
4186 %}
4187
4188 // 32 bit unit decrement
4189 operand immI_M1()
4190 %{
4191 predicate(n->get_int() == -1);
4192 match(ConI);
4193
4194 op_cost(0);
4195 format %{ %}
4196 interface(CONST_INTER);
4197 %}
4198
4199 // Shift values for add/sub extension shift
4200 operand immIExt()
4201 %{
4202 predicate(0 <= n->get_int() && (n->get_int() <= 4));
4203 match(ConI);
4204
4205 op_cost(0);
4206 format %{ %}
4207 interface(CONST_INTER);
4208 %}
4209
4210 operand immI_le_4()
4211 %{
4212 predicate(n->get_int() <= 4);
4213 match(ConI);
4214
4215 op_cost(0);
4216 format %{ %}
4217 interface(CONST_INTER);
4218 %}
4219
4220 operand immI_31()
4221 %{
4222 predicate(n->get_int() == 31);
4223 match(ConI);
4224
4225 op_cost(0);
4226 format %{ %}
4227 interface(CONST_INTER);
4228 %}
4229
4230 operand immI_2()
4231 %{
4232 predicate(n->get_int() == 2);
4233 match(ConI);
4234
4235 op_cost(0);
4236 format %{ %}
4237 interface(CONST_INTER);
4238 %}
4239
4240 operand immI_4()
4241 %{
4242 predicate(n->get_int() == 4);
4243 match(ConI);
4244
4245 op_cost(0);
4246 format %{ %}
4247 interface(CONST_INTER);
4248 %}
4249
4250 operand immI_8()
4251 %{
4252 predicate(n->get_int() == 8);
4253 match(ConI);
4254
4255 op_cost(0);
4256 format %{ %}
4257 interface(CONST_INTER);
4258 %}
4259
4260 operand immI_16()
4261 %{
4262 predicate(n->get_int() == 16);
4263 match(ConI);
4264
4265 op_cost(0);
4266 format %{ %}
4267 interface(CONST_INTER);
4268 %}
4269
4270 operand immI_24()
4271 %{
4272 predicate(n->get_int() == 24);
4273 match(ConI);
4274
4275 op_cost(0);
4276 format %{ %}
4277 interface(CONST_INTER);
4278 %}
4279
4280 operand immI_32()
4281 %{
4282 predicate(n->get_int() == 32);
4283 match(ConI);
4284
4285 op_cost(0);
4286 format %{ %}
4287 interface(CONST_INTER);
4288 %}
4289
4290 operand immI_48()
4291 %{
4292 predicate(n->get_int() == 48);
4293 match(ConI);
4294
4295 op_cost(0);
4296 format %{ %}
4297 interface(CONST_INTER);
4298 %}
4299
4300 operand immI_56()
4301 %{
4302 predicate(n->get_int() == 56);
4303 match(ConI);
4304
4305 op_cost(0);
4306 format %{ %}
4307 interface(CONST_INTER);
4308 %}
4309
4310 operand immI_63()
4311 %{
4312 predicate(n->get_int() == 63);
4313 match(ConI);
4314
4315 op_cost(0);
4316 format %{ %}
4317 interface(CONST_INTER);
4318 %}
4319
4320 operand immI_64()
4321 %{
4322 predicate(n->get_int() == 64);
4323 match(ConI);
4324
4325 op_cost(0);
4326 format %{ %}
4327 interface(CONST_INTER);
4328 %}
4329
4330 operand immI_255()
4331 %{
4332 predicate(n->get_int() == 255);
4333 match(ConI);
4334
4335 op_cost(0);
4336 format %{ %}
4337 interface(CONST_INTER);
4338 %}
4339
4340 operand immI_65535()
4341 %{
4342 predicate(n->get_int() == 65535);
4343 match(ConI);
4344
4345 op_cost(0);
4346 format %{ %}
4347 interface(CONST_INTER);
4348 %}
4349
4350 operand immI_positive()
4351 %{
4352 predicate(n->get_int() > 0);
4353 match(ConI);
4354
4355 op_cost(0);
4356 format %{ %}
4357 interface(CONST_INTER);
4358 %}
4359
4360 operand immL_255()
4361 %{
4362 predicate(n->get_long() == 255L);
4363 match(ConL);
4364
4365 op_cost(0);
4366 format %{ %}
4367 interface(CONST_INTER);
4368 %}
4369
4370 operand immL_65535()
4371 %{
4372 predicate(n->get_long() == 65535L);
4373 match(ConL);
4374
4375 op_cost(0);
4376 format %{ %}
4377 interface(CONST_INTER);
4378 %}
4379
4380 operand immL_4294967295()
4381 %{
4382 predicate(n->get_long() == 4294967295L);
4383 match(ConL);
4384
4385 op_cost(0);
4386 format %{ %}
4387 interface(CONST_INTER);
4388 %}
4389
4390 operand immL_bitmask()
4391 %{
4392 predicate((n->get_long() != 0)
4393 && ((n->get_long() & 0xc000000000000000l) == 0)
4394 && is_power_of_2(n->get_long() + 1));
4395 match(ConL);
4396
4397 op_cost(0);
4398 format %{ %}
4399 interface(CONST_INTER);
4400 %}
4401
4402 operand immI_bitmask()
4403 %{
4404 predicate((n->get_int() != 0)
4405 && ((n->get_int() & 0xc0000000) == 0)
4406 && is_power_of_2(n->get_int() + 1));
4407 match(ConI);
4408
4409 op_cost(0);
4410 format %{ %}
4411 interface(CONST_INTER);
4412 %}
4413
4414 operand immL_positive_bitmaskI()
4415 %{
4416 predicate((n->get_long() != 0)
4417 && ((julong)n->get_long() < 0x80000000ULL)
4418 && is_power_of_2(n->get_long() + 1));
4419 match(ConL);
4420
4421 op_cost(0);
4422 format %{ %}
4423 interface(CONST_INTER);
4424 %}
4425
4426 // Scale values for scaled offset addressing modes (up to long but not quad)
4427 operand immIScale()
4428 %{
4429 predicate(0 <= n->get_int() && (n->get_int() <= 3));
4430 match(ConI);
4431
4432 op_cost(0);
4433 format %{ %}
4434 interface(CONST_INTER);
4435 %}
4436
4437 // 26 bit signed offset -- for pc-relative branches
4438 operand immI26()
4439 %{
4440 predicate(((-(1 << 25)) <= n->get_int()) && (n->get_int() < (1 << 25)));
4441 match(ConI);
4442
4443 op_cost(0);
4444 format %{ %}
4445 interface(CONST_INTER);
4446 %}
4447
4448 // 19 bit signed offset -- for pc-relative loads
4449 operand immI19()
4450 %{
4451 predicate(((-(1 << 18)) <= n->get_int()) && (n->get_int() < (1 << 18)));
4452 match(ConI);
4453
4454 op_cost(0);
4455 format %{ %}
4456 interface(CONST_INTER);
4457 %}
4458
4459 // 12 bit unsigned offset -- for base plus immediate loads
4460 operand immIU12()
4461 %{
4462 predicate((0 <= n->get_int()) && (n->get_int() < (1 << 12)));
4463 match(ConI);
4464
4465 op_cost(0);
4466 format %{ %}
4467 interface(CONST_INTER);
4468 %}
4469
4470 operand immLU12()
4471 %{
4472 predicate((0 <= n->get_long()) && (n->get_long() < (1 << 12)));
4473 match(ConL);
4474
4475 op_cost(0);
4476 format %{ %}
4477 interface(CONST_INTER);
4478 %}
4479
4480 // Offset for scaled or unscaled immediate loads and stores
4481 operand immIOffset()
4482 %{
4483 predicate(Address::offset_ok_for_immed(n->get_int(), 0));
4484 match(ConI);
4485
4486 op_cost(0);
4487 format %{ %}
4488 interface(CONST_INTER);
4489 %}
4490
4491 operand immIOffset1()
4492 %{
4493 predicate(Address::offset_ok_for_immed(n->get_int(), 0));
4494 match(ConI);
4495
4496 op_cost(0);
4497 format %{ %}
4498 interface(CONST_INTER);
4499 %}
4500
4501 operand immIOffset2()
4502 %{
4503 predicate(Address::offset_ok_for_immed(n->get_int(), 1));
4504 match(ConI);
4505
4506 op_cost(0);
4507 format %{ %}
4508 interface(CONST_INTER);
4509 %}
4510
4511 operand immIOffset4()
4512 %{
4513 predicate(Address::offset_ok_for_immed(n->get_int(), 2));
4514 match(ConI);
4515
4516 op_cost(0);
4517 format %{ %}
4518 interface(CONST_INTER);
4519 %}
4520
4521 operand immIOffset8()
4522 %{
4523 predicate(Address::offset_ok_for_immed(n->get_int(), 3));
4524 match(ConI);
4525
4526 op_cost(0);
4527 format %{ %}
4528 interface(CONST_INTER);
4529 %}
4530
4531 operand immIOffset16()
4532 %{
4533 predicate(Address::offset_ok_for_immed(n->get_int(), 4));
4534 match(ConI);
4535
4536 op_cost(0);
4537 format %{ %}
4538 interface(CONST_INTER);
4539 %}
4540
4541 operand immLoffset()
4542 %{
4543 predicate(Address::offset_ok_for_immed(n->get_long(), 0));
4544 match(ConL);
4545
4546 op_cost(0);
4547 format %{ %}
4548 interface(CONST_INTER);
4549 %}
4550
4551 operand immLoffset1()
4552 %{
4553 predicate(Address::offset_ok_for_immed(n->get_long(), 0));
4554 match(ConL);
4555
4556 op_cost(0);
4557 format %{ %}
4558 interface(CONST_INTER);
4559 %}
4560
4561 operand immLoffset2()
4562 %{
4563 predicate(Address::offset_ok_for_immed(n->get_long(), 1));
4564 match(ConL);
4565
4566 op_cost(0);
4567 format %{ %}
4568 interface(CONST_INTER);
4569 %}
4570
4571 operand immLoffset4()
4572 %{
4573 predicate(Address::offset_ok_for_immed(n->get_long(), 2));
4574 match(ConL);
4575
4576 op_cost(0);
4577 format %{ %}
4578 interface(CONST_INTER);
4579 %}
4580
4581 operand immLoffset8()
4582 %{
4583 predicate(Address::offset_ok_for_immed(n->get_long(), 3));
4584 match(ConL);
4585
4586 op_cost(0);
4587 format %{ %}
4588 interface(CONST_INTER);
4589 %}
4590
4591 operand immLoffset16()
4592 %{
4593 predicate(Address::offset_ok_for_immed(n->get_long(), 4));
4594 match(ConL);
4595
4596 op_cost(0);
4597 format %{ %}
4598 interface(CONST_INTER);
4599 %}
4600
4601 // 8 bit signed value.
4602 operand immI8()
4603 %{
4604 predicate(n->get_int() <= 127 && n->get_int() >= -128);
4605 match(ConI);
4606
4607 op_cost(0);
4608 format %{ %}
4609 interface(CONST_INTER);
4610 %}
4611
4612 // 8 bit signed value (simm8), or #simm8 LSL 8.
4613 operand immI8_shift8()
4614 %{
4615 predicate((n->get_int() <= 127 && n->get_int() >= -128) ||
4616 (n->get_int() <= 32512 && n->get_int() >= -32768 && (n->get_int() & 0xff) == 0));
4617 match(ConI);
4618
4619 op_cost(0);
4620 format %{ %}
4621 interface(CONST_INTER);
4622 %}
4623
4624 // 8 bit signed value (simm8), or #simm8 LSL 8.
4625 operand immL8_shift8()
4626 %{
4627 predicate((n->get_long() <= 127 && n->get_long() >= -128) ||
4628 (n->get_long() <= 32512 && n->get_long() >= -32768 && (n->get_long() & 0xff) == 0));
4629 match(ConL);
4630
4631 op_cost(0);
4632 format %{ %}
4633 interface(CONST_INTER);
4634 %}
4635
4636 // 32 bit integer valid for add sub immediate
4637 operand immIAddSub()
4638 %{
4639 predicate(Assembler::operand_valid_for_add_sub_immediate((int64_t)n->get_int()));
4640 match(ConI);
4641 op_cost(0);
4642 format %{ %}
4643 interface(CONST_INTER);
4644 %}
4645
4646 // 32 bit unsigned integer valid for logical immediate
4647 // TODO -- check this is right when e.g the mask is 0x80000000
4648 operand immILog()
4649 %{
4650 predicate(Assembler::operand_valid_for_logical_immediate(/*is32*/true, (uint64_t)n->get_int()));
4651 match(ConI);
4652
4653 op_cost(0);
4654 format %{ %}
4655 interface(CONST_INTER);
4656 %}
4657
4658 // Integer operands 64 bit
4659 // 64 bit immediate
4660 operand immL()
4661 %{
4662 match(ConL);
4663
4664 op_cost(0);
4665 format %{ %}
4666 interface(CONST_INTER);
4667 %}
4668
4669 // 64 bit zero
4670 operand immL0()
4671 %{
4672 predicate(n->get_long() == 0);
4673 match(ConL);
4674
4675 op_cost(0);
4676 format %{ %}
4677 interface(CONST_INTER);
4678 %}
4679
4680 // 64 bit unit increment
4681 operand immL_1()
4682 %{
4683 predicate(n->get_long() == 1);
4684 match(ConL);
4685
4686 op_cost(0);
4687 format %{ %}
4688 interface(CONST_INTER);
4689 %}
4690
4691 // 64 bit unit decrement
4692 operand immL_M1()
4693 %{
4694 predicate(n->get_long() == -1);
4695 match(ConL);
4696
4697 op_cost(0);
4698 format %{ %}
4699 interface(CONST_INTER);
4700 %}
4701
4702 // 32 bit offset of pc in thread anchor
4703
4704 operand immL_pc_off()
4705 %{
4706 predicate(n->get_long() == in_bytes(JavaThread::frame_anchor_offset()) +
4707 in_bytes(JavaFrameAnchor::last_Java_pc_offset()));
4708 match(ConL);
4709
4710 op_cost(0);
4711 format %{ %}
4712 interface(CONST_INTER);
4713 %}
4714
4715 // 64 bit integer valid for add sub immediate
4716 operand immLAddSub()
4717 %{
4718 predicate(Assembler::operand_valid_for_add_sub_immediate(n->get_long()));
4719 match(ConL);
4720 op_cost(0);
4721 format %{ %}
4722 interface(CONST_INTER);
4723 %}
4724
4725 // 64 bit integer valid for logical immediate
4726 operand immLLog()
4727 %{
4728 predicate(Assembler::operand_valid_for_logical_immediate(/*is32*/false, (uint64_t)n->get_long()));
4729 match(ConL);
4730 op_cost(0);
4731 format %{ %}
4732 interface(CONST_INTER);
4733 %}
4734
4735 // Long Immediate: low 32-bit mask
4736 operand immL_32bits()
4737 %{
4738 predicate(n->get_long() == 0xFFFFFFFFL);
4739 match(ConL);
4740 op_cost(0);
4741 format %{ %}
4742 interface(CONST_INTER);
4743 %}
4744
4745 // Pointer operands
4746 // Pointer Immediate
4747 operand immP()
4748 %{
4749 match(ConP);
4750
4751 op_cost(0);
4752 format %{ %}
4753 interface(CONST_INTER);
4754 %}
4755
4756 // NULL Pointer Immediate
4757 operand immP0()
4758 %{
4759 predicate(n->get_ptr() == 0);
4760 match(ConP);
4761
4762 op_cost(0);
4763 format %{ %}
4764 interface(CONST_INTER);
4765 %}
4766
4767 // Pointer Immediate One
4768 // this is used in object initialization (initial object header)
4769 operand immP_1()
4770 %{
4771 predicate(n->get_ptr() == 1);
4772 match(ConP);
4773
4774 op_cost(0);
4775 format %{ %}
4776 interface(CONST_INTER);
4777 %}
4778
4779 // Card Table Byte Map Base
4780 operand immByteMapBase()
4781 %{
4782 // Get base of card map
4783 predicate(BarrierSet::barrier_set()->is_a(BarrierSet::CardTableBarrierSet) &&
4784 (CardTable::CardValue*)n->get_ptr() == ((CardTableBarrierSet*)(BarrierSet::barrier_set()))->card_table()->byte_map_base());
4785 match(ConP);
4786
4787 op_cost(0);
4788 format %{ %}
4789 interface(CONST_INTER);
4790 %}
4791
4792 // Pointer Immediate Minus One
4793 // this is used when we want to write the current PC to the thread anchor
4794 operand immP_M1()
4795 %{
4796 predicate(n->get_ptr() == -1);
4797 match(ConP);
4798
4799 op_cost(0);
4800 format %{ %}
4801 interface(CONST_INTER);
4802 %}
4803
4804 // Pointer Immediate Minus Two
4805 // this is used when we want to write the current PC to the thread anchor
4806 operand immP_M2()
4807 %{
4808 predicate(n->get_ptr() == -2);
4809 match(ConP);
4810
4811 op_cost(0);
4812 format %{ %}
4813 interface(CONST_INTER);
4814 %}
4815
4816 // Float and Double operands
4817 // Double Immediate
4818 operand immD()
4819 %{
4820 match(ConD);
4821 op_cost(0);
4822 format %{ %}
4823 interface(CONST_INTER);
4824 %}
4825
4826 // Double Immediate: +0.0d
4827 operand immD0()
4828 %{
4829 predicate(jlong_cast(n->getd()) == 0);
4830 match(ConD);
4831
4832 op_cost(0);
4833 format %{ %}
4834 interface(CONST_INTER);
4835 %}
4836
4837 // constant 'double +0.0'.
4838 operand immDPacked()
4839 %{
4840 predicate(Assembler::operand_valid_for_float_immediate(n->getd()));
4841 match(ConD);
4842 op_cost(0);
4843 format %{ %}
4844 interface(CONST_INTER);
4845 %}
4846
4847 // Float Immediate
4848 operand immF()
4849 %{
4850 match(ConF);
4851 op_cost(0);
4852 format %{ %}
4853 interface(CONST_INTER);
4854 %}
4855
4856 // Float Immediate: +0.0f.
4857 operand immF0()
4858 %{
4859 predicate(jint_cast(n->getf()) == 0);
4860 match(ConF);
4861
4862 op_cost(0);
4863 format %{ %}
4864 interface(CONST_INTER);
4865 %}
4866
4867 //
4868 operand immFPacked()
4869 %{
4870 predicate(Assembler::operand_valid_for_float_immediate((double)n->getf()));
4871 match(ConF);
4872 op_cost(0);
4873 format %{ %}
4874 interface(CONST_INTER);
4875 %}
4876
4877 // Narrow pointer operands
4878 // Narrow Pointer Immediate
4879 operand immN()
4880 %{
4881 match(ConN);
4882
4883 op_cost(0);
4884 format %{ %}
4885 interface(CONST_INTER);
4886 %}
4887
4888 // Narrow NULL Pointer Immediate
4889 operand immN0()
4890 %{
4891 predicate(n->get_narrowcon() == 0);
4892 match(ConN);
4893
4894 op_cost(0);
4895 format %{ %}
4896 interface(CONST_INTER);
4897 %}
4898
4899 operand immNKlass()
4900 %{
4901 match(ConNKlass);
4902
4903 op_cost(0);
4904 format %{ %}
4905 interface(CONST_INTER);
4906 %}
4907
4908 // Integer 32 bit Register Operands
4909 // Integer 32 bitRegister (excludes SP)
4910 operand iRegI()
4911 %{
4912 constraint(ALLOC_IN_RC(any_reg32));
4913 match(RegI);
4914 match(iRegINoSp);
4915 op_cost(0);
4916 format %{ %}
4917 interface(REG_INTER);
4918 %}
4919
4920 // Integer 32 bit Register not Special
4921 operand iRegINoSp()
4922 %{
4923 constraint(ALLOC_IN_RC(no_special_reg32));
4924 match(RegI);
4925 op_cost(0);
4926 format %{ %}
4927 interface(REG_INTER);
4928 %}
4929
4930 // Integer 64 bit Register Operands
4931 // Integer 64 bit Register (includes SP)
4932 operand iRegL()
4933 %{
4934 constraint(ALLOC_IN_RC(any_reg));
4935 match(RegL);
4936 match(iRegLNoSp);
4937 op_cost(0);
4938 format %{ %}
4939 interface(REG_INTER);
4940 %}
4941
4942 // Integer 64 bit Register not Special
4943 operand iRegLNoSp()
4944 %{
4945 constraint(ALLOC_IN_RC(no_special_reg));
4946 match(RegL);
4947 match(iRegL_R0);
4948 format %{ %}
4949 interface(REG_INTER);
4950 %}
4951
4952 // Pointer Register Operands
4953 // Pointer Register
4954 operand iRegP()
4955 %{
4956 constraint(ALLOC_IN_RC(ptr_reg));
4957 match(RegP);
4958 match(iRegPNoSp);
4959 match(iRegP_R0);
4960 //match(iRegP_R2);
4961 //match(iRegP_R4);
4962 match(iRegP_R5);
4963 match(thread_RegP);
4964 op_cost(0);
4965 format %{ %}
4966 interface(REG_INTER);
4967 %}
4968
4969 // Pointer 64 bit Register not Special
4970 operand iRegPNoSp()
4971 %{
4972 constraint(ALLOC_IN_RC(no_special_ptr_reg));
4973 match(RegP);
4974 // match(iRegP);
4975 // match(iRegP_R0);
4976 // match(iRegP_R2);
4977 // match(iRegP_R4);
4978 // match(iRegP_R5);
4979 // match(thread_RegP);
4980 op_cost(0);
4981 format %{ %}
4982 interface(REG_INTER);
4983 %}
4984
4985 // Pointer 64 bit Register R0 only
4986 operand iRegP_R0()
4987 %{
4988 constraint(ALLOC_IN_RC(r0_reg));
4989 match(RegP);
4990 // match(iRegP);
4991 match(iRegPNoSp);
4992 op_cost(0);
4993 format %{ %}
4994 interface(REG_INTER);
4995 %}
4996
4997 // Pointer 64 bit Register R1 only
4998 operand iRegP_R1()
4999 %{
5000 constraint(ALLOC_IN_RC(r1_reg));
5001 match(RegP);
5002 // match(iRegP);
5003 match(iRegPNoSp);
5004 op_cost(0);
5005 format %{ %}
5006 interface(REG_INTER);
5007 %}
5008
5009 // Pointer 64 bit Register R2 only
5010 operand iRegP_R2()
5011 %{
5012 constraint(ALLOC_IN_RC(r2_reg));
5013 match(RegP);
5014 // match(iRegP);
5015 match(iRegPNoSp);
5016 op_cost(0);
5017 format %{ %}
5018 interface(REG_INTER);
5019 %}
5020
5021 // Pointer 64 bit Register R3 only
5022 operand iRegP_R3()
5023 %{
5024 constraint(ALLOC_IN_RC(r3_reg));
5025 match(RegP);
5026 // match(iRegP);
5027 match(iRegPNoSp);
5028 op_cost(0);
5029 format %{ %}
5030 interface(REG_INTER);
5031 %}
5032
5033 // Pointer 64 bit Register R4 only
5034 operand iRegP_R4()
5035 %{
5036 constraint(ALLOC_IN_RC(r4_reg));
5037 match(RegP);
5038 // match(iRegP);
5039 match(iRegPNoSp);
5040 op_cost(0);
5041 format %{ %}
5042 interface(REG_INTER);
5043 %}
5044
5045 // Pointer 64 bit Register R5 only
5046 operand iRegP_R5()
5047 %{
5048 constraint(ALLOC_IN_RC(r5_reg));
5049 match(RegP);
5050 // match(iRegP);
5051 match(iRegPNoSp);
5052 op_cost(0);
5053 format %{ %}
5054 interface(REG_INTER);
5055 %}
5056
5057 // Pointer 64 bit Register R10 only
5058 operand iRegP_R10()
5059 %{
5060 constraint(ALLOC_IN_RC(r10_reg));
5061 match(RegP);
5062 // match(iRegP);
5063 match(iRegPNoSp);
5064 op_cost(0);
5065 format %{ %}
5066 interface(REG_INTER);
5067 %}
5068
5069 // Long 64 bit Register R0 only
5070 operand iRegL_R0()
5071 %{
5072 constraint(ALLOC_IN_RC(r0_reg));
5073 match(RegL);
5074 match(iRegLNoSp);
5075 op_cost(0);
5076 format %{ %}
5077 interface(REG_INTER);
5078 %}
5079
5080 // Long 64 bit Register R2 only
5081 operand iRegL_R2()
5082 %{
5083 constraint(ALLOC_IN_RC(r2_reg));
5084 match(RegL);
5085 match(iRegLNoSp);
5086 op_cost(0);
5087 format %{ %}
5088 interface(REG_INTER);
5089 %}
5090
5091 // Long 64 bit Register R3 only
5092 operand iRegL_R3()
5093 %{
5094 constraint(ALLOC_IN_RC(r3_reg));
5095 match(RegL);
5096 match(iRegLNoSp);
5097 op_cost(0);
5098 format %{ %}
5099 interface(REG_INTER);
5100 %}
5101
5102 // Long 64 bit Register R11 only
5103 operand iRegL_R11()
5104 %{
5105 constraint(ALLOC_IN_RC(r11_reg));
5106 match(RegL);
5107 match(iRegLNoSp);
5108 op_cost(0);
5109 format %{ %}
5110 interface(REG_INTER);
5111 %}
5112
5113 // Pointer 64 bit Register FP only
5114 operand iRegP_FP()
5115 %{
5116 constraint(ALLOC_IN_RC(fp_reg));
5117 match(RegP);
5118 // match(iRegP);
5119 op_cost(0);
5120 format %{ %}
5121 interface(REG_INTER);
5122 %}
5123
5124 // Register R0 only
5125 operand iRegI_R0()
5126 %{
5127 constraint(ALLOC_IN_RC(int_r0_reg));
5128 match(RegI);
5129 match(iRegINoSp);
5130 op_cost(0);
5131 format %{ %}
5132 interface(REG_INTER);
5133 %}
5134
5135 // Register R2 only
5136 operand iRegI_R2()
5137 %{
5138 constraint(ALLOC_IN_RC(int_r2_reg));
5139 match(RegI);
5140 match(iRegINoSp);
5141 op_cost(0);
5142 format %{ %}
5143 interface(REG_INTER);
5144 %}
5145
5146 // Register R3 only
5147 operand iRegI_R3()
5148 %{
5149 constraint(ALLOC_IN_RC(int_r3_reg));
5150 match(RegI);
5151 match(iRegINoSp);
5152 op_cost(0);
5153 format %{ %}
5154 interface(REG_INTER);
5155 %}
5156
5157
5158 // Register R4 only
5159 operand iRegI_R4()
5160 %{
5161 constraint(ALLOC_IN_RC(int_r4_reg));
5162 match(RegI);
5163 match(iRegINoSp);
5164 op_cost(0);
5165 format %{ %}
5166 interface(REG_INTER);
5167 %}
5168
5169
5170 // Pointer Register Operands
5171 // Narrow Pointer Register
5172 operand iRegN()
5173 %{
5174 constraint(ALLOC_IN_RC(any_reg32));
5175 match(RegN);
5176 match(iRegNNoSp);
5177 op_cost(0);
5178 format %{ %}
5179 interface(REG_INTER);
5180 %}
5181
5182 operand iRegN_R0()
5183 %{
5184 constraint(ALLOC_IN_RC(r0_reg));
5185 match(iRegN);
5186 op_cost(0);
5187 format %{ %}
5188 interface(REG_INTER);
5189 %}
5190
5191 operand iRegN_R2()
5192 %{
5193 constraint(ALLOC_IN_RC(r2_reg));
5194 match(iRegN);
5195 op_cost(0);
5196 format %{ %}
5197 interface(REG_INTER);
5198 %}
5199
5200 operand iRegN_R3()
5201 %{
5202 constraint(ALLOC_IN_RC(r3_reg));
5203 match(iRegN);
5204 op_cost(0);
5205 format %{ %}
5206 interface(REG_INTER);
5207 %}
5208
5209 // Integer 64 bit Register not Special
5210 operand iRegNNoSp()
5211 %{
5212 constraint(ALLOC_IN_RC(no_special_reg32));
5213 match(RegN);
5214 op_cost(0);
5215 format %{ %}
5216 interface(REG_INTER);
5217 %}
5218
5219 // heap base register -- used for encoding immN0
5220
5221 operand iRegIHeapbase()
5222 %{
5223 constraint(ALLOC_IN_RC(heapbase_reg));
5224 match(RegI);
5225 op_cost(0);
5226 format %{ %}
5227 interface(REG_INTER);
5228 %}
5229
5230 // Float Register
5231 // Float register operands
5232 operand vRegF()
5233 %{
5234 constraint(ALLOC_IN_RC(float_reg));
5235 match(RegF);
5236
5237 op_cost(0);
5238 format %{ %}
5239 interface(REG_INTER);
5240 %}
5241
5242 // Double Register
5243 // Double register operands
5244 operand vRegD()
5245 %{
5246 constraint(ALLOC_IN_RC(double_reg));
5247 match(RegD);
5248
5249 op_cost(0);
5250 format %{ %}
5251 interface(REG_INTER);
5252 %}
5253
5254 // Generic vector class. This will be used for
5255 // all vector operands, including NEON and SVE,
5256 // but currently only used for SVE VecA.
5257 operand vReg()
5258 %{
5259 constraint(ALLOC_IN_RC(vectora_reg));
5260 match(VecA);
5261 op_cost(0);
5262 format %{ %}
5263 interface(REG_INTER);
5264 %}
5265
5266 operand vecD()
5267 %{
5268 constraint(ALLOC_IN_RC(vectord_reg));
5269 match(VecD);
5270
5271 op_cost(0);
5272 format %{ %}
5273 interface(REG_INTER);
5274 %}
5275
5276 operand vecX()
5277 %{
5278 constraint(ALLOC_IN_RC(vectorx_reg));
5279 match(VecX);
5280
5281 op_cost(0);
5282 format %{ %}
5283 interface(REG_INTER);
5284 %}
5285
5286 operand vRegD_V0()
5287 %{
5288 constraint(ALLOC_IN_RC(v0_reg));
5289 match(RegD);
5290 op_cost(0);
5291 format %{ %}
5292 interface(REG_INTER);
5293 %}
5294
5295 operand vRegD_V1()
5296 %{
5297 constraint(ALLOC_IN_RC(v1_reg));
5298 match(RegD);
5299 op_cost(0);
5300 format %{ %}
5301 interface(REG_INTER);
5302 %}
5303
5304 operand vRegD_V2()
5305 %{
5306 constraint(ALLOC_IN_RC(v2_reg));
5307 match(RegD);
5308 op_cost(0);
5309 format %{ %}
5310 interface(REG_INTER);
5311 %}
5312
5313 operand vRegD_V3()
5314 %{
5315 constraint(ALLOC_IN_RC(v3_reg));
5316 match(RegD);
5317 op_cost(0);
5318 format %{ %}
5319 interface(REG_INTER);
5320 %}
5321
5322 operand vRegD_V4()
5323 %{
5324 constraint(ALLOC_IN_RC(v4_reg));
5325 match(RegD);
5326 op_cost(0);
5327 format %{ %}
5328 interface(REG_INTER);
5329 %}
5330
5331 operand vRegD_V5()
5332 %{
5333 constraint(ALLOC_IN_RC(v5_reg));
5334 match(RegD);
5335 op_cost(0);
5336 format %{ %}
5337 interface(REG_INTER);
5338 %}
5339
5340 operand vRegD_V6()
5341 %{
5342 constraint(ALLOC_IN_RC(v6_reg));
5343 match(RegD);
5344 op_cost(0);
5345 format %{ %}
5346 interface(REG_INTER);
5347 %}
5348
5349 operand vRegD_V7()
5350 %{
5351 constraint(ALLOC_IN_RC(v7_reg));
5352 match(RegD);
5353 op_cost(0);
5354 format %{ %}
5355 interface(REG_INTER);
5356 %}
5357
5358 operand vRegD_V8()
5359 %{
5360 constraint(ALLOC_IN_RC(v8_reg));
5361 match(RegD);
5362 op_cost(0);
5363 format %{ %}
5364 interface(REG_INTER);
5365 %}
5366
5367 operand vRegD_V9()
5368 %{
5369 constraint(ALLOC_IN_RC(v9_reg));
5370 match(RegD);
5371 op_cost(0);
5372 format %{ %}
5373 interface(REG_INTER);
5374 %}
5375
5376 operand vRegD_V10()
5377 %{
5378 constraint(ALLOC_IN_RC(v10_reg));
5379 match(RegD);
5380 op_cost(0);
5381 format %{ %}
5382 interface(REG_INTER);
5383 %}
5384
5385 operand vRegD_V11()
5386 %{
5387 constraint(ALLOC_IN_RC(v11_reg));
5388 match(RegD);
5389 op_cost(0);
5390 format %{ %}
5391 interface(REG_INTER);
5392 %}
5393
5394 operand vRegD_V12()
5395 %{
5396 constraint(ALLOC_IN_RC(v12_reg));
5397 match(RegD);
5398 op_cost(0);
5399 format %{ %}
5400 interface(REG_INTER);
5401 %}
5402
5403 operand vRegD_V13()
5404 %{
5405 constraint(ALLOC_IN_RC(v13_reg));
5406 match(RegD);
5407 op_cost(0);
5408 format %{ %}
5409 interface(REG_INTER);
5410 %}
5411
5412 operand vRegD_V14()
5413 %{
5414 constraint(ALLOC_IN_RC(v14_reg));
5415 match(RegD);
5416 op_cost(0);
5417 format %{ %}
5418 interface(REG_INTER);
5419 %}
5420
5421 operand vRegD_V15()
5422 %{
5423 constraint(ALLOC_IN_RC(v15_reg));
5424 match(RegD);
5425 op_cost(0);
5426 format %{ %}
5427 interface(REG_INTER);
5428 %}
5429
5430 operand vRegD_V16()
5431 %{
5432 constraint(ALLOC_IN_RC(v16_reg));
5433 match(RegD);
5434 op_cost(0);
5435 format %{ %}
5436 interface(REG_INTER);
5437 %}
5438
5439 operand vRegD_V17()
5440 %{
5441 constraint(ALLOC_IN_RC(v17_reg));
5442 match(RegD);
5443 op_cost(0);
5444 format %{ %}
5445 interface(REG_INTER);
5446 %}
5447
5448 operand vRegD_V18()
5449 %{
5450 constraint(ALLOC_IN_RC(v18_reg));
5451 match(RegD);
5452 op_cost(0);
5453 format %{ %}
5454 interface(REG_INTER);
5455 %}
5456
5457 operand vRegD_V19()
5458 %{
5459 constraint(ALLOC_IN_RC(v19_reg));
5460 match(RegD);
5461 op_cost(0);
5462 format %{ %}
5463 interface(REG_INTER);
5464 %}
5465
5466 operand vRegD_V20()
5467 %{
5468 constraint(ALLOC_IN_RC(v20_reg));
5469 match(RegD);
5470 op_cost(0);
5471 format %{ %}
5472 interface(REG_INTER);
5473 %}
5474
5475 operand vRegD_V21()
5476 %{
5477 constraint(ALLOC_IN_RC(v21_reg));
5478 match(RegD);
5479 op_cost(0);
5480 format %{ %}
5481 interface(REG_INTER);
5482 %}
5483
5484 operand vRegD_V22()
5485 %{
5486 constraint(ALLOC_IN_RC(v22_reg));
5487 match(RegD);
5488 op_cost(0);
5489 format %{ %}
5490 interface(REG_INTER);
5491 %}
5492
5493 operand vRegD_V23()
5494 %{
5495 constraint(ALLOC_IN_RC(v23_reg));
5496 match(RegD);
5497 op_cost(0);
5498 format %{ %}
5499 interface(REG_INTER);
5500 %}
5501
5502 operand vRegD_V24()
5503 %{
5504 constraint(ALLOC_IN_RC(v24_reg));
5505 match(RegD);
5506 op_cost(0);
5507 format %{ %}
5508 interface(REG_INTER);
5509 %}
5510
5511 operand vRegD_V25()
5512 %{
5513 constraint(ALLOC_IN_RC(v25_reg));
5514 match(RegD);
5515 op_cost(0);
5516 format %{ %}
5517 interface(REG_INTER);
5518 %}
5519
5520 operand vRegD_V26()
5521 %{
5522 constraint(ALLOC_IN_RC(v26_reg));
5523 match(RegD);
5524 op_cost(0);
5525 format %{ %}
5526 interface(REG_INTER);
5527 %}
5528
5529 operand vRegD_V27()
5530 %{
5531 constraint(ALLOC_IN_RC(v27_reg));
5532 match(RegD);
5533 op_cost(0);
5534 format %{ %}
5535 interface(REG_INTER);
5536 %}
5537
5538 operand vRegD_V28()
5539 %{
5540 constraint(ALLOC_IN_RC(v28_reg));
5541 match(RegD);
5542 op_cost(0);
5543 format %{ %}
5544 interface(REG_INTER);
5545 %}
5546
5547 operand vRegD_V29()
5548 %{
5549 constraint(ALLOC_IN_RC(v29_reg));
5550 match(RegD);
5551 op_cost(0);
5552 format %{ %}
5553 interface(REG_INTER);
5554 %}
5555
5556 operand vRegD_V30()
5557 %{
5558 constraint(ALLOC_IN_RC(v30_reg));
5559 match(RegD);
5560 op_cost(0);
5561 format %{ %}
5562 interface(REG_INTER);
5563 %}
5564
5565 operand vRegD_V31()
5566 %{
5567 constraint(ALLOC_IN_RC(v31_reg));
5568 match(RegD);
5569 op_cost(0);
5570 format %{ %}
5571 interface(REG_INTER);
5572 %}
5573
5574 operand pRegGov()
5575 %{
5576 constraint(ALLOC_IN_RC(gov_pr));
5577 match(RegVectMask);
5578 op_cost(0);
5579 format %{ %}
5580 interface(REG_INTER);
5581 %}
5582
5583 // Flags register, used as output of signed compare instructions
5584
5585 // note that on AArch64 we also use this register as the output for
5586 // for floating point compare instructions (CmpF CmpD). this ensures
5587 // that ordered inequality tests use GT, GE, LT or LE none of which
5588 // pass through cases where the result is unordered i.e. one or both
5589 // inputs to the compare is a NaN. this means that the ideal code can
5590 // replace e.g. a GT with an LE and not end up capturing the NaN case
5591 // (where the comparison should always fail). EQ and NE tests are
5592 // always generated in ideal code so that unordered folds into the NE
5593 // case, matching the behaviour of AArch64 NE.
5594 //
5595 // This differs from x86 where the outputs of FP compares use a
5596 // special FP flags registers and where compares based on this
5597 // register are distinguished into ordered inequalities (cmpOpUCF) and
5598 // EQ/NEQ tests (cmpOpUCF2). x86 has to special case the latter tests
5599 // to explicitly handle the unordered case in branches. x86 also has
5600 // to include extra CMoveX rules to accept a cmpOpUCF input.
5601
5602 operand rFlagsReg()
5603 %{
5604 constraint(ALLOC_IN_RC(int_flags));
5605 match(RegFlags);
5606
5607 op_cost(0);
5608 format %{ "RFLAGS" %}
5609 interface(REG_INTER);
5610 %}
5611
5612 // Flags register, used as output of unsigned compare instructions
5613 operand rFlagsRegU()
5614 %{
5615 constraint(ALLOC_IN_RC(int_flags));
5616 match(RegFlags);
5617
5618 op_cost(0);
5619 format %{ "RFLAGSU" %}
5620 interface(REG_INTER);
5621 %}
5622
5623 // Special Registers
5624
5625 // Method Register
5626 operand inline_cache_RegP(iRegP reg)
5627 %{
5628 constraint(ALLOC_IN_RC(method_reg)); // inline_cache_reg
5629 match(reg);
5630 match(iRegPNoSp);
5631 op_cost(0);
5632 format %{ %}
5633 interface(REG_INTER);
5634 %}
5635
5636 // Thread Register
5637 operand thread_RegP(iRegP reg)
5638 %{
5639 constraint(ALLOC_IN_RC(thread_reg)); // link_reg
5640 match(reg);
5641 op_cost(0);
5642 format %{ %}
5643 interface(REG_INTER);
5644 %}
5645
5646 operand lr_RegP(iRegP reg)
5647 %{
5648 constraint(ALLOC_IN_RC(lr_reg)); // link_reg
5649 match(reg);
5650 op_cost(0);
5651 format %{ %}
5652 interface(REG_INTER);
5653 %}
5654
5655 //----------Memory Operands----------------------------------------------------
5656
5657 operand indirect(iRegP reg)
5658 %{
5659 constraint(ALLOC_IN_RC(ptr_reg));
5660 match(reg);
5661 op_cost(0);
5662 format %{ "[$reg]" %}
5663 interface(MEMORY_INTER) %{
5664 base($reg);
5665 index(0xffffffff);
5666 scale(0x0);
5667 disp(0x0);
5668 %}
5669 %}
5670
5671 operand indIndexScaledI2L(iRegP reg, iRegI ireg, immIScale scale)
5672 %{
5673 constraint(ALLOC_IN_RC(ptr_reg));
5674 predicate(size_fits_all_mem_uses(n->as_AddP(), n->in(AddPNode::Offset)->in(2)->get_int()));
5675 match(AddP reg (LShiftL (ConvI2L ireg) scale));
5676 op_cost(0);
5677 format %{ "$reg, $ireg sxtw($scale), 0, I2L" %}
5678 interface(MEMORY_INTER) %{
5679 base($reg);
5680 index($ireg);
5681 scale($scale);
5682 disp(0x0);
5683 %}
5684 %}
5685
5686 operand indIndexScaled(iRegP reg, iRegL lreg, immIScale scale)
5687 %{
5688 constraint(ALLOC_IN_RC(ptr_reg));
5689 predicate(size_fits_all_mem_uses(n->as_AddP(), n->in(AddPNode::Offset)->in(2)->get_int()));
5690 match(AddP reg (LShiftL lreg scale));
5691 op_cost(0);
5692 format %{ "$reg, $lreg lsl($scale)" %}
5693 interface(MEMORY_INTER) %{
5694 base($reg);
5695 index($lreg);
5696 scale($scale);
5697 disp(0x0);
5698 %}
5699 %}
5700
5701 operand indIndexI2L(iRegP reg, iRegI ireg)
5702 %{
5703 constraint(ALLOC_IN_RC(ptr_reg));
5704 match(AddP reg (ConvI2L ireg));
5705 op_cost(0);
5706 format %{ "$reg, $ireg, 0, I2L" %}
5707 interface(MEMORY_INTER) %{
5708 base($reg);
5709 index($ireg);
5710 scale(0x0);
5711 disp(0x0);
5712 %}
5713 %}
5714
5715 operand indIndex(iRegP reg, iRegL lreg)
5716 %{
5717 constraint(ALLOC_IN_RC(ptr_reg));
5718 match(AddP reg lreg);
5719 op_cost(0);
5720 format %{ "$reg, $lreg" %}
5721 interface(MEMORY_INTER) %{
5722 base($reg);
5723 index($lreg);
5724 scale(0x0);
5725 disp(0x0);
5726 %}
5727 %}
5728
5729 operand indOffI(iRegP reg, immIOffset off)
5730 %{
5731 constraint(ALLOC_IN_RC(ptr_reg));
5732 match(AddP reg off);
5733 op_cost(0);
5734 format %{ "[$reg, $off]" %}
5735 interface(MEMORY_INTER) %{
5736 base($reg);
5737 index(0xffffffff);
5738 scale(0x0);
5739 disp($off);
5740 %}
5741 %}
5742
5743 operand indOffI1(iRegP reg, immIOffset1 off)
5744 %{
5745 constraint(ALLOC_IN_RC(ptr_reg));
5746 match(AddP reg off);
5747 op_cost(0);
5748 format %{ "[$reg, $off]" %}
5749 interface(MEMORY_INTER) %{
5750 base($reg);
5751 index(0xffffffff);
5752 scale(0x0);
5753 disp($off);
5754 %}
5755 %}
5756
5757 operand indOffI2(iRegP reg, immIOffset2 off)
5758 %{
5759 constraint(ALLOC_IN_RC(ptr_reg));
5760 match(AddP reg off);
5761 op_cost(0);
5762 format %{ "[$reg, $off]" %}
5763 interface(MEMORY_INTER) %{
5764 base($reg);
5765 index(0xffffffff);
5766 scale(0x0);
5767 disp($off);
5768 %}
5769 %}
5770
5771 operand indOffI4(iRegP reg, immIOffset4 off)
5772 %{
5773 constraint(ALLOC_IN_RC(ptr_reg));
5774 match(AddP reg off);
5775 op_cost(0);
5776 format %{ "[$reg, $off]" %}
5777 interface(MEMORY_INTER) %{
5778 base($reg);
5779 index(0xffffffff);
5780 scale(0x0);
5781 disp($off);
5782 %}
5783 %}
5784
5785 operand indOffI8(iRegP reg, immIOffset8 off)
5786 %{
5787 constraint(ALLOC_IN_RC(ptr_reg));
5788 match(AddP reg off);
5789 op_cost(0);
5790 format %{ "[$reg, $off]" %}
5791 interface(MEMORY_INTER) %{
5792 base($reg);
5793 index(0xffffffff);
5794 scale(0x0);
5795 disp($off);
5796 %}
5797 %}
5798
5799 operand indOffI16(iRegP reg, immIOffset16 off)
5800 %{
5801 constraint(ALLOC_IN_RC(ptr_reg));
5802 match(AddP reg off);
5803 op_cost(0);
5804 format %{ "[$reg, $off]" %}
5805 interface(MEMORY_INTER) %{
5806 base($reg);
5807 index(0xffffffff);
5808 scale(0x0);
5809 disp($off);
5810 %}
5811 %}
5812
5813 operand indOffL(iRegP reg, immLoffset off)
5814 %{
5815 constraint(ALLOC_IN_RC(ptr_reg));
5816 match(AddP reg off);
5817 op_cost(0);
5818 format %{ "[$reg, $off]" %}
5819 interface(MEMORY_INTER) %{
5820 base($reg);
5821 index(0xffffffff);
5822 scale(0x0);
5823 disp($off);
5824 %}
5825 %}
5826
5827 operand indOffL1(iRegP reg, immLoffset1 off)
5828 %{
5829 constraint(ALLOC_IN_RC(ptr_reg));
5830 match(AddP reg off);
5831 op_cost(0);
5832 format %{ "[$reg, $off]" %}
5833 interface(MEMORY_INTER) %{
5834 base($reg);
5835 index(0xffffffff);
5836 scale(0x0);
5837 disp($off);
5838 %}
5839 %}
5840
5841 operand indOffL2(iRegP reg, immLoffset2 off)
5842 %{
5843 constraint(ALLOC_IN_RC(ptr_reg));
5844 match(AddP reg off);
5845 op_cost(0);
5846 format %{ "[$reg, $off]" %}
5847 interface(MEMORY_INTER) %{
5848 base($reg);
5849 index(0xffffffff);
5850 scale(0x0);
5851 disp($off);
5852 %}
5853 %}
5854
5855 operand indOffL4(iRegP reg, immLoffset4 off)
5856 %{
5857 constraint(ALLOC_IN_RC(ptr_reg));
5858 match(AddP reg off);
5859 op_cost(0);
5860 format %{ "[$reg, $off]" %}
5861 interface(MEMORY_INTER) %{
5862 base($reg);
5863 index(0xffffffff);
5864 scale(0x0);
5865 disp($off);
5866 %}
5867 %}
5868
5869 operand indOffL8(iRegP reg, immLoffset8 off)
5870 %{
5871 constraint(ALLOC_IN_RC(ptr_reg));
5872 match(AddP reg off);
5873 op_cost(0);
5874 format %{ "[$reg, $off]" %}
5875 interface(MEMORY_INTER) %{
5876 base($reg);
5877 index(0xffffffff);
5878 scale(0x0);
5879 disp($off);
5880 %}
5881 %}
5882
5883 operand indOffL16(iRegP reg, immLoffset16 off)
5884 %{
5885 constraint(ALLOC_IN_RC(ptr_reg));
5886 match(AddP reg off);
5887 op_cost(0);
5888 format %{ "[$reg, $off]" %}
5889 interface(MEMORY_INTER) %{
5890 base($reg);
5891 index(0xffffffff);
5892 scale(0x0);
5893 disp($off);
5894 %}
5895 %}
5896
5897 operand indirectN(iRegN reg)
5898 %{
5899 predicate(CompressedOops::shift() == 0);
5900 constraint(ALLOC_IN_RC(ptr_reg));
5901 match(DecodeN reg);
5902 op_cost(0);
5903 format %{ "[$reg]\t# narrow" %}
5904 interface(MEMORY_INTER) %{
5905 base($reg);
5906 index(0xffffffff);
5907 scale(0x0);
5908 disp(0x0);
5909 %}
5910 %}
5911
5912 operand indIndexScaledI2LN(iRegN reg, iRegI ireg, immIScale scale)
5913 %{
5914 predicate(CompressedOops::shift() == 0 && size_fits_all_mem_uses(n->as_AddP(), n->in(AddPNode::Offset)->in(2)->get_int()));
5915 constraint(ALLOC_IN_RC(ptr_reg));
5916 match(AddP (DecodeN reg) (LShiftL (ConvI2L ireg) scale));
5917 op_cost(0);
5918 format %{ "$reg, $ireg sxtw($scale), 0, I2L\t# narrow" %}
5919 interface(MEMORY_INTER) %{
5920 base($reg);
5921 index($ireg);
5922 scale($scale);
5923 disp(0x0);
5924 %}
5925 %}
5926
5927 operand indIndexScaledN(iRegN reg, iRegL lreg, immIScale scale)
5928 %{
5929 predicate(CompressedOops::shift() == 0 && size_fits_all_mem_uses(n->as_AddP(), n->in(AddPNode::Offset)->in(2)->get_int()));
5930 constraint(ALLOC_IN_RC(ptr_reg));
5931 match(AddP (DecodeN reg) (LShiftL lreg scale));
5932 op_cost(0);
5933 format %{ "$reg, $lreg lsl($scale)\t# narrow" %}
5934 interface(MEMORY_INTER) %{
5935 base($reg);
5936 index($lreg);
5937 scale($scale);
5938 disp(0x0);
5939 %}
5940 %}
5941
5942 operand indIndexI2LN(iRegN reg, iRegI ireg)
5943 %{
5944 predicate(CompressedOops::shift() == 0);
5945 constraint(ALLOC_IN_RC(ptr_reg));
5946 match(AddP (DecodeN reg) (ConvI2L ireg));
5947 op_cost(0);
5948 format %{ "$reg, $ireg, 0, I2L\t# narrow" %}
5949 interface(MEMORY_INTER) %{
5950 base($reg);
5951 index($ireg);
5952 scale(0x0);
5953 disp(0x0);
5954 %}
5955 %}
5956
5957 operand indIndexN(iRegN reg, iRegL lreg)
5958 %{
5959 predicate(CompressedOops::shift() == 0);
5960 constraint(ALLOC_IN_RC(ptr_reg));
5961 match(AddP (DecodeN reg) lreg);
5962 op_cost(0);
5963 format %{ "$reg, $lreg\t# narrow" %}
5964 interface(MEMORY_INTER) %{
5965 base($reg);
5966 index($lreg);
5967 scale(0x0);
5968 disp(0x0);
5969 %}
5970 %}
5971
5972 operand indOffIN(iRegN reg, immIOffset off)
5973 %{
5974 predicate(CompressedOops::shift() == 0);
5975 constraint(ALLOC_IN_RC(ptr_reg));
5976 match(AddP (DecodeN reg) off);
5977 op_cost(0);
5978 format %{ "[$reg, $off]\t# narrow" %}
5979 interface(MEMORY_INTER) %{
5980 base($reg);
5981 index(0xffffffff);
5982 scale(0x0);
5983 disp($off);
5984 %}
5985 %}
5986
5987 operand indOffLN(iRegN reg, immLoffset off)
5988 %{
5989 predicate(CompressedOops::shift() == 0);
5990 constraint(ALLOC_IN_RC(ptr_reg));
5991 match(AddP (DecodeN reg) off);
5992 op_cost(0);
5993 format %{ "[$reg, $off]\t# narrow" %}
5994 interface(MEMORY_INTER) %{
5995 base($reg);
5996 index(0xffffffff);
5997 scale(0x0);
5998 disp($off);
5999 %}
6000 %}
6001
6002
6003
6004 // AArch64 opto stubs need to write to the pc slot in the thread anchor
6005 operand thread_anchor_pc(thread_RegP reg, immL_pc_off off)
6006 %{
6007 constraint(ALLOC_IN_RC(ptr_reg));
6008 match(AddP reg off);
6009 op_cost(0);
6010 format %{ "[$reg, $off]" %}
6011 interface(MEMORY_INTER) %{
6012 base($reg);
6013 index(0xffffffff);
6014 scale(0x0);
6015 disp($off);
6016 %}
6017 %}
6018
6019 //----------Special Memory Operands--------------------------------------------
6020 // Stack Slot Operand - This operand is used for loading and storing temporary
6021 // values on the stack where a match requires a value to
6022 // flow through memory.
6023 operand stackSlotP(sRegP reg)
6024 %{
6025 constraint(ALLOC_IN_RC(stack_slots));
6026 op_cost(100);
6027 // No match rule because this operand is only generated in matching
6028 // match(RegP);
6029 format %{ "[$reg]" %}
6030 interface(MEMORY_INTER) %{
6031 base(0x1e); // RSP
6032 index(0x0); // No Index
6033 scale(0x0); // No Scale
6034 disp($reg); // Stack Offset
6035 %}
6036 %}
6037
6038 operand stackSlotI(sRegI reg)
6039 %{
6040 constraint(ALLOC_IN_RC(stack_slots));
6041 // No match rule because this operand is only generated in matching
6042 // match(RegI);
6043 format %{ "[$reg]" %}
6044 interface(MEMORY_INTER) %{
6045 base(0x1e); // RSP
6046 index(0x0); // No Index
6047 scale(0x0); // No Scale
6048 disp($reg); // Stack Offset
6049 %}
6050 %}
6051
6052 operand stackSlotF(sRegF reg)
6053 %{
6054 constraint(ALLOC_IN_RC(stack_slots));
6055 // No match rule because this operand is only generated in matching
6056 // match(RegF);
6057 format %{ "[$reg]" %}
6058 interface(MEMORY_INTER) %{
6059 base(0x1e); // RSP
6060 index(0x0); // No Index
6061 scale(0x0); // No Scale
6062 disp($reg); // Stack Offset
6063 %}
6064 %}
6065
6066 operand stackSlotD(sRegD reg)
6067 %{
6068 constraint(ALLOC_IN_RC(stack_slots));
6069 // No match rule because this operand is only generated in matching
6070 // match(RegD);
6071 format %{ "[$reg]" %}
6072 interface(MEMORY_INTER) %{
6073 base(0x1e); // RSP
6074 index(0x0); // No Index
6075 scale(0x0); // No Scale
6076 disp($reg); // Stack Offset
6077 %}
6078 %}
6079
6080 operand stackSlotL(sRegL reg)
6081 %{
6082 constraint(ALLOC_IN_RC(stack_slots));
6083 // No match rule because this operand is only generated in matching
6084 // match(RegL);
6085 format %{ "[$reg]" %}
6086 interface(MEMORY_INTER) %{
6087 base(0x1e); // RSP
6088 index(0x0); // No Index
6089 scale(0x0); // No Scale
6090 disp($reg); // Stack Offset
6091 %}
6092 %}
6093
6094 // Operands for expressing Control Flow
6095 // NOTE: Label is a predefined operand which should not be redefined in
6096 // the AD file. It is generically handled within the ADLC.
6097
6098 //----------Conditional Branch Operands----------------------------------------
6099 // Comparison Op - This is the operation of the comparison, and is limited to
6100 // the following set of codes:
6101 // L (<), LE (<=), G (>), GE (>=), E (==), NE (!=)
6102 //
6103 // Other attributes of the comparison, such as unsignedness, are specified
6104 // by the comparison instruction that sets a condition code flags register.
6105 // That result is represented by a flags operand whose subtype is appropriate
6106 // to the unsignedness (etc.) of the comparison.
6107 //
6108 // Later, the instruction which matches both the Comparison Op (a Bool) and
6109 // the flags (produced by the Cmp) specifies the coding of the comparison op
6110 // by matching a specific subtype of Bool operand below, such as cmpOpU.
6111
6112 // used for signed integral comparisons and fp comparisons
6113
6114 operand cmpOp()
6115 %{
6116 match(Bool);
6117
6118 format %{ "" %}
6119 interface(COND_INTER) %{
6120 equal(0x0, "eq");
6121 not_equal(0x1, "ne");
6122 less(0xb, "lt");
6123 greater_equal(0xa, "ge");
6124 less_equal(0xd, "le");
6125 greater(0xc, "gt");
6126 overflow(0x6, "vs");
6127 no_overflow(0x7, "vc");
6128 %}
6129 %}
6130
6131 // used for unsigned integral comparisons
6132
6133 operand cmpOpU()
6134 %{
6135 match(Bool);
6136
6137 format %{ "" %}
6138 interface(COND_INTER) %{
6139 equal(0x0, "eq");
6140 not_equal(0x1, "ne");
6141 less(0x3, "lo");
6142 greater_equal(0x2, "hs");
6143 less_equal(0x9, "ls");
6144 greater(0x8, "hi");
6145 overflow(0x6, "vs");
6146 no_overflow(0x7, "vc");
6147 %}
6148 %}
6149
6150 // used for certain integral comparisons which can be
6151 // converted to cbxx or tbxx instructions
6152
6153 operand cmpOpEqNe()
6154 %{
6155 match(Bool);
6156 op_cost(0);
6157 predicate(n->as_Bool()->_test._test == BoolTest::ne
6158 || n->as_Bool()->_test._test == BoolTest::eq);
6159
6160 format %{ "" %}
6161 interface(COND_INTER) %{
6162 equal(0x0, "eq");
6163 not_equal(0x1, "ne");
6164 less(0xb, "lt");
6165 greater_equal(0xa, "ge");
6166 less_equal(0xd, "le");
6167 greater(0xc, "gt");
6168 overflow(0x6, "vs");
6169 no_overflow(0x7, "vc");
6170 %}
6171 %}
6172
6173 // used for certain integral comparisons which can be
6174 // converted to cbxx or tbxx instructions
6175
6176 operand cmpOpLtGe()
6177 %{
6178 match(Bool);
6179 op_cost(0);
6180
6181 predicate(n->as_Bool()->_test._test == BoolTest::lt
6182 || n->as_Bool()->_test._test == BoolTest::ge);
6183
6184 format %{ "" %}
6185 interface(COND_INTER) %{
6186 equal(0x0, "eq");
6187 not_equal(0x1, "ne");
6188 less(0xb, "lt");
6189 greater_equal(0xa, "ge");
6190 less_equal(0xd, "le");
6191 greater(0xc, "gt");
6192 overflow(0x6, "vs");
6193 no_overflow(0x7, "vc");
6194 %}
6195 %}
6196
6197 // used for certain unsigned integral comparisons which can be
6198 // converted to cbxx or tbxx instructions
6199
6200 operand cmpOpUEqNeLtGe()
6201 %{
6202 match(Bool);
6203 op_cost(0);
6204
6205 predicate(n->as_Bool()->_test._test == BoolTest::eq
6206 || n->as_Bool()->_test._test == BoolTest::ne
6207 || n->as_Bool()->_test._test == BoolTest::lt
6208 || n->as_Bool()->_test._test == BoolTest::ge);
6209
6210 format %{ "" %}
6211 interface(COND_INTER) %{
6212 equal(0x0, "eq");
6213 not_equal(0x1, "ne");
6214 less(0xb, "lt");
6215 greater_equal(0xa, "ge");
6216 less_equal(0xd, "le");
6217 greater(0xc, "gt");
6218 overflow(0x6, "vs");
6219 no_overflow(0x7, "vc");
6220 %}
6221 %}
6222
6223 // Special operand allowing long args to int ops to be truncated for free
6224
6225 operand iRegL2I(iRegL reg) %{
6226
6227 op_cost(0);
6228
6229 match(ConvL2I reg);
6230
6231 format %{ "l2i($reg)" %}
6232
6233 interface(REG_INTER)
6234 %}
6235
6236 opclass vmem2(indirect, indIndex, indOffI2, indOffL2);
6237 opclass vmem4(indirect, indIndex, indOffI4, indOffL4);
6238 opclass vmem8(indirect, indIndex, indOffI8, indOffL8);
6239 opclass vmem16(indirect, indIndex, indOffI16, indOffL16);
6240
6241 //----------OPERAND CLASSES----------------------------------------------------
6242 // Operand Classes are groups of operands that are used as to simplify
6243 // instruction definitions by not requiring the AD writer to specify
6244 // separate instructions for every form of operand when the
6245 // instruction accepts multiple operand types with the same basic
6246 // encoding and format. The classic case of this is memory operands.
6247
6248 // memory is used to define read/write location for load/store
6249 // instruction defs. we can turn a memory op into an Address
6250
6251 opclass memory1(indirect, indIndexScaled, indIndexScaledI2L, indIndexI2L, indIndex, indOffI1, indOffL1,
6252 indirectN, indIndexScaledN, indIndexScaledI2LN, indIndexI2LN, indIndexN);
6253
6254 opclass memory2(indirect, indIndexScaled, indIndexScaledI2L, indIndexI2L, indIndex, indOffI2, indOffL2,
6255 indirectN, indIndexScaledN, indIndexScaledI2LN, indIndexI2LN, indIndexN);
6256
6257 opclass memory4(indirect, indIndexScaled, indIndexScaledI2L, indIndexI2L, indIndex, indOffI4, indOffL4,
6258 indirectN, indIndexScaledN, indIndexScaledI2LN, indIndexI2LN, indIndexN, indOffIN, indOffLN);
6259
6260 opclass memory8(indirect, indIndexScaled, indIndexScaledI2L, indIndexI2L, indIndex, indOffI8, indOffL8,
6261 indirectN, indIndexScaledN, indIndexScaledI2LN, indIndexI2LN, indIndexN, indOffIN, indOffLN);
6262
6263 // All of the memory operands. For the pipeline description.
6264 opclass memory(indirect, indIndexScaled, indIndexScaledI2L, indIndexI2L, indIndex,
6265 indOffI1, indOffL1, indOffI2, indOffL2, indOffI4, indOffL4, indOffI8, indOffL8,
6266 indirectN, indIndexScaledN, indIndexScaledI2LN, indIndexI2LN, indIndexN, indOffIN, indOffLN);
6267
6268
6269 // iRegIorL2I is used for src inputs in rules for 32 bit int (I)
6270 // operations. it allows the src to be either an iRegI or a (ConvL2I
6271 // iRegL). in the latter case the l2i normally planted for a ConvL2I
6272 // can be elided because the 32-bit instruction will just employ the
6273 // lower 32 bits anyway.
6274 //
6275 // n.b. this does not elide all L2I conversions. if the truncated
6276 // value is consumed by more than one operation then the ConvL2I
6277 // cannot be bundled into the consuming nodes so an l2i gets planted
6278 // (actually a movw $dst $src) and the downstream instructions consume
6279 // the result of the l2i as an iRegI input. That's a shame since the
6280 // movw is actually redundant but its not too costly.
6281
6282 opclass iRegIorL2I(iRegI, iRegL2I);
6283
6284 //----------PIPELINE-----------------------------------------------------------
6285 // Rules which define the behavior of the target architectures pipeline.
6286
6287 // For specific pipelines, eg A53, define the stages of that pipeline
6288 //pipe_desc(ISS, EX1, EX2, WR);
6289 #define ISS S0
6290 #define EX1 S1
6291 #define EX2 S2
6292 #define WR S3
6293
6294 // Integer ALU reg operation
6295 pipeline %{
6296
6297 attributes %{
6298 // ARM instructions are of fixed length
6299 fixed_size_instructions; // Fixed size instructions TODO does
6300 max_instructions_per_bundle = 4; // A53 = 2, A57 = 4
6301 // ARM instructions come in 32-bit word units
6302 instruction_unit_size = 4; // An instruction is 4 bytes long
6303 instruction_fetch_unit_size = 64; // The processor fetches one line
6304 instruction_fetch_units = 1; // of 64 bytes
6305
6306 // List of nop instructions
6307 nops( MachNop );
6308 %}
6309
6310 // We don't use an actual pipeline model so don't care about resources
6311 // or description. we do use pipeline classes to introduce fixed
6312 // latencies
6313
6314 //----------RESOURCES----------------------------------------------------------
6315 // Resources are the functional units available to the machine
6316
6317 resources( INS0, INS1, INS01 = INS0 | INS1,
6318 ALU0, ALU1, ALU = ALU0 | ALU1,
6319 MAC,
6320 DIV,
6321 BRANCH,
6322 LDST,
6323 NEON_FP);
6324
6325 //----------PIPELINE DESCRIPTION-----------------------------------------------
6326 // Pipeline Description specifies the stages in the machine's pipeline
6327
6328 // Define the pipeline as a generic 6 stage pipeline
6329 pipe_desc(S0, S1, S2, S3, S4, S5);
6330
6331 //----------PIPELINE CLASSES---------------------------------------------------
6332 // Pipeline Classes describe the stages in which input and output are
6333 // referenced by the hardware pipeline.
6334
6335 pipe_class fp_dop_reg_reg_s(vRegF dst, vRegF src1, vRegF src2)
6336 %{
6337 single_instruction;
6338 src1 : S1(read);
6339 src2 : S2(read);
6340 dst : S5(write);
6341 INS01 : ISS;
6342 NEON_FP : S5;
6343 %}
6344
6345 pipe_class fp_dop_reg_reg_d(vRegD dst, vRegD src1, vRegD src2)
6346 %{
6347 single_instruction;
6348 src1 : S1(read);
6349 src2 : S2(read);
6350 dst : S5(write);
6351 INS01 : ISS;
6352 NEON_FP : S5;
6353 %}
6354
6355 pipe_class fp_uop_s(vRegF dst, vRegF src)
6356 %{
6357 single_instruction;
6358 src : S1(read);
6359 dst : S5(write);
6360 INS01 : ISS;
6361 NEON_FP : S5;
6362 %}
6363
6364 pipe_class fp_uop_d(vRegD dst, vRegD src)
6365 %{
6366 single_instruction;
6367 src : S1(read);
6368 dst : S5(write);
6369 INS01 : ISS;
6370 NEON_FP : S5;
6371 %}
6372
6373 pipe_class fp_d2f(vRegF dst, vRegD src)
6374 %{
6375 single_instruction;
6376 src : S1(read);
6377 dst : S5(write);
6378 INS01 : ISS;
6379 NEON_FP : S5;
6380 %}
6381
6382 pipe_class fp_f2d(vRegD dst, vRegF src)
6383 %{
6384 single_instruction;
6385 src : S1(read);
6386 dst : S5(write);
6387 INS01 : ISS;
6388 NEON_FP : S5;
6389 %}
6390
6391 pipe_class fp_f2i(iRegINoSp dst, vRegF src)
6392 %{
6393 single_instruction;
6394 src : S1(read);
6395 dst : S5(write);
6396 INS01 : ISS;
6397 NEON_FP : S5;
6398 %}
6399
6400 pipe_class fp_f2l(iRegLNoSp dst, vRegF src)
6401 %{
6402 single_instruction;
6403 src : S1(read);
6404 dst : S5(write);
6405 INS01 : ISS;
6406 NEON_FP : S5;
6407 %}
6408
6409 pipe_class fp_i2f(vRegF dst, iRegIorL2I src)
6410 %{
6411 single_instruction;
6412 src : S1(read);
6413 dst : S5(write);
6414 INS01 : ISS;
6415 NEON_FP : S5;
6416 %}
6417
6418 pipe_class fp_l2f(vRegF dst, iRegL src)
6419 %{
6420 single_instruction;
6421 src : S1(read);
6422 dst : S5(write);
6423 INS01 : ISS;
6424 NEON_FP : S5;
6425 %}
6426
6427 pipe_class fp_d2i(iRegINoSp dst, vRegD src)
6428 %{
6429 single_instruction;
6430 src : S1(read);
6431 dst : S5(write);
6432 INS01 : ISS;
6433 NEON_FP : S5;
6434 %}
6435
6436 pipe_class fp_d2l(iRegLNoSp dst, vRegD src)
6437 %{
6438 single_instruction;
6439 src : S1(read);
6440 dst : S5(write);
6441 INS01 : ISS;
6442 NEON_FP : S5;
6443 %}
6444
6445 pipe_class fp_i2d(vRegD dst, iRegIorL2I src)
6446 %{
6447 single_instruction;
6448 src : S1(read);
6449 dst : S5(write);
6450 INS01 : ISS;
6451 NEON_FP : S5;
6452 %}
6453
6454 pipe_class fp_l2d(vRegD dst, iRegIorL2I src)
6455 %{
6456 single_instruction;
6457 src : S1(read);
6458 dst : S5(write);
6459 INS01 : ISS;
6460 NEON_FP : S5;
6461 %}
6462
6463 pipe_class fp_div_s(vRegF dst, vRegF src1, vRegF src2)
6464 %{
6465 single_instruction;
6466 src1 : S1(read);
6467 src2 : S2(read);
6468 dst : S5(write);
6469 INS0 : ISS;
6470 NEON_FP : S5;
6471 %}
6472
6473 pipe_class fp_div_d(vRegD dst, vRegD src1, vRegD src2)
6474 %{
6475 single_instruction;
6476 src1 : S1(read);
6477 src2 : S2(read);
6478 dst : S5(write);
6479 INS0 : ISS;
6480 NEON_FP : S5;
6481 %}
6482
6483 pipe_class fp_cond_reg_reg_s(vRegF dst, vRegF src1, vRegF src2, rFlagsReg cr)
6484 %{
6485 single_instruction;
6486 cr : S1(read);
6487 src1 : S1(read);
6488 src2 : S1(read);
6489 dst : S3(write);
6490 INS01 : ISS;
6491 NEON_FP : S3;
6492 %}
6493
6494 pipe_class fp_cond_reg_reg_d(vRegD dst, vRegD src1, vRegD src2, rFlagsReg cr)
6495 %{
6496 single_instruction;
6497 cr : S1(read);
6498 src1 : S1(read);
6499 src2 : S1(read);
6500 dst : S3(write);
6501 INS01 : ISS;
6502 NEON_FP : S3;
6503 %}
6504
6505 pipe_class fp_imm_s(vRegF dst)
6506 %{
6507 single_instruction;
6508 dst : S3(write);
6509 INS01 : ISS;
6510 NEON_FP : S3;
6511 %}
6512
6513 pipe_class fp_imm_d(vRegD dst)
6514 %{
6515 single_instruction;
6516 dst : S3(write);
6517 INS01 : ISS;
6518 NEON_FP : S3;
6519 %}
6520
6521 pipe_class fp_load_constant_s(vRegF dst)
6522 %{
6523 single_instruction;
6524 dst : S4(write);
6525 INS01 : ISS;
6526 NEON_FP : S4;
6527 %}
6528
6529 pipe_class fp_load_constant_d(vRegD dst)
6530 %{
6531 single_instruction;
6532 dst : S4(write);
6533 INS01 : ISS;
6534 NEON_FP : S4;
6535 %}
6536
6537 pipe_class vmul64(vecD dst, vecD src1, vecD src2)
6538 %{
6539 single_instruction;
6540 dst : S5(write);
6541 src1 : S1(read);
6542 src2 : S1(read);
6543 INS01 : ISS;
6544 NEON_FP : S5;
6545 %}
6546
6547 pipe_class vmul128(vecX dst, vecX src1, vecX src2)
6548 %{
6549 single_instruction;
6550 dst : S5(write);
6551 src1 : S1(read);
6552 src2 : S1(read);
6553 INS0 : ISS;
6554 NEON_FP : S5;
6555 %}
6556
6557 pipe_class vmla64(vecD dst, vecD src1, vecD src2)
6558 %{
6559 single_instruction;
6560 dst : S5(write);
6561 src1 : S1(read);
6562 src2 : S1(read);
6563 dst : S1(read);
6564 INS01 : ISS;
6565 NEON_FP : S5;
6566 %}
6567
6568 pipe_class vmla128(vecX dst, vecX src1, vecX src2)
6569 %{
6570 single_instruction;
6571 dst : S5(write);
6572 src1 : S1(read);
6573 src2 : S1(read);
6574 dst : S1(read);
6575 INS0 : ISS;
6576 NEON_FP : S5;
6577 %}
6578
6579 pipe_class vdop64(vecD dst, vecD src1, vecD src2)
6580 %{
6581 single_instruction;
6582 dst : S4(write);
6583 src1 : S2(read);
6584 src2 : S2(read);
6585 INS01 : ISS;
6586 NEON_FP : S4;
6587 %}
6588
6589 pipe_class vdop128(vecX dst, vecX src1, vecX src2)
6590 %{
6591 single_instruction;
6592 dst : S4(write);
6593 src1 : S2(read);
6594 src2 : S2(read);
6595 INS0 : ISS;
6596 NEON_FP : S4;
6597 %}
6598
6599 pipe_class vlogical64(vecD dst, vecD src1, vecD src2)
6600 %{
6601 single_instruction;
6602 dst : S3(write);
6603 src1 : S2(read);
6604 src2 : S2(read);
6605 INS01 : ISS;
6606 NEON_FP : S3;
6607 %}
6608
6609 pipe_class vlogical128(vecX dst, vecX src1, vecX src2)
6610 %{
6611 single_instruction;
6612 dst : S3(write);
6613 src1 : S2(read);
6614 src2 : S2(read);
6615 INS0 : ISS;
6616 NEON_FP : S3;
6617 %}
6618
6619 pipe_class vshift64(vecD dst, vecD src, vecX shift)
6620 %{
6621 single_instruction;
6622 dst : S3(write);
6623 src : S1(read);
6624 shift : S1(read);
6625 INS01 : ISS;
6626 NEON_FP : S3;
6627 %}
6628
6629 pipe_class vshift128(vecX dst, vecX src, vecX shift)
6630 %{
6631 single_instruction;
6632 dst : S3(write);
6633 src : S1(read);
6634 shift : S1(read);
6635 INS0 : ISS;
6636 NEON_FP : S3;
6637 %}
6638
6639 pipe_class vshift64_imm(vecD dst, vecD src, immI shift)
6640 %{
6641 single_instruction;
6642 dst : S3(write);
6643 src : S1(read);
6644 INS01 : ISS;
6645 NEON_FP : S3;
6646 %}
6647
6648 pipe_class vshift128_imm(vecX dst, vecX src, immI shift)
6649 %{
6650 single_instruction;
6651 dst : S3(write);
6652 src : S1(read);
6653 INS0 : ISS;
6654 NEON_FP : S3;
6655 %}
6656
6657 pipe_class vdop_fp64(vecD dst, vecD src1, vecD src2)
6658 %{
6659 single_instruction;
6660 dst : S5(write);
6661 src1 : S1(read);
6662 src2 : S1(read);
6663 INS01 : ISS;
6664 NEON_FP : S5;
6665 %}
6666
6667 pipe_class vdop_fp128(vecX dst, vecX src1, vecX src2)
6668 %{
6669 single_instruction;
6670 dst : S5(write);
6671 src1 : S1(read);
6672 src2 : S1(read);
6673 INS0 : ISS;
6674 NEON_FP : S5;
6675 %}
6676
6677 pipe_class vmuldiv_fp64(vecD dst, vecD src1, vecD src2)
6678 %{
6679 single_instruction;
6680 dst : S5(write);
6681 src1 : S1(read);
6682 src2 : S1(read);
6683 INS0 : ISS;
6684 NEON_FP : S5;
6685 %}
6686
6687 pipe_class vmuldiv_fp128(vecX dst, vecX src1, vecX src2)
6688 %{
6689 single_instruction;
6690 dst : S5(write);
6691 src1 : S1(read);
6692 src2 : S1(read);
6693 INS0 : ISS;
6694 NEON_FP : S5;
6695 %}
6696
6697 pipe_class vsqrt_fp128(vecX dst, vecX src)
6698 %{
6699 single_instruction;
6700 dst : S5(write);
6701 src : S1(read);
6702 INS0 : ISS;
6703 NEON_FP : S5;
6704 %}
6705
6706 pipe_class vunop_fp64(vecD dst, vecD src)
6707 %{
6708 single_instruction;
6709 dst : S5(write);
6710 src : S1(read);
6711 INS01 : ISS;
6712 NEON_FP : S5;
6713 %}
6714
6715 pipe_class vunop_fp128(vecX dst, vecX src)
6716 %{
6717 single_instruction;
6718 dst : S5(write);
6719 src : S1(read);
6720 INS0 : ISS;
6721 NEON_FP : S5;
6722 %}
6723
6724 pipe_class vdup_reg_reg64(vecD dst, iRegI src)
6725 %{
6726 single_instruction;
6727 dst : S3(write);
6728 src : S1(read);
6729 INS01 : ISS;
6730 NEON_FP : S3;
6731 %}
6732
6733 pipe_class vdup_reg_reg128(vecX dst, iRegI src)
6734 %{
6735 single_instruction;
6736 dst : S3(write);
6737 src : S1(read);
6738 INS01 : ISS;
6739 NEON_FP : S3;
6740 %}
6741
6742 pipe_class vdup_reg_freg64(vecD dst, vRegF src)
6743 %{
6744 single_instruction;
6745 dst : S3(write);
6746 src : S1(read);
6747 INS01 : ISS;
6748 NEON_FP : S3;
6749 %}
6750
6751 pipe_class vdup_reg_freg128(vecX dst, vRegF src)
6752 %{
6753 single_instruction;
6754 dst : S3(write);
6755 src : S1(read);
6756 INS01 : ISS;
6757 NEON_FP : S3;
6758 %}
6759
6760 pipe_class vdup_reg_dreg128(vecX dst, vRegD src)
6761 %{
6762 single_instruction;
6763 dst : S3(write);
6764 src : S1(read);
6765 INS01 : ISS;
6766 NEON_FP : S3;
6767 %}
6768
6769 pipe_class vmovi_reg_imm64(vecD dst)
6770 %{
6771 single_instruction;
6772 dst : S3(write);
6773 INS01 : ISS;
6774 NEON_FP : S3;
6775 %}
6776
6777 pipe_class vmovi_reg_imm128(vecX dst)
6778 %{
6779 single_instruction;
6780 dst : S3(write);
6781 INS0 : ISS;
6782 NEON_FP : S3;
6783 %}
6784
6785 pipe_class vload_reg_mem64(vecD dst, vmem8 mem)
6786 %{
6787 single_instruction;
6788 dst : S5(write);
6789 mem : ISS(read);
6790 INS01 : ISS;
6791 NEON_FP : S3;
6792 %}
6793
6794 pipe_class vload_reg_mem128(vecX dst, vmem16 mem)
6795 %{
6796 single_instruction;
6797 dst : S5(write);
6798 mem : ISS(read);
6799 INS01 : ISS;
6800 NEON_FP : S3;
6801 %}
6802
6803 pipe_class vstore_reg_mem64(vecD src, vmem8 mem)
6804 %{
6805 single_instruction;
6806 mem : ISS(read);
6807 src : S2(read);
6808 INS01 : ISS;
6809 NEON_FP : S3;
6810 %}
6811
6812 pipe_class vstore_reg_mem128(vecD src, vmem16 mem)
6813 %{
6814 single_instruction;
6815 mem : ISS(read);
6816 src : S2(read);
6817 INS01 : ISS;
6818 NEON_FP : S3;
6819 %}
6820
6821 //------- Integer ALU operations --------------------------
6822
6823 // Integer ALU reg-reg operation
6824 // Operands needed in EX1, result generated in EX2
6825 // Eg. ADD x0, x1, x2
6826 pipe_class ialu_reg_reg(iRegI dst, iRegI src1, iRegI src2)
6827 %{
6828 single_instruction;
6829 dst : EX2(write);
6830 src1 : EX1(read);
6831 src2 : EX1(read);
6832 INS01 : ISS; // Dual issue as instruction 0 or 1
6833 ALU : EX2;
6834 %}
6835
6836 // Integer ALU reg-reg operation with constant shift
6837 // Shifted register must be available in LATE_ISS instead of EX1
6838 // Eg. ADD x0, x1, x2, LSL #2
6839 pipe_class ialu_reg_reg_shift(iRegI dst, iRegI src1, iRegI src2, immI shift)
6840 %{
6841 single_instruction;
6842 dst : EX2(write);
6843 src1 : EX1(read);
6844 src2 : ISS(read);
6845 INS01 : ISS;
6846 ALU : EX2;
6847 %}
6848
6849 // Integer ALU reg operation with constant shift
6850 // Eg. LSL x0, x1, #shift
6851 pipe_class ialu_reg_shift(iRegI dst, iRegI src1)
6852 %{
6853 single_instruction;
6854 dst : EX2(write);
6855 src1 : ISS(read);
6856 INS01 : ISS;
6857 ALU : EX2;
6858 %}
6859
6860 // Integer ALU reg-reg operation with variable shift
6861 // Both operands must be available in LATE_ISS instead of EX1
6862 // Result is available in EX1 instead of EX2
6863 // Eg. LSLV x0, x1, x2
6864 pipe_class ialu_reg_reg_vshift(iRegI dst, iRegI src1, iRegI src2)
6865 %{
6866 single_instruction;
6867 dst : EX1(write);
6868 src1 : ISS(read);
6869 src2 : ISS(read);
6870 INS01 : ISS;
6871 ALU : EX1;
6872 %}
6873
6874 // Integer ALU reg-reg operation with extract
6875 // As for _vshift above, but result generated in EX2
6876 // Eg. EXTR x0, x1, x2, #N
6877 pipe_class ialu_reg_reg_extr(iRegI dst, iRegI src1, iRegI src2)
6878 %{
6879 single_instruction;
6880 dst : EX2(write);
6881 src1 : ISS(read);
6882 src2 : ISS(read);
6883 INS1 : ISS; // Can only dual issue as Instruction 1
6884 ALU : EX1;
6885 %}
6886
6887 // Integer ALU reg operation
6888 // Eg. NEG x0, x1
6889 pipe_class ialu_reg(iRegI dst, iRegI src)
6890 %{
6891 single_instruction;
6892 dst : EX2(write);
6893 src : EX1(read);
6894 INS01 : ISS;
6895 ALU : EX2;
6896 %}
6897
6898 // Integer ALU reg mmediate operation
6899 // Eg. ADD x0, x1, #N
6900 pipe_class ialu_reg_imm(iRegI dst, iRegI src1)
6901 %{
6902 single_instruction;
6903 dst : EX2(write);
6904 src1 : EX1(read);
6905 INS01 : ISS;
6906 ALU : EX2;
6907 %}
6908
6909 // Integer ALU immediate operation (no source operands)
6910 // Eg. MOV x0, #N
6911 pipe_class ialu_imm(iRegI dst)
6912 %{
6913 single_instruction;
6914 dst : EX1(write);
6915 INS01 : ISS;
6916 ALU : EX1;
6917 %}
6918
6919 //------- Compare operation -------------------------------
6920
6921 // Compare reg-reg
6922 // Eg. CMP x0, x1
6923 pipe_class icmp_reg_reg(rFlagsReg cr, iRegI op1, iRegI op2)
6924 %{
6925 single_instruction;
6926 // fixed_latency(16);
6927 cr : EX2(write);
6928 op1 : EX1(read);
6929 op2 : EX1(read);
6930 INS01 : ISS;
6931 ALU : EX2;
6932 %}
6933
6934 // Compare reg-reg
6935 // Eg. CMP x0, #N
6936 pipe_class icmp_reg_imm(rFlagsReg cr, iRegI op1)
6937 %{
6938 single_instruction;
6939 // fixed_latency(16);
6940 cr : EX2(write);
6941 op1 : EX1(read);
6942 INS01 : ISS;
6943 ALU : EX2;
6944 %}
6945
6946 //------- Conditional instructions ------------------------
6947
6948 // Conditional no operands
6949 // Eg. CSINC x0, zr, zr, <cond>
6950 pipe_class icond_none(iRegI dst, rFlagsReg cr)
6951 %{
6952 single_instruction;
6953 cr : EX1(read);
6954 dst : EX2(write);
6955 INS01 : ISS;
6956 ALU : EX2;
6957 %}
6958
6959 // Conditional 2 operand
6960 // EG. CSEL X0, X1, X2, <cond>
6961 pipe_class icond_reg_reg(iRegI dst, iRegI src1, iRegI src2, rFlagsReg cr)
6962 %{
6963 single_instruction;
6964 cr : EX1(read);
6965 src1 : EX1(read);
6966 src2 : EX1(read);
6967 dst : EX2(write);
6968 INS01 : ISS;
6969 ALU : EX2;
6970 %}
6971
6972 // Conditional 2 operand
6973 // EG. CSEL X0, X1, X2, <cond>
6974 pipe_class icond_reg(iRegI dst, iRegI src, rFlagsReg cr)
6975 %{
6976 single_instruction;
6977 cr : EX1(read);
6978 src : EX1(read);
6979 dst : EX2(write);
6980 INS01 : ISS;
6981 ALU : EX2;
6982 %}
6983
6984 //------- Multiply pipeline operations --------------------
6985
6986 // Multiply reg-reg
6987 // Eg. MUL w0, w1, w2
6988 pipe_class imul_reg_reg(iRegI dst, iRegI src1, iRegI src2)
6989 %{
6990 single_instruction;
6991 dst : WR(write);
6992 src1 : ISS(read);
6993 src2 : ISS(read);
6994 INS01 : ISS;
6995 MAC : WR;
6996 %}
6997
6998 // Multiply accumulate
6999 // Eg. MADD w0, w1, w2, w3
7000 pipe_class imac_reg_reg(iRegI dst, iRegI src1, iRegI src2, iRegI src3)
7001 %{
7002 single_instruction;
7003 dst : WR(write);
7004 src1 : ISS(read);
7005 src2 : ISS(read);
7006 src3 : ISS(read);
7007 INS01 : ISS;
7008 MAC : WR;
7009 %}
7010
7011 // Eg. MUL w0, w1, w2
7012 pipe_class lmul_reg_reg(iRegI dst, iRegI src1, iRegI src2)
7013 %{
7014 single_instruction;
7015 fixed_latency(3); // Maximum latency for 64 bit mul
7016 dst : WR(write);
7017 src1 : ISS(read);
7018 src2 : ISS(read);
7019 INS01 : ISS;
7020 MAC : WR;
7021 %}
7022
7023 // Multiply accumulate
7024 // Eg. MADD w0, w1, w2, w3
7025 pipe_class lmac_reg_reg(iRegI dst, iRegI src1, iRegI src2, iRegI src3)
7026 %{
7027 single_instruction;
7028 fixed_latency(3); // Maximum latency for 64 bit mul
7029 dst : WR(write);
7030 src1 : ISS(read);
7031 src2 : ISS(read);
7032 src3 : ISS(read);
7033 INS01 : ISS;
7034 MAC : WR;
7035 %}
7036
7037 //------- Divide pipeline operations --------------------
7038
7039 // Eg. SDIV w0, w1, w2
7040 pipe_class idiv_reg_reg(iRegI dst, iRegI src1, iRegI src2)
7041 %{
7042 single_instruction;
7043 fixed_latency(8); // Maximum latency for 32 bit divide
7044 dst : WR(write);
7045 src1 : ISS(read);
7046 src2 : ISS(read);
7047 INS0 : ISS; // Can only dual issue as instruction 0
7048 DIV : WR;
7049 %}
7050
7051 // Eg. SDIV x0, x1, x2
7052 pipe_class ldiv_reg_reg(iRegI dst, iRegI src1, iRegI src2)
7053 %{
7054 single_instruction;
7055 fixed_latency(16); // Maximum latency for 64 bit divide
7056 dst : WR(write);
7057 src1 : ISS(read);
7058 src2 : ISS(read);
7059 INS0 : ISS; // Can only dual issue as instruction 0
7060 DIV : WR;
7061 %}
7062
7063 //------- Load pipeline operations ------------------------
7064
7065 // Load - prefetch
7066 // Eg. PFRM <mem>
7067 pipe_class iload_prefetch(memory mem)
7068 %{
7069 single_instruction;
7070 mem : ISS(read);
7071 INS01 : ISS;
7072 LDST : WR;
7073 %}
7074
7075 // Load - reg, mem
7076 // Eg. LDR x0, <mem>
7077 pipe_class iload_reg_mem(iRegI dst, memory mem)
7078 %{
7079 single_instruction;
7080 dst : WR(write);
7081 mem : ISS(read);
7082 INS01 : ISS;
7083 LDST : WR;
7084 %}
7085
7086 // Load - reg, reg
7087 // Eg. LDR x0, [sp, x1]
7088 pipe_class iload_reg_reg(iRegI dst, iRegI src)
7089 %{
7090 single_instruction;
7091 dst : WR(write);
7092 src : ISS(read);
7093 INS01 : ISS;
7094 LDST : WR;
7095 %}
7096
7097 //------- Store pipeline operations -----------------------
7098
7099 // Store - zr, mem
7100 // Eg. STR zr, <mem>
7101 pipe_class istore_mem(memory mem)
7102 %{
7103 single_instruction;
7104 mem : ISS(read);
7105 INS01 : ISS;
7106 LDST : WR;
7107 %}
7108
7109 // Store - reg, mem
7110 // Eg. STR x0, <mem>
7111 pipe_class istore_reg_mem(iRegI src, memory mem)
7112 %{
7113 single_instruction;
7114 mem : ISS(read);
7115 src : EX2(read);
7116 INS01 : ISS;
7117 LDST : WR;
7118 %}
7119
7120 // Store - reg, reg
7121 // Eg. STR x0, [sp, x1]
7122 pipe_class istore_reg_reg(iRegI dst, iRegI src)
7123 %{
7124 single_instruction;
7125 dst : ISS(read);
7126 src : EX2(read);
7127 INS01 : ISS;
7128 LDST : WR;
7129 %}
7130
7131 //------- Store pipeline operations -----------------------
7132
7133 // Branch
7134 pipe_class pipe_branch()
7135 %{
7136 single_instruction;
7137 INS01 : ISS;
7138 BRANCH : EX1;
7139 %}
7140
7141 // Conditional branch
7142 pipe_class pipe_branch_cond(rFlagsReg cr)
7143 %{
7144 single_instruction;
7145 cr : EX1(read);
7146 INS01 : ISS;
7147 BRANCH : EX1;
7148 %}
7149
7150 // Compare & Branch
7151 // EG. CBZ/CBNZ
7152 pipe_class pipe_cmp_branch(iRegI op1)
7153 %{
7154 single_instruction;
7155 op1 : EX1(read);
7156 INS01 : ISS;
7157 BRANCH : EX1;
7158 %}
7159
7160 //------- Synchronisation operations ----------------------
7161
7162 // Any operation requiring serialization.
7163 // EG. DMB/Atomic Ops/Load Acquire/Str Release
7164 pipe_class pipe_serial()
7165 %{
7166 single_instruction;
7167 force_serialization;
7168 fixed_latency(16);
7169 INS01 : ISS(2); // Cannot dual issue with any other instruction
7170 LDST : WR;
7171 %}
7172
7173 // Generic big/slow expanded idiom - also serialized
7174 pipe_class pipe_slow()
7175 %{
7176 instruction_count(10);
7177 multiple_bundles;
7178 force_serialization;
7179 fixed_latency(16);
7180 INS01 : ISS(2); // Cannot dual issue with any other instruction
7181 LDST : WR;
7182 %}
7183
7184 // Empty pipeline class
7185 pipe_class pipe_class_empty()
7186 %{
7187 single_instruction;
7188 fixed_latency(0);
7189 %}
7190
7191 // Default pipeline class.
7192 pipe_class pipe_class_default()
7193 %{
7194 single_instruction;
7195 fixed_latency(2);
7196 %}
7197
7198 // Pipeline class for compares.
7199 pipe_class pipe_class_compare()
7200 %{
7201 single_instruction;
7202 fixed_latency(16);
7203 %}
7204
7205 // Pipeline class for memory operations.
7206 pipe_class pipe_class_memory()
7207 %{
7208 single_instruction;
7209 fixed_latency(16);
7210 %}
7211
7212 // Pipeline class for call.
7213 pipe_class pipe_class_call()
7214 %{
7215 single_instruction;
7216 fixed_latency(100);
7217 %}
7218
7219 // Define the class for the Nop node.
7220 define %{
7221 MachNop = pipe_class_empty;
7222 %}
7223
7224 %}
7225 //----------INSTRUCTIONS-------------------------------------------------------
7226 //
7227 // match -- States which machine-independent subtree may be replaced
7228 // by this instruction.
7229 // ins_cost -- The estimated cost of this instruction is used by instruction
7230 // selection to identify a minimum cost tree of machine
7231 // instructions that matches a tree of machine-independent
7232 // instructions.
7233 // format -- A string providing the disassembly for this instruction.
7234 // The value of an instruction's operand may be inserted
7235 // by referring to it with a '$' prefix.
7236 // opcode -- Three instruction opcodes may be provided. These are referred
7237 // to within an encode class as $primary, $secondary, and $tertiary
7238 // rrspectively. The primary opcode is commonly used to
7239 // indicate the type of machine instruction, while secondary
7240 // and tertiary are often used for prefix options or addressing
7241 // modes.
7242 // ins_encode -- A list of encode classes with parameters. The encode class
7243 // name must have been defined in an 'enc_class' specification
7244 // in the encode section of the architecture description.
7245
7246 // ============================================================================
7247 // Memory (Load/Store) Instructions
7248
7249 // Load Instructions
7250
7251 // Load Byte (8 bit signed)
7252 instruct loadB(iRegINoSp dst, memory1 mem)
7253 %{
7254 match(Set dst (LoadB mem));
7255 predicate(!needs_acquiring_load(n));
7256
7257 ins_cost(4 * INSN_COST);
7258 format %{ "ldrsbw $dst, $mem\t# byte" %}
7259
7260 ins_encode(aarch64_enc_ldrsbw(dst, mem));
7261
7262 ins_pipe(iload_reg_mem);
7263 %}
7264
7265 // Load Byte (8 bit signed) into long
7266 instruct loadB2L(iRegLNoSp dst, memory1 mem)
7267 %{
7268 match(Set dst (ConvI2L (LoadB mem)));
7269 predicate(!needs_acquiring_load(n->in(1)));
7270
7271 ins_cost(4 * INSN_COST);
7272 format %{ "ldrsb $dst, $mem\t# byte" %}
7273
7274 ins_encode(aarch64_enc_ldrsb(dst, mem));
7275
7276 ins_pipe(iload_reg_mem);
7277 %}
7278
7279 // Load Byte (8 bit unsigned)
7280 instruct loadUB(iRegINoSp dst, memory1 mem)
7281 %{
7282 match(Set dst (LoadUB mem));
7283 predicate(!needs_acquiring_load(n));
7284
7285 ins_cost(4 * INSN_COST);
7286 format %{ "ldrbw $dst, $mem\t# byte" %}
7287
7288 ins_encode(aarch64_enc_ldrb(dst, mem));
7289
7290 ins_pipe(iload_reg_mem);
7291 %}
7292
7293 // Load Byte (8 bit unsigned) into long
7294 instruct loadUB2L(iRegLNoSp dst, memory1 mem)
7295 %{
7296 match(Set dst (ConvI2L (LoadUB mem)));
7297 predicate(!needs_acquiring_load(n->in(1)));
7298
7299 ins_cost(4 * INSN_COST);
7300 format %{ "ldrb $dst, $mem\t# byte" %}
7301
7302 ins_encode(aarch64_enc_ldrb(dst, mem));
7303
7304 ins_pipe(iload_reg_mem);
7305 %}
7306
7307 // Load Short (16 bit signed)
7308 instruct loadS(iRegINoSp dst, memory2 mem)
7309 %{
7310 match(Set dst (LoadS mem));
7311 predicate(!needs_acquiring_load(n));
7312
7313 ins_cost(4 * INSN_COST);
7314 format %{ "ldrshw $dst, $mem\t# short" %}
7315
7316 ins_encode(aarch64_enc_ldrshw(dst, mem));
7317
7318 ins_pipe(iload_reg_mem);
7319 %}
7320
7321 // Load Short (16 bit signed) into long
7322 instruct loadS2L(iRegLNoSp dst, memory2 mem)
7323 %{
7324 match(Set dst (ConvI2L (LoadS mem)));
7325 predicate(!needs_acquiring_load(n->in(1)));
7326
7327 ins_cost(4 * INSN_COST);
7328 format %{ "ldrsh $dst, $mem\t# short" %}
7329
7330 ins_encode(aarch64_enc_ldrsh(dst, mem));
7331
7332 ins_pipe(iload_reg_mem);
7333 %}
7334
7335 // Load Char (16 bit unsigned)
7336 instruct loadUS(iRegINoSp dst, memory2 mem)
7337 %{
7338 match(Set dst (LoadUS mem));
7339 predicate(!needs_acquiring_load(n));
7340
7341 ins_cost(4 * INSN_COST);
7342 format %{ "ldrh $dst, $mem\t# short" %}
7343
7344 ins_encode(aarch64_enc_ldrh(dst, mem));
7345
7346 ins_pipe(iload_reg_mem);
7347 %}
7348
7349 // Load Short/Char (16 bit unsigned) into long
7350 instruct loadUS2L(iRegLNoSp dst, memory2 mem)
7351 %{
7352 match(Set dst (ConvI2L (LoadUS mem)));
7353 predicate(!needs_acquiring_load(n->in(1)));
7354
7355 ins_cost(4 * INSN_COST);
7356 format %{ "ldrh $dst, $mem\t# short" %}
7357
7358 ins_encode(aarch64_enc_ldrh(dst, mem));
7359
7360 ins_pipe(iload_reg_mem);
7361 %}
7362
7363 // Load Integer (32 bit signed)
7364 instruct loadI(iRegINoSp dst, memory4 mem)
7365 %{
7366 match(Set dst (LoadI mem));
7367 predicate(!needs_acquiring_load(n));
7368
7369 ins_cost(4 * INSN_COST);
7370 format %{ "ldrw $dst, $mem\t# int" %}
7371
7372 ins_encode(aarch64_enc_ldrw(dst, mem));
7373
7374 ins_pipe(iload_reg_mem);
7375 %}
7376
7377 // Load Integer (32 bit signed) into long
7378 instruct loadI2L(iRegLNoSp dst, memory4 mem)
7379 %{
7380 match(Set dst (ConvI2L (LoadI mem)));
7381 predicate(!needs_acquiring_load(n->in(1)));
7382
7383 ins_cost(4 * INSN_COST);
7384 format %{ "ldrsw $dst, $mem\t# int" %}
7385
7386 ins_encode(aarch64_enc_ldrsw(dst, mem));
7387
7388 ins_pipe(iload_reg_mem);
7389 %}
7390
7391 // Load Integer (32 bit unsigned) into long
7392 instruct loadUI2L(iRegLNoSp dst, memory4 mem, immL_32bits mask)
7393 %{
7394 match(Set dst (AndL (ConvI2L (LoadI mem)) mask));
7395 predicate(!needs_acquiring_load(n->in(1)->in(1)->as_Load()));
7396
7397 ins_cost(4 * INSN_COST);
7398 format %{ "ldrw $dst, $mem\t# int" %}
7399
7400 ins_encode(aarch64_enc_ldrw(dst, mem));
7401
7402 ins_pipe(iload_reg_mem);
7403 %}
7404
7405 // Load Long (64 bit signed)
7406 instruct loadL(iRegLNoSp dst, memory8 mem)
7407 %{
7408 match(Set dst (LoadL mem));
7409 predicate(!needs_acquiring_load(n));
7410
7411 ins_cost(4 * INSN_COST);
7412 format %{ "ldr $dst, $mem\t# int" %}
7413
7414 ins_encode(aarch64_enc_ldr(dst, mem));
7415
7416 ins_pipe(iload_reg_mem);
7417 %}
7418
7419 // Load Range
7420 instruct loadRange(iRegINoSp dst, memory4 mem)
7421 %{
7422 match(Set dst (LoadRange mem));
7423
7424 ins_cost(4 * INSN_COST);
7425 format %{ "ldrw $dst, $mem\t# range" %}
7426
7427 ins_encode(aarch64_enc_ldrw(dst, mem));
7428
7429 ins_pipe(iload_reg_mem);
7430 %}
7431
7432 // Load Pointer
7433 instruct loadP(iRegPNoSp dst, memory8 mem)
7434 %{
7435 match(Set dst (LoadP mem));
7436 predicate(!needs_acquiring_load(n) && (n->as_Load()->barrier_data() == 0));
7437
7438 ins_cost(4 * INSN_COST);
7439 format %{ "ldr $dst, $mem\t# ptr" %}
7440
7441 ins_encode(aarch64_enc_ldr(dst, mem));
7442
7443 ins_pipe(iload_reg_mem);
7444 %}
7445
7446 // Load Compressed Pointer
7447 instruct loadN(iRegNNoSp dst, memory4 mem)
7448 %{
7449 match(Set dst (LoadN mem));
7450 predicate(!needs_acquiring_load(n));
7451
7452 ins_cost(4 * INSN_COST);
7453 format %{ "ldrw $dst, $mem\t# compressed ptr" %}
7454
7455 ins_encode(aarch64_enc_ldrw(dst, mem));
7456
7457 ins_pipe(iload_reg_mem);
7458 %}
7459
7460 // Load Klass Pointer
7461 instruct loadKlass(iRegPNoSp dst, memory8 mem)
7462 %{
7463 match(Set dst (LoadKlass mem));
7464 predicate(!needs_acquiring_load(n));
7465
7466 ins_cost(4 * INSN_COST);
7467 format %{ "ldr $dst, $mem\t# class" %}
7468
7469 ins_encode(aarch64_enc_ldr(dst, mem));
7470
7471 ins_pipe(iload_reg_mem);
7472 %}
7473
7474 // Load Narrow Klass Pointer
7475 instruct loadNKlass(iRegNNoSp dst, memory4 mem)
7476 %{
7477 match(Set dst (LoadNKlass mem));
7478 predicate(!needs_acquiring_load(n) && !UseCompactObjectHeaders);
7479
7480 ins_cost(4 * INSN_COST);
7481 format %{ "ldrw $dst, $mem\t# compressed class ptr" %}
7482
7483 ins_encode(aarch64_enc_ldrw(dst, mem));
7484
7485 ins_pipe(iload_reg_mem);
7486 %}
7487
7488 instruct loadNKlassLilliput(iRegNNoSp dst, memory4 mem, rFlagsReg cr)
7489 %{
7490 match(Set dst (LoadNKlass mem));
7491 effect(KILL cr);
7492 predicate(!needs_acquiring_load(n) && UseCompactObjectHeaders);
7493
7494 ins_cost(4 * INSN_COST);
7495 format %{ "ldrw $dst, $mem\t# compressed class ptr" %}
7496 ins_encode %{
7497 assert($mem$$disp == oopDesc::klass_offset_in_bytes(), "expect correct offset");
7498 assert($mem$$index$$Register == noreg, "expect no index");
7499 Register dst = $dst$$Register;
7500 Register obj = $mem$$base$$Register;
7501 C2LoadNKlassStub* stub = new (Compile::current()->comp_arena()) C2LoadNKlassStub(dst);
7502 Compile::current()->output()->add_stub(stub);
7503 __ ldr(dst, Address(obj, oopDesc::mark_offset_in_bytes()));
7504 // NOTE: We can't use tbnz here, because the target is sometimes too far away
7505 // and cannot be encoded.
7506 __ tst(dst, markWord::monitor_value);
7507 __ br(Assembler::NE, stub->entry());
7508 __ bind(stub->continuation());
7509 __ lsr(dst, dst, markWord::klass_shift);
7510 %}
7511 ins_pipe(pipe_slow);
7512 %}
7513
7514 // Load Float
7515 instruct loadF(vRegF dst, memory4 mem)
7516 %{
7517 match(Set dst (LoadF mem));
7518 predicate(!needs_acquiring_load(n));
7519
7520 ins_cost(4 * INSN_COST);
7521 format %{ "ldrs $dst, $mem\t# float" %}
7522
7523 ins_encode( aarch64_enc_ldrs(dst, mem) );
7524
7525 ins_pipe(pipe_class_memory);
7526 %}
7527
7528 // Load Double
7529 instruct loadD(vRegD dst, memory8 mem)
7530 %{
7531 match(Set dst (LoadD mem));
7532 predicate(!needs_acquiring_load(n));
7533
7534 ins_cost(4 * INSN_COST);
7535 format %{ "ldrd $dst, $mem\t# double" %}
7536
7537 ins_encode( aarch64_enc_ldrd(dst, mem) );
7538
7539 ins_pipe(pipe_class_memory);
7540 %}
7541
7542
7543 // Load Int Constant
7544 instruct loadConI(iRegINoSp dst, immI src)
7545 %{
7546 match(Set dst src);
7547
7548 ins_cost(INSN_COST);
7549 format %{ "mov $dst, $src\t# int" %}
7550
7551 ins_encode( aarch64_enc_movw_imm(dst, src) );
7552
7553 ins_pipe(ialu_imm);
7554 %}
7555
7556 // Load Long Constant
7557 instruct loadConL(iRegLNoSp dst, immL src)
7558 %{
7559 match(Set dst src);
7560
7561 ins_cost(INSN_COST);
7562 format %{ "mov $dst, $src\t# long" %}
7563
7564 ins_encode( aarch64_enc_mov_imm(dst, src) );
7565
7566 ins_pipe(ialu_imm);
7567 %}
7568
7569 // Load Pointer Constant
7570
7571 instruct loadConP(iRegPNoSp dst, immP con)
7572 %{
7573 match(Set dst con);
7574
7575 ins_cost(INSN_COST * 4);
7576 format %{
7577 "mov $dst, $con\t# ptr\n\t"
7578 %}
7579
7580 ins_encode(aarch64_enc_mov_p(dst, con));
7581
7582 ins_pipe(ialu_imm);
7583 %}
7584
7585 // Load Null Pointer Constant
7586
7587 instruct loadConP0(iRegPNoSp dst, immP0 con)
7588 %{
7589 match(Set dst con);
7590
7591 ins_cost(INSN_COST);
7592 format %{ "mov $dst, $con\t# NULL ptr" %}
7593
7594 ins_encode(aarch64_enc_mov_p0(dst, con));
7595
7596 ins_pipe(ialu_imm);
7597 %}
7598
7599 // Load Pointer Constant One
7600
7601 instruct loadConP1(iRegPNoSp dst, immP_1 con)
7602 %{
7603 match(Set dst con);
7604
7605 ins_cost(INSN_COST);
7606 format %{ "mov $dst, $con\t# NULL ptr" %}
7607
7608 ins_encode(aarch64_enc_mov_p1(dst, con));
7609
7610 ins_pipe(ialu_imm);
7611 %}
7612
7613 // Load Byte Map Base Constant
7614
7615 instruct loadByteMapBase(iRegPNoSp dst, immByteMapBase con)
7616 %{
7617 match(Set dst con);
7618
7619 ins_cost(INSN_COST);
7620 format %{ "adr $dst, $con\t# Byte Map Base" %}
7621
7622 ins_encode(aarch64_enc_mov_byte_map_base(dst, con));
7623
7624 ins_pipe(ialu_imm);
7625 %}
7626
7627 // Load Narrow Pointer Constant
7628
7629 instruct loadConN(iRegNNoSp dst, immN con)
7630 %{
7631 match(Set dst con);
7632
7633 ins_cost(INSN_COST * 4);
7634 format %{ "mov $dst, $con\t# compressed ptr" %}
7635
7636 ins_encode(aarch64_enc_mov_n(dst, con));
7637
7638 ins_pipe(ialu_imm);
7639 %}
7640
7641 // Load Narrow Null Pointer Constant
7642
7643 instruct loadConN0(iRegNNoSp dst, immN0 con)
7644 %{
7645 match(Set dst con);
7646
7647 ins_cost(INSN_COST);
7648 format %{ "mov $dst, $con\t# compressed NULL ptr" %}
7649
7650 ins_encode(aarch64_enc_mov_n0(dst, con));
7651
7652 ins_pipe(ialu_imm);
7653 %}
7654
7655 // Load Narrow Klass Constant
7656
7657 instruct loadConNKlass(iRegNNoSp dst, immNKlass con)
7658 %{
7659 match(Set dst con);
7660
7661 ins_cost(INSN_COST);
7662 format %{ "mov $dst, $con\t# compressed klass ptr" %}
7663
7664 ins_encode(aarch64_enc_mov_nk(dst, con));
7665
7666 ins_pipe(ialu_imm);
7667 %}
7668
7669 // Load Packed Float Constant
7670
7671 instruct loadConF_packed(vRegF dst, immFPacked con) %{
7672 match(Set dst con);
7673 ins_cost(INSN_COST * 4);
7674 format %{ "fmovs $dst, $con"%}
7675 ins_encode %{
7676 __ fmovs(as_FloatRegister($dst$$reg), (double)$con$$constant);
7677 %}
7678
7679 ins_pipe(fp_imm_s);
7680 %}
7681
7682 // Load Float Constant
7683
7684 instruct loadConF(vRegF dst, immF con) %{
7685 match(Set dst con);
7686
7687 ins_cost(INSN_COST * 4);
7688
7689 format %{
7690 "ldrs $dst, [$constantaddress]\t# load from constant table: float=$con\n\t"
7691 %}
7692
7693 ins_encode %{
7694 __ ldrs(as_FloatRegister($dst$$reg), $constantaddress($con));
7695 %}
7696
7697 ins_pipe(fp_load_constant_s);
7698 %}
7699
7700 // Load Packed Double Constant
7701
7702 instruct loadConD_packed(vRegD dst, immDPacked con) %{
7703 match(Set dst con);
7704 ins_cost(INSN_COST);
7705 format %{ "fmovd $dst, $con"%}
7706 ins_encode %{
7707 __ fmovd(as_FloatRegister($dst$$reg), $con$$constant);
7708 %}
7709
7710 ins_pipe(fp_imm_d);
7711 %}
7712
7713 // Load Double Constant
7714
7715 instruct loadConD(vRegD dst, immD con) %{
7716 match(Set dst con);
7717
7718 ins_cost(INSN_COST * 5);
7719 format %{
7720 "ldrd $dst, [$constantaddress]\t# load from constant table: float=$con\n\t"
7721 %}
7722
7723 ins_encode %{
7724 __ ldrd(as_FloatRegister($dst$$reg), $constantaddress($con));
7725 %}
7726
7727 ins_pipe(fp_load_constant_d);
7728 %}
7729
7730 // Store Instructions
7731
7732 // Store CMS card-mark Immediate
7733 instruct storeimmCM0(immI0 zero, memory1 mem)
7734 %{
7735 match(Set mem (StoreCM mem zero));
7736
7737 ins_cost(INSN_COST);
7738 format %{ "storestore (elided)\n\t"
7739 "strb zr, $mem\t# byte" %}
7740
7741 ins_encode(aarch64_enc_strb0(mem));
7742
7743 ins_pipe(istore_mem);
7744 %}
7745
7746 // Store CMS card-mark Immediate with intervening StoreStore
7747 // needed when using CMS with no conditional card marking
7748 instruct storeimmCM0_ordered(immI0 zero, memory1 mem)
7749 %{
7750 match(Set mem (StoreCM mem zero));
7751
7752 ins_cost(INSN_COST * 2);
7753 format %{ "storestore\n\t"
7754 "dmb ishst"
7755 "\n\tstrb zr, $mem\t# byte" %}
7756
7757 ins_encode(aarch64_enc_strb0_ordered(mem));
7758
7759 ins_pipe(istore_mem);
7760 %}
7761
7762 // Store Byte
7763 instruct storeB(iRegIorL2I src, memory1 mem)
7764 %{
7765 match(Set mem (StoreB mem src));
7766 predicate(!needs_releasing_store(n));
7767
7768 ins_cost(INSN_COST);
7769 format %{ "strb $src, $mem\t# byte" %}
7770
7771 ins_encode(aarch64_enc_strb(src, mem));
7772
7773 ins_pipe(istore_reg_mem);
7774 %}
7775
7776
7777 instruct storeimmB0(immI0 zero, memory1 mem)
7778 %{
7779 match(Set mem (StoreB mem zero));
7780 predicate(!needs_releasing_store(n));
7781
7782 ins_cost(INSN_COST);
7783 format %{ "strb rscractch2, $mem\t# byte" %}
7784
7785 ins_encode(aarch64_enc_strb0(mem));
7786
7787 ins_pipe(istore_mem);
7788 %}
7789
7790 // Store Char/Short
7791 instruct storeC(iRegIorL2I src, memory2 mem)
7792 %{
7793 match(Set mem (StoreC mem src));
7794 predicate(!needs_releasing_store(n));
7795
7796 ins_cost(INSN_COST);
7797 format %{ "strh $src, $mem\t# short" %}
7798
7799 ins_encode(aarch64_enc_strh(src, mem));
7800
7801 ins_pipe(istore_reg_mem);
7802 %}
7803
7804 instruct storeimmC0(immI0 zero, memory2 mem)
7805 %{
7806 match(Set mem (StoreC mem zero));
7807 predicate(!needs_releasing_store(n));
7808
7809 ins_cost(INSN_COST);
7810 format %{ "strh zr, $mem\t# short" %}
7811
7812 ins_encode(aarch64_enc_strh0(mem));
7813
7814 ins_pipe(istore_mem);
7815 %}
7816
7817 // Store Integer
7818
7819 instruct storeI(iRegIorL2I src, memory4 mem)
7820 %{
7821 match(Set mem(StoreI mem src));
7822 predicate(!needs_releasing_store(n));
7823
7824 ins_cost(INSN_COST);
7825 format %{ "strw $src, $mem\t# int" %}
7826
7827 ins_encode(aarch64_enc_strw(src, mem));
7828
7829 ins_pipe(istore_reg_mem);
7830 %}
7831
7832 instruct storeimmI0(immI0 zero, memory4 mem)
7833 %{
7834 match(Set mem(StoreI mem zero));
7835 predicate(!needs_releasing_store(n));
7836
7837 ins_cost(INSN_COST);
7838 format %{ "strw zr, $mem\t# int" %}
7839
7840 ins_encode(aarch64_enc_strw0(mem));
7841
7842 ins_pipe(istore_mem);
7843 %}
7844
7845 // Store Long (64 bit signed)
7846 instruct storeL(iRegL src, memory8 mem)
7847 %{
7848 match(Set mem (StoreL mem src));
7849 predicate(!needs_releasing_store(n));
7850
7851 ins_cost(INSN_COST);
7852 format %{ "str $src, $mem\t# int" %}
7853
7854 ins_encode(aarch64_enc_str(src, mem));
7855
7856 ins_pipe(istore_reg_mem);
7857 %}
7858
7859 // Store Long (64 bit signed)
7860 instruct storeimmL0(immL0 zero, memory8 mem)
7861 %{
7862 match(Set mem (StoreL mem zero));
7863 predicate(!needs_releasing_store(n));
7864
7865 ins_cost(INSN_COST);
7866 format %{ "str zr, $mem\t# int" %}
7867
7868 ins_encode(aarch64_enc_str0(mem));
7869
7870 ins_pipe(istore_mem);
7871 %}
7872
7873 // Store Pointer
7874 instruct storeP(iRegP src, memory8 mem)
7875 %{
7876 match(Set mem (StoreP mem src));
7877 predicate(!needs_releasing_store(n));
7878
7879 ins_cost(INSN_COST);
7880 format %{ "str $src, $mem\t# ptr" %}
7881
7882 ins_encode(aarch64_enc_str(src, mem));
7883
7884 ins_pipe(istore_reg_mem);
7885 %}
7886
7887 // Store Pointer
7888 instruct storeimmP0(immP0 zero, memory8 mem)
7889 %{
7890 match(Set mem (StoreP mem zero));
7891 predicate(!needs_releasing_store(n));
7892
7893 ins_cost(INSN_COST);
7894 format %{ "str zr, $mem\t# ptr" %}
7895
7896 ins_encode(aarch64_enc_str0(mem));
7897
7898 ins_pipe(istore_mem);
7899 %}
7900
7901 // Store Compressed Pointer
7902 instruct storeN(iRegN src, memory4 mem)
7903 %{
7904 match(Set mem (StoreN mem src));
7905 predicate(!needs_releasing_store(n));
7906
7907 ins_cost(INSN_COST);
7908 format %{ "strw $src, $mem\t# compressed ptr" %}
7909
7910 ins_encode(aarch64_enc_strw(src, mem));
7911
7912 ins_pipe(istore_reg_mem);
7913 %}
7914
7915 instruct storeImmN0(immN0 zero, memory4 mem)
7916 %{
7917 match(Set mem (StoreN mem zero));
7918 predicate(!needs_releasing_store(n));
7919
7920 ins_cost(INSN_COST);
7921 format %{ "strw zr, $mem\t# compressed ptr" %}
7922
7923 ins_encode(aarch64_enc_strw0(mem));
7924
7925 ins_pipe(istore_mem);
7926 %}
7927
7928 // Store Float
7929 instruct storeF(vRegF src, memory4 mem)
7930 %{
7931 match(Set mem (StoreF mem src));
7932 predicate(!needs_releasing_store(n));
7933
7934 ins_cost(INSN_COST);
7935 format %{ "strs $src, $mem\t# float" %}
7936
7937 ins_encode( aarch64_enc_strs(src, mem) );
7938
7939 ins_pipe(pipe_class_memory);
7940 %}
7941
7942 // TODO
7943 // implement storeImmF0 and storeFImmPacked
7944
7945 // Store Double
7946 instruct storeD(vRegD src, memory8 mem)
7947 %{
7948 match(Set mem (StoreD mem src));
7949 predicate(!needs_releasing_store(n));
7950
7951 ins_cost(INSN_COST);
7952 format %{ "strd $src, $mem\t# double" %}
7953
7954 ins_encode( aarch64_enc_strd(src, mem) );
7955
7956 ins_pipe(pipe_class_memory);
7957 %}
7958
7959 // Store Compressed Klass Pointer
7960 instruct storeNKlass(iRegN src, memory4 mem)
7961 %{
7962 predicate(!needs_releasing_store(n));
7963 match(Set mem (StoreNKlass mem src));
7964
7965 ins_cost(INSN_COST);
7966 format %{ "strw $src, $mem\t# compressed klass ptr" %}
7967
7968 ins_encode(aarch64_enc_strw(src, mem));
7969
7970 ins_pipe(istore_reg_mem);
7971 %}
7972
7973 // TODO
7974 // implement storeImmD0 and storeDImmPacked
7975
7976 // prefetch instructions
7977 // Must be safe to execute with invalid address (cannot fault).
7978
7979 instruct prefetchalloc( memory8 mem ) %{
7980 match(PrefetchAllocation mem);
7981
7982 ins_cost(INSN_COST);
7983 format %{ "prfm $mem, PSTL1KEEP\t# Prefetch into level 1 cache write keep" %}
7984
7985 ins_encode( aarch64_enc_prefetchw(mem) );
7986
7987 ins_pipe(iload_prefetch);
7988 %}
7989
7990 // ---------------- volatile loads and stores ----------------
7991
7992 // Load Byte (8 bit signed)
7993 instruct loadB_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
7994 %{
7995 match(Set dst (LoadB mem));
7996
7997 ins_cost(VOLATILE_REF_COST);
7998 format %{ "ldarsb $dst, $mem\t# byte" %}
7999
8000 ins_encode(aarch64_enc_ldarsb(dst, mem));
8001
8002 ins_pipe(pipe_serial);
8003 %}
8004
8005 // Load Byte (8 bit signed) into long
8006 instruct loadB2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
8007 %{
8008 match(Set dst (ConvI2L (LoadB mem)));
8009
8010 ins_cost(VOLATILE_REF_COST);
8011 format %{ "ldarsb $dst, $mem\t# byte" %}
8012
8013 ins_encode(aarch64_enc_ldarsb(dst, mem));
8014
8015 ins_pipe(pipe_serial);
8016 %}
8017
8018 // Load Byte (8 bit unsigned)
8019 instruct loadUB_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
8020 %{
8021 match(Set dst (LoadUB mem));
8022
8023 ins_cost(VOLATILE_REF_COST);
8024 format %{ "ldarb $dst, $mem\t# byte" %}
8025
8026 ins_encode(aarch64_enc_ldarb(dst, mem));
8027
8028 ins_pipe(pipe_serial);
8029 %}
8030
8031 // Load Byte (8 bit unsigned) into long
8032 instruct loadUB2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
8033 %{
8034 match(Set dst (ConvI2L (LoadUB mem)));
8035
8036 ins_cost(VOLATILE_REF_COST);
8037 format %{ "ldarb $dst, $mem\t# byte" %}
8038
8039 ins_encode(aarch64_enc_ldarb(dst, mem));
8040
8041 ins_pipe(pipe_serial);
8042 %}
8043
8044 // Load Short (16 bit signed)
8045 instruct loadS_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
8046 %{
8047 match(Set dst (LoadS mem));
8048
8049 ins_cost(VOLATILE_REF_COST);
8050 format %{ "ldarshw $dst, $mem\t# short" %}
8051
8052 ins_encode(aarch64_enc_ldarshw(dst, mem));
8053
8054 ins_pipe(pipe_serial);
8055 %}
8056
8057 instruct loadUS_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
8058 %{
8059 match(Set dst (LoadUS mem));
8060
8061 ins_cost(VOLATILE_REF_COST);
8062 format %{ "ldarhw $dst, $mem\t# short" %}
8063
8064 ins_encode(aarch64_enc_ldarhw(dst, mem));
8065
8066 ins_pipe(pipe_serial);
8067 %}
8068
8069 // Load Short/Char (16 bit unsigned) into long
8070 instruct loadUS2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
8071 %{
8072 match(Set dst (ConvI2L (LoadUS mem)));
8073
8074 ins_cost(VOLATILE_REF_COST);
8075 format %{ "ldarh $dst, $mem\t# short" %}
8076
8077 ins_encode(aarch64_enc_ldarh(dst, mem));
8078
8079 ins_pipe(pipe_serial);
8080 %}
8081
8082 // Load Short/Char (16 bit signed) into long
8083 instruct loadS2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
8084 %{
8085 match(Set dst (ConvI2L (LoadS mem)));
8086
8087 ins_cost(VOLATILE_REF_COST);
8088 format %{ "ldarh $dst, $mem\t# short" %}
8089
8090 ins_encode(aarch64_enc_ldarsh(dst, mem));
8091
8092 ins_pipe(pipe_serial);
8093 %}
8094
8095 // Load Integer (32 bit signed)
8096 instruct loadI_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
8097 %{
8098 match(Set dst (LoadI mem));
8099
8100 ins_cost(VOLATILE_REF_COST);
8101 format %{ "ldarw $dst, $mem\t# int" %}
8102
8103 ins_encode(aarch64_enc_ldarw(dst, mem));
8104
8105 ins_pipe(pipe_serial);
8106 %}
8107
8108 // Load Integer (32 bit unsigned) into long
8109 instruct loadUI2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem, immL_32bits mask)
8110 %{
8111 match(Set dst (AndL (ConvI2L (LoadI mem)) mask));
8112
8113 ins_cost(VOLATILE_REF_COST);
8114 format %{ "ldarw $dst, $mem\t# int" %}
8115
8116 ins_encode(aarch64_enc_ldarw(dst, mem));
8117
8118 ins_pipe(pipe_serial);
8119 %}
8120
8121 // Load Long (64 bit signed)
8122 instruct loadL_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
8123 %{
8124 match(Set dst (LoadL mem));
8125
8126 ins_cost(VOLATILE_REF_COST);
8127 format %{ "ldar $dst, $mem\t# int" %}
8128
8129 ins_encode(aarch64_enc_ldar(dst, mem));
8130
8131 ins_pipe(pipe_serial);
8132 %}
8133
8134 // Load Pointer
8135 instruct loadP_volatile(iRegPNoSp dst, /* sync_memory*/indirect mem)
8136 %{
8137 match(Set dst (LoadP mem));
8138 predicate(n->as_Load()->barrier_data() == 0);
8139
8140 ins_cost(VOLATILE_REF_COST);
8141 format %{ "ldar $dst, $mem\t# ptr" %}
8142
8143 ins_encode(aarch64_enc_ldar(dst, mem));
8144
8145 ins_pipe(pipe_serial);
8146 %}
8147
8148 // Load Compressed Pointer
8149 instruct loadN_volatile(iRegNNoSp dst, /* sync_memory*/indirect mem)
8150 %{
8151 match(Set dst (LoadN mem));
8152
8153 ins_cost(VOLATILE_REF_COST);
8154 format %{ "ldarw $dst, $mem\t# compressed ptr" %}
8155
8156 ins_encode(aarch64_enc_ldarw(dst, mem));
8157
8158 ins_pipe(pipe_serial);
8159 %}
8160
8161 // Load Float
8162 instruct loadF_volatile(vRegF dst, /* sync_memory*/indirect mem)
8163 %{
8164 match(Set dst (LoadF mem));
8165
8166 ins_cost(VOLATILE_REF_COST);
8167 format %{ "ldars $dst, $mem\t# float" %}
8168
8169 ins_encode( aarch64_enc_fldars(dst, mem) );
8170
8171 ins_pipe(pipe_serial);
8172 %}
8173
8174 // Load Double
8175 instruct loadD_volatile(vRegD dst, /* sync_memory*/indirect mem)
8176 %{
8177 match(Set dst (LoadD mem));
8178
8179 ins_cost(VOLATILE_REF_COST);
8180 format %{ "ldard $dst, $mem\t# double" %}
8181
8182 ins_encode( aarch64_enc_fldard(dst, mem) );
8183
8184 ins_pipe(pipe_serial);
8185 %}
8186
8187 // Store Byte
8188 instruct storeB_volatile(iRegIorL2I src, /* sync_memory*/indirect mem)
8189 %{
8190 match(Set mem (StoreB mem src));
8191
8192 ins_cost(VOLATILE_REF_COST);
8193 format %{ "stlrb $src, $mem\t# byte" %}
8194
8195 ins_encode(aarch64_enc_stlrb(src, mem));
8196
8197 ins_pipe(pipe_class_memory);
8198 %}
8199
8200 instruct storeimmB0_volatile(immI0 zero, /* sync_memory*/indirect mem)
8201 %{
8202 match(Set mem (StoreB mem zero));
8203
8204 ins_cost(VOLATILE_REF_COST);
8205 format %{ "stlrb zr, $mem\t# byte" %}
8206
8207 ins_encode(aarch64_enc_stlrb0(mem));
8208
8209 ins_pipe(pipe_class_memory);
8210 %}
8211
8212 // Store Char/Short
8213 instruct storeC_volatile(iRegIorL2I src, /* sync_memory*/indirect mem)
8214 %{
8215 match(Set mem (StoreC mem src));
8216
8217 ins_cost(VOLATILE_REF_COST);
8218 format %{ "stlrh $src, $mem\t# short" %}
8219
8220 ins_encode(aarch64_enc_stlrh(src, mem));
8221
8222 ins_pipe(pipe_class_memory);
8223 %}
8224
8225 instruct storeimmC0_volatile(immI0 zero, /* sync_memory*/indirect mem)
8226 %{
8227 match(Set mem (StoreC mem zero));
8228
8229 ins_cost(VOLATILE_REF_COST);
8230 format %{ "stlrh zr, $mem\t# short" %}
8231
8232 ins_encode(aarch64_enc_stlrh0(mem));
8233
8234 ins_pipe(pipe_class_memory);
8235 %}
8236
8237 // Store Integer
8238
8239 instruct storeI_volatile(iRegIorL2I src, /* sync_memory*/indirect mem)
8240 %{
8241 match(Set mem(StoreI mem src));
8242
8243 ins_cost(VOLATILE_REF_COST);
8244 format %{ "stlrw $src, $mem\t# int" %}
8245
8246 ins_encode(aarch64_enc_stlrw(src, mem));
8247
8248 ins_pipe(pipe_class_memory);
8249 %}
8250
8251 instruct storeimmI0_volatile(immI0 zero, /* sync_memory*/indirect mem)
8252 %{
8253 match(Set mem(StoreI mem zero));
8254
8255 ins_cost(VOLATILE_REF_COST);
8256 format %{ "stlrw zr, $mem\t# int" %}
8257
8258 ins_encode(aarch64_enc_stlrw0(mem));
8259
8260 ins_pipe(pipe_class_memory);
8261 %}
8262
8263 // Store Long (64 bit signed)
8264 instruct storeL_volatile(iRegL src, /* sync_memory*/indirect mem)
8265 %{
8266 match(Set mem (StoreL mem src));
8267
8268 ins_cost(VOLATILE_REF_COST);
8269 format %{ "stlr $src, $mem\t# int" %}
8270
8271 ins_encode(aarch64_enc_stlr(src, mem));
8272
8273 ins_pipe(pipe_class_memory);
8274 %}
8275
8276 instruct storeimmL0_volatile(immL0 zero, /* sync_memory*/indirect mem)
8277 %{
8278 match(Set mem (StoreL mem zero));
8279
8280 ins_cost(VOLATILE_REF_COST);
8281 format %{ "stlr zr, $mem\t# int" %}
8282
8283 ins_encode(aarch64_enc_stlr0(mem));
8284
8285 ins_pipe(pipe_class_memory);
8286 %}
8287
8288 // Store Pointer
8289 instruct storeP_volatile(iRegP src, /* sync_memory*/indirect mem)
8290 %{
8291 match(Set mem (StoreP mem src));
8292
8293 ins_cost(VOLATILE_REF_COST);
8294 format %{ "stlr $src, $mem\t# ptr" %}
8295
8296 ins_encode(aarch64_enc_stlr(src, mem));
8297
8298 ins_pipe(pipe_class_memory);
8299 %}
8300
8301 instruct storeimmP0_volatile(immP0 zero, /* sync_memory*/indirect mem)
8302 %{
8303 match(Set mem (StoreP mem zero));
8304
8305 ins_cost(VOLATILE_REF_COST);
8306 format %{ "stlr zr, $mem\t# ptr" %}
8307
8308 ins_encode(aarch64_enc_stlr0(mem));
8309
8310 ins_pipe(pipe_class_memory);
8311 %}
8312
8313 // Store Compressed Pointer
8314 instruct storeN_volatile(iRegN src, /* sync_memory*/indirect mem)
8315 %{
8316 match(Set mem (StoreN mem src));
8317
8318 ins_cost(VOLATILE_REF_COST);
8319 format %{ "stlrw $src, $mem\t# compressed ptr" %}
8320
8321 ins_encode(aarch64_enc_stlrw(src, mem));
8322
8323 ins_pipe(pipe_class_memory);
8324 %}
8325
8326 instruct storeimmN0_volatile(immN0 zero, /* sync_memory*/indirect mem)
8327 %{
8328 match(Set mem (StoreN mem zero));
8329
8330 ins_cost(VOLATILE_REF_COST);
8331 format %{ "stlrw zr, $mem\t# compressed ptr" %}
8332
8333 ins_encode(aarch64_enc_stlrw0(mem));
8334
8335 ins_pipe(pipe_class_memory);
8336 %}
8337
8338 // Store Float
8339 instruct storeF_volatile(vRegF src, /* sync_memory*/indirect mem)
8340 %{
8341 match(Set mem (StoreF mem src));
8342
8343 ins_cost(VOLATILE_REF_COST);
8344 format %{ "stlrs $src, $mem\t# float" %}
8345
8346 ins_encode( aarch64_enc_fstlrs(src, mem) );
8347
8348 ins_pipe(pipe_class_memory);
8349 %}
8350
8351 // TODO
8352 // implement storeImmF0 and storeFImmPacked
8353
8354 // Store Double
8355 instruct storeD_volatile(vRegD src, /* sync_memory*/indirect mem)
8356 %{
8357 match(Set mem (StoreD mem src));
8358
8359 ins_cost(VOLATILE_REF_COST);
8360 format %{ "stlrd $src, $mem\t# double" %}
8361
8362 ins_encode( aarch64_enc_fstlrd(src, mem) );
8363
8364 ins_pipe(pipe_class_memory);
8365 %}
8366
8367 // ---------------- end of volatile loads and stores ----------------
8368
8369 instruct cacheWB(indirect addr)
8370 %{
8371 predicate(VM_Version::supports_data_cache_line_flush());
8372 match(CacheWB addr);
8373
8374 ins_cost(100);
8375 format %{"cache wb $addr" %}
8376 ins_encode %{
8377 assert($addr->index_position() < 0, "should be");
8378 assert($addr$$disp == 0, "should be");
8379 __ cache_wb(Address($addr$$base$$Register, 0));
8380 %}
8381 ins_pipe(pipe_slow); // XXX
8382 %}
8383
8384 instruct cacheWBPreSync()
8385 %{
8386 predicate(VM_Version::supports_data_cache_line_flush());
8387 match(CacheWBPreSync);
8388
8389 ins_cost(100);
8390 format %{"cache wb presync" %}
8391 ins_encode %{
8392 __ cache_wbsync(true);
8393 %}
8394 ins_pipe(pipe_slow); // XXX
8395 %}
8396
8397 instruct cacheWBPostSync()
8398 %{
8399 predicate(VM_Version::supports_data_cache_line_flush());
8400 match(CacheWBPostSync);
8401
8402 ins_cost(100);
8403 format %{"cache wb postsync" %}
8404 ins_encode %{
8405 __ cache_wbsync(false);
8406 %}
8407 ins_pipe(pipe_slow); // XXX
8408 %}
8409
8410 // ============================================================================
8411 // BSWAP Instructions
8412
8413 instruct bytes_reverse_int(iRegINoSp dst, iRegIorL2I src) %{
8414 match(Set dst (ReverseBytesI src));
8415
8416 ins_cost(INSN_COST);
8417 format %{ "revw $dst, $src" %}
8418
8419 ins_encode %{
8420 __ revw(as_Register($dst$$reg), as_Register($src$$reg));
8421 %}
8422
8423 ins_pipe(ialu_reg);
8424 %}
8425
8426 instruct bytes_reverse_long(iRegLNoSp dst, iRegL src) %{
8427 match(Set dst (ReverseBytesL src));
8428
8429 ins_cost(INSN_COST);
8430 format %{ "rev $dst, $src" %}
8431
8432 ins_encode %{
8433 __ rev(as_Register($dst$$reg), as_Register($src$$reg));
8434 %}
8435
8436 ins_pipe(ialu_reg);
8437 %}
8438
8439 instruct bytes_reverse_unsigned_short(iRegINoSp dst, iRegIorL2I src) %{
8440 match(Set dst (ReverseBytesUS src));
8441
8442 ins_cost(INSN_COST);
8443 format %{ "rev16w $dst, $src" %}
8444
8445 ins_encode %{
8446 __ rev16w(as_Register($dst$$reg), as_Register($src$$reg));
8447 %}
8448
8449 ins_pipe(ialu_reg);
8450 %}
8451
8452 instruct bytes_reverse_short(iRegINoSp dst, iRegIorL2I src) %{
8453 match(Set dst (ReverseBytesS src));
8454
8455 ins_cost(INSN_COST);
8456 format %{ "rev16w $dst, $src\n\t"
8457 "sbfmw $dst, $dst, #0, #15" %}
8458
8459 ins_encode %{
8460 __ rev16w(as_Register($dst$$reg), as_Register($src$$reg));
8461 __ sbfmw(as_Register($dst$$reg), as_Register($dst$$reg), 0U, 15U);
8462 %}
8463
8464 ins_pipe(ialu_reg);
8465 %}
8466
8467 // ============================================================================
8468 // Zero Count Instructions
8469
8470 instruct countLeadingZerosI(iRegINoSp dst, iRegIorL2I src) %{
8471 match(Set dst (CountLeadingZerosI src));
8472
8473 ins_cost(INSN_COST);
8474 format %{ "clzw $dst, $src" %}
8475 ins_encode %{
8476 __ clzw(as_Register($dst$$reg), as_Register($src$$reg));
8477 %}
8478
8479 ins_pipe(ialu_reg);
8480 %}
8481
8482 instruct countLeadingZerosL(iRegINoSp dst, iRegL src) %{
8483 match(Set dst (CountLeadingZerosL src));
8484
8485 ins_cost(INSN_COST);
8486 format %{ "clz $dst, $src" %}
8487 ins_encode %{
8488 __ clz(as_Register($dst$$reg), as_Register($src$$reg));
8489 %}
8490
8491 ins_pipe(ialu_reg);
8492 %}
8493
8494 instruct countTrailingZerosI(iRegINoSp dst, iRegIorL2I src) %{
8495 match(Set dst (CountTrailingZerosI src));
8496
8497 ins_cost(INSN_COST * 2);
8498 format %{ "rbitw $dst, $src\n\t"
8499 "clzw $dst, $dst" %}
8500 ins_encode %{
8501 __ rbitw(as_Register($dst$$reg), as_Register($src$$reg));
8502 __ clzw(as_Register($dst$$reg), as_Register($dst$$reg));
8503 %}
8504
8505 ins_pipe(ialu_reg);
8506 %}
8507
8508 instruct countTrailingZerosL(iRegINoSp dst, iRegL src) %{
8509 match(Set dst (CountTrailingZerosL src));
8510
8511 ins_cost(INSN_COST * 2);
8512 format %{ "rbit $dst, $src\n\t"
8513 "clz $dst, $dst" %}
8514 ins_encode %{
8515 __ rbit(as_Register($dst$$reg), as_Register($src$$reg));
8516 __ clz(as_Register($dst$$reg), as_Register($dst$$reg));
8517 %}
8518
8519 ins_pipe(ialu_reg);
8520 %}
8521
8522 //---------- Population Count Instructions -------------------------------------
8523 //
8524
8525 instruct popCountI(iRegINoSp dst, iRegIorL2I src, vRegF tmp) %{
8526 predicate(UsePopCountInstruction);
8527 match(Set dst (PopCountI src));
8528 effect(TEMP tmp);
8529 ins_cost(INSN_COST * 13);
8530
8531 format %{ "movw $src, $src\n\t"
8532 "mov $tmp, $src\t# vector (1D)\n\t"
8533 "cnt $tmp, $tmp\t# vector (8B)\n\t"
8534 "addv $tmp, $tmp\t# vector (8B)\n\t"
8535 "mov $dst, $tmp\t# vector (1D)" %}
8536 ins_encode %{
8537 __ movw($src$$Register, $src$$Register); // ensure top 32 bits 0
8538 __ mov($tmp$$FloatRegister, __ T1D, 0, $src$$Register);
8539 __ cnt($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
8540 __ addv($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
8541 __ mov($dst$$Register, $tmp$$FloatRegister, __ T1D, 0);
8542 %}
8543
8544 ins_pipe(pipe_class_default);
8545 %}
8546
8547 instruct popCountI_mem(iRegINoSp dst, memory4 mem, vRegF tmp) %{
8548 predicate(UsePopCountInstruction);
8549 match(Set dst (PopCountI (LoadI mem)));
8550 effect(TEMP tmp);
8551 ins_cost(INSN_COST * 13);
8552
8553 format %{ "ldrs $tmp, $mem\n\t"
8554 "cnt $tmp, $tmp\t# vector (8B)\n\t"
8555 "addv $tmp, $tmp\t# vector (8B)\n\t"
8556 "mov $dst, $tmp\t# vector (1D)" %}
8557 ins_encode %{
8558 FloatRegister tmp_reg = as_FloatRegister($tmp$$reg);
8559 loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrs, tmp_reg, $mem->opcode(),
8560 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
8561 __ cnt($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
8562 __ addv($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
8563 __ mov($dst$$Register, $tmp$$FloatRegister, __ T1D, 0);
8564 %}
8565
8566 ins_pipe(pipe_class_default);
8567 %}
8568
8569 // Note: Long.bitCount(long) returns an int.
8570 instruct popCountL(iRegINoSp dst, iRegL src, vRegD tmp) %{
8571 predicate(UsePopCountInstruction);
8572 match(Set dst (PopCountL src));
8573 effect(TEMP tmp);
8574 ins_cost(INSN_COST * 13);
8575
8576 format %{ "mov $tmp, $src\t# vector (1D)\n\t"
8577 "cnt $tmp, $tmp\t# vector (8B)\n\t"
8578 "addv $tmp, $tmp\t# vector (8B)\n\t"
8579 "mov $dst, $tmp\t# vector (1D)" %}
8580 ins_encode %{
8581 __ mov($tmp$$FloatRegister, __ T1D, 0, $src$$Register);
8582 __ cnt($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
8583 __ addv($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
8584 __ mov($dst$$Register, $tmp$$FloatRegister, __ T1D, 0);
8585 %}
8586
8587 ins_pipe(pipe_class_default);
8588 %}
8589
8590 instruct popCountL_mem(iRegINoSp dst, memory8 mem, vRegD tmp) %{
8591 predicate(UsePopCountInstruction);
8592 match(Set dst (PopCountL (LoadL mem)));
8593 effect(TEMP tmp);
8594 ins_cost(INSN_COST * 13);
8595
8596 format %{ "ldrd $tmp, $mem\n\t"
8597 "cnt $tmp, $tmp\t# vector (8B)\n\t"
8598 "addv $tmp, $tmp\t# vector (8B)\n\t"
8599 "mov $dst, $tmp\t# vector (1D)" %}
8600 ins_encode %{
8601 FloatRegister tmp_reg = as_FloatRegister($tmp$$reg);
8602 loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrd, tmp_reg, $mem->opcode(),
8603 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
8604 __ cnt($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
8605 __ addv($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
8606 __ mov($dst$$Register, $tmp$$FloatRegister, __ T1D, 0);
8607 %}
8608
8609 ins_pipe(pipe_class_default);
8610 %}
8611
8612 // ============================================================================
8613 // MemBar Instruction
8614
8615 instruct load_fence() %{
8616 match(LoadFence);
8617 ins_cost(VOLATILE_REF_COST);
8618
8619 format %{ "load_fence" %}
8620
8621 ins_encode %{
8622 __ membar(Assembler::LoadLoad|Assembler::LoadStore);
8623 %}
8624 ins_pipe(pipe_serial);
8625 %}
8626
8627 instruct unnecessary_membar_acquire() %{
8628 predicate(unnecessary_acquire(n));
8629 match(MemBarAcquire);
8630 ins_cost(0);
8631
8632 format %{ "membar_acquire (elided)" %}
8633
8634 ins_encode %{
8635 __ block_comment("membar_acquire (elided)");
8636 %}
8637
8638 ins_pipe(pipe_class_empty);
8639 %}
8640
8641 instruct membar_acquire() %{
8642 match(MemBarAcquire);
8643 ins_cost(VOLATILE_REF_COST);
8644
8645 format %{ "membar_acquire\n\t"
8646 "dmb ish" %}
8647
8648 ins_encode %{
8649 __ block_comment("membar_acquire");
8650 __ membar(Assembler::LoadLoad|Assembler::LoadStore);
8651 %}
8652
8653 ins_pipe(pipe_serial);
8654 %}
8655
8656
8657 instruct membar_acquire_lock() %{
8658 match(MemBarAcquireLock);
8659 ins_cost(VOLATILE_REF_COST);
8660
8661 format %{ "membar_acquire_lock (elided)" %}
8662
8663 ins_encode %{
8664 __ block_comment("membar_acquire_lock (elided)");
8665 %}
8666
8667 ins_pipe(pipe_serial);
8668 %}
8669
8670 instruct store_fence() %{
8671 match(StoreFence);
8672 ins_cost(VOLATILE_REF_COST);
8673
8674 format %{ "store_fence" %}
8675
8676 ins_encode %{
8677 __ membar(Assembler::LoadStore|Assembler::StoreStore);
8678 %}
8679 ins_pipe(pipe_serial);
8680 %}
8681
8682 instruct unnecessary_membar_release() %{
8683 predicate(unnecessary_release(n));
8684 match(MemBarRelease);
8685 ins_cost(0);
8686
8687 format %{ "membar_release (elided)" %}
8688
8689 ins_encode %{
8690 __ block_comment("membar_release (elided)");
8691 %}
8692 ins_pipe(pipe_serial);
8693 %}
8694
8695 instruct membar_release() %{
8696 match(MemBarRelease);
8697 ins_cost(VOLATILE_REF_COST);
8698
8699 format %{ "membar_release\n\t"
8700 "dmb ish" %}
8701
8702 ins_encode %{
8703 __ block_comment("membar_release");
8704 __ membar(Assembler::LoadStore|Assembler::StoreStore);
8705 %}
8706 ins_pipe(pipe_serial);
8707 %}
8708
8709 instruct membar_storestore() %{
8710 match(MemBarStoreStore);
8711 match(StoreStoreFence);
8712 ins_cost(VOLATILE_REF_COST);
8713
8714 format %{ "MEMBAR-store-store" %}
8715
8716 ins_encode %{
8717 __ membar(Assembler::StoreStore);
8718 %}
8719 ins_pipe(pipe_serial);
8720 %}
8721
8722 instruct membar_release_lock() %{
8723 match(MemBarReleaseLock);
8724 ins_cost(VOLATILE_REF_COST);
8725
8726 format %{ "membar_release_lock (elided)" %}
8727
8728 ins_encode %{
8729 __ block_comment("membar_release_lock (elided)");
8730 %}
8731
8732 ins_pipe(pipe_serial);
8733 %}
8734
8735 instruct unnecessary_membar_volatile() %{
8736 predicate(unnecessary_volatile(n));
8737 match(MemBarVolatile);
8738 ins_cost(0);
8739
8740 format %{ "membar_volatile (elided)" %}
8741
8742 ins_encode %{
8743 __ block_comment("membar_volatile (elided)");
8744 %}
8745
8746 ins_pipe(pipe_serial);
8747 %}
8748
8749 instruct membar_volatile() %{
8750 match(MemBarVolatile);
8751 ins_cost(VOLATILE_REF_COST*100);
8752
8753 format %{ "membar_volatile\n\t"
8754 "dmb ish"%}
8755
8756 ins_encode %{
8757 __ block_comment("membar_volatile");
8758 __ membar(Assembler::StoreLoad);
8759 %}
8760
8761 ins_pipe(pipe_serial);
8762 %}
8763
8764 // ============================================================================
8765 // Cast/Convert Instructions
8766
8767 instruct castX2P(iRegPNoSp dst, iRegL src) %{
8768 match(Set dst (CastX2P src));
8769
8770 ins_cost(INSN_COST);
8771 format %{ "mov $dst, $src\t# long -> ptr" %}
8772
8773 ins_encode %{
8774 if ($dst$$reg != $src$$reg) {
8775 __ mov(as_Register($dst$$reg), as_Register($src$$reg));
8776 }
8777 %}
8778
8779 ins_pipe(ialu_reg);
8780 %}
8781
8782 instruct castP2X(iRegLNoSp dst, iRegP src) %{
8783 match(Set dst (CastP2X src));
8784
8785 ins_cost(INSN_COST);
8786 format %{ "mov $dst, $src\t# ptr -> long" %}
8787
8788 ins_encode %{
8789 if ($dst$$reg != $src$$reg) {
8790 __ mov(as_Register($dst$$reg), as_Register($src$$reg));
8791 }
8792 %}
8793
8794 ins_pipe(ialu_reg);
8795 %}
8796
8797 // Convert oop into int for vectors alignment masking
8798 instruct convP2I(iRegINoSp dst, iRegP src) %{
8799 match(Set dst (ConvL2I (CastP2X src)));
8800
8801 ins_cost(INSN_COST);
8802 format %{ "movw $dst, $src\t# ptr -> int" %}
8803 ins_encode %{
8804 __ movw($dst$$Register, $src$$Register);
8805 %}
8806
8807 ins_pipe(ialu_reg);
8808 %}
8809
8810 // Convert compressed oop into int for vectors alignment masking
8811 // in case of 32bit oops (heap < 4Gb).
8812 instruct convN2I(iRegINoSp dst, iRegN src)
8813 %{
8814 predicate(CompressedOops::shift() == 0);
8815 match(Set dst (ConvL2I (CastP2X (DecodeN src))));
8816
8817 ins_cost(INSN_COST);
8818 format %{ "mov dst, $src\t# compressed ptr -> int" %}
8819 ins_encode %{
8820 __ movw($dst$$Register, $src$$Register);
8821 %}
8822
8823 ins_pipe(ialu_reg);
8824 %}
8825
8826
8827 // Convert oop pointer into compressed form
8828 instruct encodeHeapOop(iRegNNoSp dst, iRegP src, rFlagsReg cr) %{
8829 predicate(n->bottom_type()->make_ptr()->ptr() != TypePtr::NotNull);
8830 match(Set dst (EncodeP src));
8831 effect(KILL cr);
8832 ins_cost(INSN_COST * 3);
8833 format %{ "encode_heap_oop $dst, $src" %}
8834 ins_encode %{
8835 Register s = $src$$Register;
8836 Register d = $dst$$Register;
8837 __ encode_heap_oop(d, s);
8838 %}
8839 ins_pipe(ialu_reg);
8840 %}
8841
8842 instruct encodeHeapOop_not_null(iRegNNoSp dst, iRegP src, rFlagsReg cr) %{
8843 predicate(n->bottom_type()->make_ptr()->ptr() == TypePtr::NotNull);
8844 match(Set dst (EncodeP src));
8845 ins_cost(INSN_COST * 3);
8846 format %{ "encode_heap_oop_not_null $dst, $src" %}
8847 ins_encode %{
8848 __ encode_heap_oop_not_null($dst$$Register, $src$$Register);
8849 %}
8850 ins_pipe(ialu_reg);
8851 %}
8852
8853 instruct decodeHeapOop(iRegPNoSp dst, iRegN src, rFlagsReg cr) %{
8854 predicate(n->bottom_type()->is_ptr()->ptr() != TypePtr::NotNull &&
8855 n->bottom_type()->is_ptr()->ptr() != TypePtr::Constant);
8856 match(Set dst (DecodeN src));
8857 ins_cost(INSN_COST * 3);
8858 format %{ "decode_heap_oop $dst, $src" %}
8859 ins_encode %{
8860 Register s = $src$$Register;
8861 Register d = $dst$$Register;
8862 __ decode_heap_oop(d, s);
8863 %}
8864 ins_pipe(ialu_reg);
8865 %}
8866
8867 instruct decodeHeapOop_not_null(iRegPNoSp dst, iRegN src, rFlagsReg cr) %{
8868 predicate(n->bottom_type()->is_ptr()->ptr() == TypePtr::NotNull ||
8869 n->bottom_type()->is_ptr()->ptr() == TypePtr::Constant);
8870 match(Set dst (DecodeN src));
8871 ins_cost(INSN_COST * 3);
8872 format %{ "decode_heap_oop_not_null $dst, $src" %}
8873 ins_encode %{
8874 Register s = $src$$Register;
8875 Register d = $dst$$Register;
8876 __ decode_heap_oop_not_null(d, s);
8877 %}
8878 ins_pipe(ialu_reg);
8879 %}
8880
8881 // n.b. AArch64 implementations of encode_klass_not_null and
8882 // decode_klass_not_null do not modify the flags register so, unlike
8883 // Intel, we don't kill CR as a side effect here
8884
8885 instruct encodeKlass_not_null(iRegNNoSp dst, iRegP src) %{
8886 match(Set dst (EncodePKlass src));
8887
8888 ins_cost(INSN_COST * 3);
8889 format %{ "encode_klass_not_null $dst,$src" %}
8890
8891 ins_encode %{
8892 Register src_reg = as_Register($src$$reg);
8893 Register dst_reg = as_Register($dst$$reg);
8894 __ encode_klass_not_null(dst_reg, src_reg);
8895 %}
8896
8897 ins_pipe(ialu_reg);
8898 %}
8899
8900 instruct decodeKlass_not_null(iRegPNoSp dst, iRegN src) %{
8901 match(Set dst (DecodeNKlass src));
8902
8903 ins_cost(INSN_COST * 3);
8904 format %{ "decode_klass_not_null $dst,$src" %}
8905
8906 ins_encode %{
8907 Register src_reg = as_Register($src$$reg);
8908 Register dst_reg = as_Register($dst$$reg);
8909 if (dst_reg != src_reg) {
8910 __ decode_klass_not_null(dst_reg, src_reg);
8911 } else {
8912 __ decode_klass_not_null(dst_reg);
8913 }
8914 %}
8915
8916 ins_pipe(ialu_reg);
8917 %}
8918
8919 instruct checkCastPP(iRegPNoSp dst)
8920 %{
8921 match(Set dst (CheckCastPP dst));
8922
8923 size(0);
8924 format %{ "# checkcastPP of $dst" %}
8925 ins_encode(/* empty encoding */);
8926 ins_pipe(pipe_class_empty);
8927 %}
8928
8929 instruct castPP(iRegPNoSp dst)
8930 %{
8931 match(Set dst (CastPP dst));
8932
8933 size(0);
8934 format %{ "# castPP of $dst" %}
8935 ins_encode(/* empty encoding */);
8936 ins_pipe(pipe_class_empty);
8937 %}
8938
8939 instruct castII(iRegI dst)
8940 %{
8941 match(Set dst (CastII dst));
8942
8943 size(0);
8944 format %{ "# castII of $dst" %}
8945 ins_encode(/* empty encoding */);
8946 ins_cost(0);
8947 ins_pipe(pipe_class_empty);
8948 %}
8949
8950 instruct castLL(iRegL dst)
8951 %{
8952 match(Set dst (CastLL dst));
8953
8954 size(0);
8955 format %{ "# castLL of $dst" %}
8956 ins_encode(/* empty encoding */);
8957 ins_cost(0);
8958 ins_pipe(pipe_class_empty);
8959 %}
8960
8961 instruct castFF(vRegF dst)
8962 %{
8963 match(Set dst (CastFF dst));
8964
8965 size(0);
8966 format %{ "# castFF of $dst" %}
8967 ins_encode(/* empty encoding */);
8968 ins_cost(0);
8969 ins_pipe(pipe_class_empty);
8970 %}
8971
8972 instruct castDD(vRegD dst)
8973 %{
8974 match(Set dst (CastDD dst));
8975
8976 size(0);
8977 format %{ "# castDD of $dst" %}
8978 ins_encode(/* empty encoding */);
8979 ins_cost(0);
8980 ins_pipe(pipe_class_empty);
8981 %}
8982
8983 instruct castVVD(vecD dst)
8984 %{
8985 match(Set dst (CastVV dst));
8986
8987 size(0);
8988 format %{ "# castVV of $dst" %}
8989 ins_encode(/* empty encoding */);
8990 ins_cost(0);
8991 ins_pipe(pipe_class_empty);
8992 %}
8993
8994 instruct castVVX(vecX dst)
8995 %{
8996 match(Set dst (CastVV dst));
8997
8998 size(0);
8999 format %{ "# castVV of $dst" %}
9000 ins_encode(/* empty encoding */);
9001 ins_cost(0);
9002 ins_pipe(pipe_class_empty);
9003 %}
9004
9005 instruct castVV(vReg dst)
9006 %{
9007 match(Set dst (CastVV dst));
9008
9009 size(0);
9010 format %{ "# castVV of $dst" %}
9011 ins_encode(/* empty encoding */);
9012 ins_cost(0);
9013 ins_pipe(pipe_class_empty);
9014 %}
9015
9016 // ============================================================================
9017 // Atomic operation instructions
9018 //
9019 // Intel and SPARC both implement Ideal Node LoadPLocked and
9020 // Store{PIL}Conditional instructions using a normal load for the
9021 // LoadPLocked and a CAS for the Store{PIL}Conditional.
9022 //
9023 // The ideal code appears only to use LoadPLocked/StorePLocked as a
9024 // pair to lock object allocations from Eden space when not using
9025 // TLABs.
9026 //
9027 // There does not appear to be a Load{IL}Locked Ideal Node and the
9028 // Ideal code appears to use Store{IL}Conditional as an alias for CAS
9029 // and to use StoreIConditional only for 32-bit and StoreLConditional
9030 // only for 64-bit.
9031 //
9032 // We implement LoadPLocked and StorePLocked instructions using,
9033 // respectively the AArch64 hw load-exclusive and store-conditional
9034 // instructions. Whereas we must implement each of
9035 // Store{IL}Conditional using a CAS which employs a pair of
9036 // instructions comprising a load-exclusive followed by a
9037 // store-conditional.
9038
9039
9040 // Locked-load (linked load) of the current heap-top
9041 // used when updating the eden heap top
9042 // implemented using ldaxr on AArch64
9043
9044 instruct loadPLocked(iRegPNoSp dst, indirect mem)
9045 %{
9046 match(Set dst (LoadPLocked mem));
9047
9048 ins_cost(VOLATILE_REF_COST);
9049
9050 format %{ "ldaxr $dst, $mem\t# ptr linked acquire" %}
9051
9052 ins_encode(aarch64_enc_ldaxr(dst, mem));
9053
9054 ins_pipe(pipe_serial);
9055 %}
9056
9057 // Conditional-store of the updated heap-top.
9058 // Used during allocation of the shared heap.
9059 // Sets flag (EQ) on success.
9060 // implemented using stlxr on AArch64.
9061
9062 instruct storePConditional(memory8 heap_top_ptr, iRegP oldval, iRegP newval, rFlagsReg cr)
9063 %{
9064 match(Set cr (StorePConditional heap_top_ptr (Binary oldval newval)));
9065
9066 ins_cost(VOLATILE_REF_COST);
9067
9068 // TODO
9069 // do we need to do a store-conditional release or can we just use a
9070 // plain store-conditional?
9071
9072 format %{
9073 "stlxr rscratch1, $newval, $heap_top_ptr\t# ptr cond release"
9074 "cmpw rscratch1, zr\t# EQ on successful write"
9075 %}
9076
9077 ins_encode(aarch64_enc_stlxr(newval, heap_top_ptr));
9078
9079 ins_pipe(pipe_serial);
9080 %}
9081
9082
9083 // storeLConditional is used by PhaseMacroExpand::expand_lock_node
9084 // when attempting to rebias a lock towards the current thread. We
9085 // must use the acquire form of cmpxchg in order to guarantee acquire
9086 // semantics in this case.
9087 instruct storeLConditional(indirect mem, iRegLNoSp oldval, iRegLNoSp newval, rFlagsReg cr)
9088 %{
9089 match(Set cr (StoreLConditional mem (Binary oldval newval)));
9090
9091 ins_cost(VOLATILE_REF_COST);
9092
9093 format %{
9094 "cmpxchg rscratch1, $mem, $oldval, $newval, $mem\t# if $mem == $oldval then $mem <-- $newval"
9095 "cmpw rscratch1, zr\t# EQ on successful write"
9096 %}
9097
9098 ins_encode(aarch64_enc_cmpxchg_acq(mem, oldval, newval));
9099
9100 ins_pipe(pipe_slow);
9101 %}
9102
9103 // storeIConditional also has acquire semantics, for no better reason
9104 // than matching storeLConditional. At the time of writing this
9105 // comment storeIConditional was not used anywhere by AArch64.
9106 instruct storeIConditional(indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr)
9107 %{
9108 match(Set cr (StoreIConditional mem (Binary oldval newval)));
9109
9110 ins_cost(VOLATILE_REF_COST);
9111
9112 format %{
9113 "cmpxchgw rscratch1, $mem, $oldval, $newval, $mem\t# if $mem == $oldval then $mem <-- $newval"
9114 "cmpw rscratch1, zr\t# EQ on successful write"
9115 %}
9116
9117 ins_encode(aarch64_enc_cmpxchgw_acq(mem, oldval, newval));
9118
9119 ins_pipe(pipe_slow);
9120 %}
9121
9122 // standard CompareAndSwapX when we are using barriers
9123 // these have higher priority than the rules selected by a predicate
9124
9125 // XXX No flag versions for CompareAndSwap{I,L,P,N} because matcher
9126 // can't match them
9127
9128 instruct compareAndSwapB(iRegINoSp res, indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr) %{
9129
9130 match(Set res (CompareAndSwapB mem (Binary oldval newval)));
9131 ins_cost(2 * VOLATILE_REF_COST);
9132
9133 effect(KILL cr);
9134
9135 format %{
9136 "cmpxchgb $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval"
9137 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
9138 %}
9139
9140 ins_encode(aarch64_enc_cmpxchgb(mem, oldval, newval),
9141 aarch64_enc_cset_eq(res));
9142
9143 ins_pipe(pipe_slow);
9144 %}
9145
9146 instruct compareAndSwapS(iRegINoSp res, indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr) %{
9147
9148 match(Set res (CompareAndSwapS mem (Binary oldval newval)));
9149 ins_cost(2 * VOLATILE_REF_COST);
9150
9151 effect(KILL cr);
9152
9153 format %{
9154 "cmpxchgs $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval"
9155 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
9156 %}
9157
9158 ins_encode(aarch64_enc_cmpxchgs(mem, oldval, newval),
9159 aarch64_enc_cset_eq(res));
9160
9161 ins_pipe(pipe_slow);
9162 %}
9163
9164 instruct compareAndSwapI(iRegINoSp res, indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr) %{
9165
9166 match(Set res (CompareAndSwapI mem (Binary oldval newval)));
9167 ins_cost(2 * VOLATILE_REF_COST);
9168
9169 effect(KILL cr);
9170
9171 format %{
9172 "cmpxchgw $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval"
9173 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
9174 %}
9175
9176 ins_encode(aarch64_enc_cmpxchgw(mem, oldval, newval),
9177 aarch64_enc_cset_eq(res));
9178
9179 ins_pipe(pipe_slow);
9180 %}
9181
9182 instruct compareAndSwapL(iRegINoSp res, indirect mem, iRegLNoSp oldval, iRegLNoSp newval, rFlagsReg cr) %{
9183
9184 match(Set res (CompareAndSwapL mem (Binary oldval newval)));
9185 ins_cost(2 * VOLATILE_REF_COST);
9186
9187 effect(KILL cr);
9188
9189 format %{
9190 "cmpxchg $mem, $oldval, $newval\t# (long) if $mem == $oldval then $mem <-- $newval"
9191 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
9192 %}
9193
9194 ins_encode(aarch64_enc_cmpxchg(mem, oldval, newval),
9195 aarch64_enc_cset_eq(res));
9196
9197 ins_pipe(pipe_slow);
9198 %}
9199
9200 instruct compareAndSwapP(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
9201
9202 match(Set res (CompareAndSwapP mem (Binary oldval newval)));
9203 predicate(n->as_LoadStore()->barrier_data() == 0);
9204 ins_cost(2 * VOLATILE_REF_COST);
9205
9206 effect(KILL cr);
9207
9208 format %{
9209 "cmpxchg $mem, $oldval, $newval\t# (ptr) if $mem == $oldval then $mem <-- $newval"
9210 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
9211 %}
9212
9213 ins_encode(aarch64_enc_cmpxchg(mem, oldval, newval),
9214 aarch64_enc_cset_eq(res));
9215
9216 ins_pipe(pipe_slow);
9217 %}
9218
9219 instruct compareAndSwapN(iRegINoSp res, indirect mem, iRegNNoSp oldval, iRegNNoSp newval, rFlagsReg cr) %{
9220
9221 match(Set res (CompareAndSwapN mem (Binary oldval newval)));
9222 ins_cost(2 * VOLATILE_REF_COST);
9223
9224 effect(KILL cr);
9225
9226 format %{
9227 "cmpxchgw $mem, $oldval, $newval\t# (narrow oop) if $mem == $oldval then $mem <-- $newval"
9228 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
9229 %}
9230
9231 ins_encode(aarch64_enc_cmpxchgw(mem, oldval, newval),
9232 aarch64_enc_cset_eq(res));
9233
9234 ins_pipe(pipe_slow);
9235 %}
9236
9237 // alternative CompareAndSwapX when we are eliding barriers
9238
9239 instruct compareAndSwapBAcq(iRegINoSp res, indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr) %{
9240
9241 predicate(needs_acquiring_load_exclusive(n));
9242 match(Set res (CompareAndSwapB mem (Binary oldval newval)));
9243 ins_cost(VOLATILE_REF_COST);
9244
9245 effect(KILL cr);
9246
9247 format %{
9248 "cmpxchgb_acq $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval"
9249 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
9250 %}
9251
9252 ins_encode(aarch64_enc_cmpxchgb_acq(mem, oldval, newval),
9253 aarch64_enc_cset_eq(res));
9254
9255 ins_pipe(pipe_slow);
9256 %}
9257
9258 instruct compareAndSwapSAcq(iRegINoSp res, indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr) %{
9259
9260 predicate(needs_acquiring_load_exclusive(n));
9261 match(Set res (CompareAndSwapS mem (Binary oldval newval)));
9262 ins_cost(VOLATILE_REF_COST);
9263
9264 effect(KILL cr);
9265
9266 format %{
9267 "cmpxchgs_acq $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval"
9268 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
9269 %}
9270
9271 ins_encode(aarch64_enc_cmpxchgs_acq(mem, oldval, newval),
9272 aarch64_enc_cset_eq(res));
9273
9274 ins_pipe(pipe_slow);
9275 %}
9276
9277 instruct compareAndSwapIAcq(iRegINoSp res, indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr) %{
9278
9279 predicate(needs_acquiring_load_exclusive(n));
9280 match(Set res (CompareAndSwapI mem (Binary oldval newval)));
9281 ins_cost(VOLATILE_REF_COST);
9282
9283 effect(KILL cr);
9284
9285 format %{
9286 "cmpxchgw_acq $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval"
9287 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
9288 %}
9289
9290 ins_encode(aarch64_enc_cmpxchgw_acq(mem, oldval, newval),
9291 aarch64_enc_cset_eq(res));
9292
9293 ins_pipe(pipe_slow);
9294 %}
9295
9296 instruct compareAndSwapLAcq(iRegINoSp res, indirect mem, iRegLNoSp oldval, iRegLNoSp newval, rFlagsReg cr) %{
9297
9298 predicate(needs_acquiring_load_exclusive(n));
9299 match(Set res (CompareAndSwapL mem (Binary oldval newval)));
9300 ins_cost(VOLATILE_REF_COST);
9301
9302 effect(KILL cr);
9303
9304 format %{
9305 "cmpxchg_acq $mem, $oldval, $newval\t# (long) if $mem == $oldval then $mem <-- $newval"
9306 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
9307 %}
9308
9309 ins_encode(aarch64_enc_cmpxchg_acq(mem, oldval, newval),
9310 aarch64_enc_cset_eq(res));
9311
9312 ins_pipe(pipe_slow);
9313 %}
9314
9315 instruct compareAndSwapPAcq(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
9316
9317 predicate(needs_acquiring_load_exclusive(n) && (n->as_LoadStore()->barrier_data() == 0));
9318 match(Set res (CompareAndSwapP mem (Binary oldval newval)));
9319 ins_cost(VOLATILE_REF_COST);
9320
9321 effect(KILL cr);
9322
9323 format %{
9324 "cmpxchg_acq $mem, $oldval, $newval\t# (ptr) if $mem == $oldval then $mem <-- $newval"
9325 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
9326 %}
9327
9328 ins_encode(aarch64_enc_cmpxchg_acq(mem, oldval, newval),
9329 aarch64_enc_cset_eq(res));
9330
9331 ins_pipe(pipe_slow);
9332 %}
9333
9334 instruct compareAndSwapNAcq(iRegINoSp res, indirect mem, iRegNNoSp oldval, iRegNNoSp newval, rFlagsReg cr) %{
9335
9336 predicate(needs_acquiring_load_exclusive(n));
9337 match(Set res (CompareAndSwapN mem (Binary oldval newval)));
9338 ins_cost(VOLATILE_REF_COST);
9339
9340 effect(KILL cr);
9341
9342 format %{
9343 "cmpxchgw_acq $mem, $oldval, $newval\t# (narrow oop) if $mem == $oldval then $mem <-- $newval"
9344 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
9345 %}
9346
9347 ins_encode(aarch64_enc_cmpxchgw_acq(mem, oldval, newval),
9348 aarch64_enc_cset_eq(res));
9349
9350 ins_pipe(pipe_slow);
9351 %}
9352
9353
9354 // ---------------------------------------------------------------------
9355
9356 // BEGIN This section of the file is automatically generated. Do not edit --------------
9357
9358 // Sundry CAS operations. Note that release is always true,
9359 // regardless of the memory ordering of the CAS. This is because we
9360 // need the volatile case to be sequentially consistent but there is
9361 // no trailing StoreLoad barrier emitted by C2. Unfortunately we
9362 // can't check the type of memory ordering here, so we always emit a
9363 // STLXR.
9364
9365 // This section is generated from aarch64_ad_cas.m4
9366
9367
9368
9369 // This pattern is generated automatically from cas.m4.
9370 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9371 instruct compareAndExchangeB(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
9372
9373 match(Set res (CompareAndExchangeB mem (Binary oldval newval)));
9374 ins_cost(2 * VOLATILE_REF_COST);
9375 effect(TEMP_DEF res, KILL cr);
9376 format %{
9377 "cmpxchgb $res = $mem, $oldval, $newval\t# (byte, weak) if $mem == $oldval then $mem <-- $newval"
9378 %}
9379 ins_encode %{
9380 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
9381 Assembler::byte, /*acquire*/ false, /*release*/ true,
9382 /*weak*/ false, $res$$Register);
9383 __ sxtbw($res$$Register, $res$$Register);
9384 %}
9385 ins_pipe(pipe_slow);
9386 %}
9387
9388 // This pattern is generated automatically from cas.m4.
9389 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9390 instruct compareAndExchangeS(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
9391
9392 match(Set res (CompareAndExchangeS mem (Binary oldval newval)));
9393 ins_cost(2 * VOLATILE_REF_COST);
9394 effect(TEMP_DEF res, KILL cr);
9395 format %{
9396 "cmpxchgs $res = $mem, $oldval, $newval\t# (short, weak) if $mem == $oldval then $mem <-- $newval"
9397 %}
9398 ins_encode %{
9399 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
9400 Assembler::halfword, /*acquire*/ false, /*release*/ true,
9401 /*weak*/ false, $res$$Register);
9402 __ sxthw($res$$Register, $res$$Register);
9403 %}
9404 ins_pipe(pipe_slow);
9405 %}
9406
9407 // This pattern is generated automatically from cas.m4.
9408 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9409 instruct compareAndExchangeI(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
9410
9411 match(Set res (CompareAndExchangeI mem (Binary oldval newval)));
9412 ins_cost(2 * VOLATILE_REF_COST);
9413 effect(TEMP_DEF res, KILL cr);
9414 format %{
9415 "cmpxchgw $res = $mem, $oldval, $newval\t# (int, weak) if $mem == $oldval then $mem <-- $newval"
9416 %}
9417 ins_encode %{
9418 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
9419 Assembler::word, /*acquire*/ false, /*release*/ true,
9420 /*weak*/ false, $res$$Register);
9421 %}
9422 ins_pipe(pipe_slow);
9423 %}
9424
9425 // This pattern is generated automatically from cas.m4.
9426 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9427 instruct compareAndExchangeL(iRegLNoSp res, indirect mem, iRegL oldval, iRegL newval, rFlagsReg cr) %{
9428
9429 match(Set res (CompareAndExchangeL mem (Binary oldval newval)));
9430 ins_cost(2 * VOLATILE_REF_COST);
9431 effect(TEMP_DEF res, KILL cr);
9432 format %{
9433 "cmpxchg $res = $mem, $oldval, $newval\t# (long, weak) if $mem == $oldval then $mem <-- $newval"
9434 %}
9435 ins_encode %{
9436 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
9437 Assembler::xword, /*acquire*/ false, /*release*/ true,
9438 /*weak*/ false, $res$$Register);
9439 %}
9440 ins_pipe(pipe_slow);
9441 %}
9442
9443 // This pattern is generated automatically from cas.m4.
9444 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9445 instruct compareAndExchangeN(iRegNNoSp res, indirect mem, iRegN oldval, iRegN newval, rFlagsReg cr) %{
9446
9447 match(Set res (CompareAndExchangeN mem (Binary oldval newval)));
9448 ins_cost(2 * VOLATILE_REF_COST);
9449 effect(TEMP_DEF res, KILL cr);
9450 format %{
9451 "cmpxchgw $res = $mem, $oldval, $newval\t# (narrow oop, weak) if $mem == $oldval then $mem <-- $newval"
9452 %}
9453 ins_encode %{
9454 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
9455 Assembler::word, /*acquire*/ false, /*release*/ true,
9456 /*weak*/ false, $res$$Register);
9457 %}
9458 ins_pipe(pipe_slow);
9459 %}
9460
9461 // This pattern is generated automatically from cas.m4.
9462 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9463 instruct compareAndExchangeP(iRegPNoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
9464 predicate(n->as_LoadStore()->barrier_data() == 0);
9465 match(Set res (CompareAndExchangeP mem (Binary oldval newval)));
9466 ins_cost(2 * VOLATILE_REF_COST);
9467 effect(TEMP_DEF res, KILL cr);
9468 format %{
9469 "cmpxchg $res = $mem, $oldval, $newval\t# (ptr, weak) if $mem == $oldval then $mem <-- $newval"
9470 %}
9471 ins_encode %{
9472 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
9473 Assembler::xword, /*acquire*/ false, /*release*/ true,
9474 /*weak*/ false, $res$$Register);
9475 %}
9476 ins_pipe(pipe_slow);
9477 %}
9478
9479 // This pattern is generated automatically from cas.m4.
9480 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9481 instruct compareAndExchangeBAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
9482 predicate(needs_acquiring_load_exclusive(n));
9483 match(Set res (CompareAndExchangeB mem (Binary oldval newval)));
9484 ins_cost(VOLATILE_REF_COST);
9485 effect(TEMP_DEF res, KILL cr);
9486 format %{
9487 "cmpxchgb_acq $res = $mem, $oldval, $newval\t# (byte, weak) if $mem == $oldval then $mem <-- $newval"
9488 %}
9489 ins_encode %{
9490 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
9491 Assembler::byte, /*acquire*/ true, /*release*/ true,
9492 /*weak*/ false, $res$$Register);
9493 __ sxtbw($res$$Register, $res$$Register);
9494 %}
9495 ins_pipe(pipe_slow);
9496 %}
9497
9498 // This pattern is generated automatically from cas.m4.
9499 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9500 instruct compareAndExchangeSAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
9501 predicate(needs_acquiring_load_exclusive(n));
9502 match(Set res (CompareAndExchangeS mem (Binary oldval newval)));
9503 ins_cost(VOLATILE_REF_COST);
9504 effect(TEMP_DEF res, KILL cr);
9505 format %{
9506 "cmpxchgs_acq $res = $mem, $oldval, $newval\t# (short, weak) if $mem == $oldval then $mem <-- $newval"
9507 %}
9508 ins_encode %{
9509 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
9510 Assembler::halfword, /*acquire*/ true, /*release*/ true,
9511 /*weak*/ false, $res$$Register);
9512 __ sxthw($res$$Register, $res$$Register);
9513 %}
9514 ins_pipe(pipe_slow);
9515 %}
9516
9517 // This pattern is generated automatically from cas.m4.
9518 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9519 instruct compareAndExchangeIAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
9520 predicate(needs_acquiring_load_exclusive(n));
9521 match(Set res (CompareAndExchangeI mem (Binary oldval newval)));
9522 ins_cost(VOLATILE_REF_COST);
9523 effect(TEMP_DEF res, KILL cr);
9524 format %{
9525 "cmpxchgw_acq $res = $mem, $oldval, $newval\t# (int, weak) if $mem == $oldval then $mem <-- $newval"
9526 %}
9527 ins_encode %{
9528 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
9529 Assembler::word, /*acquire*/ true, /*release*/ true,
9530 /*weak*/ false, $res$$Register);
9531 %}
9532 ins_pipe(pipe_slow);
9533 %}
9534
9535 // This pattern is generated automatically from cas.m4.
9536 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9537 instruct compareAndExchangeLAcq(iRegLNoSp res, indirect mem, iRegL oldval, iRegL newval, rFlagsReg cr) %{
9538 predicate(needs_acquiring_load_exclusive(n));
9539 match(Set res (CompareAndExchangeL mem (Binary oldval newval)));
9540 ins_cost(VOLATILE_REF_COST);
9541 effect(TEMP_DEF res, KILL cr);
9542 format %{
9543 "cmpxchg_acq $res = $mem, $oldval, $newval\t# (long, weak) if $mem == $oldval then $mem <-- $newval"
9544 %}
9545 ins_encode %{
9546 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
9547 Assembler::xword, /*acquire*/ true, /*release*/ true,
9548 /*weak*/ false, $res$$Register);
9549 %}
9550 ins_pipe(pipe_slow);
9551 %}
9552
9553 // This pattern is generated automatically from cas.m4.
9554 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9555 instruct compareAndExchangeNAcq(iRegNNoSp res, indirect mem, iRegN oldval, iRegN newval, rFlagsReg cr) %{
9556 predicate(needs_acquiring_load_exclusive(n));
9557 match(Set res (CompareAndExchangeN mem (Binary oldval newval)));
9558 ins_cost(VOLATILE_REF_COST);
9559 effect(TEMP_DEF res, KILL cr);
9560 format %{
9561 "cmpxchgw_acq $res = $mem, $oldval, $newval\t# (narrow oop, weak) if $mem == $oldval then $mem <-- $newval"
9562 %}
9563 ins_encode %{
9564 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
9565 Assembler::word, /*acquire*/ true, /*release*/ true,
9566 /*weak*/ false, $res$$Register);
9567 %}
9568 ins_pipe(pipe_slow);
9569 %}
9570
9571 // This pattern is generated automatically from cas.m4.
9572 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9573 instruct compareAndExchangePAcq(iRegPNoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
9574 predicate(needs_acquiring_load_exclusive(n) && (n->as_LoadStore()->barrier_data() == 0));
9575 match(Set res (CompareAndExchangeP mem (Binary oldval newval)));
9576 ins_cost(VOLATILE_REF_COST);
9577 effect(TEMP_DEF res, KILL cr);
9578 format %{
9579 "cmpxchg_acq $res = $mem, $oldval, $newval\t# (ptr, weak) if $mem == $oldval then $mem <-- $newval"
9580 %}
9581 ins_encode %{
9582 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
9583 Assembler::xword, /*acquire*/ true, /*release*/ true,
9584 /*weak*/ false, $res$$Register);
9585 %}
9586 ins_pipe(pipe_slow);
9587 %}
9588
9589 // This pattern is generated automatically from cas.m4.
9590 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9591 instruct weakCompareAndSwapB(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
9592
9593 match(Set res (WeakCompareAndSwapB mem (Binary oldval newval)));
9594 ins_cost(2 * VOLATILE_REF_COST);
9595 effect(KILL cr);
9596 format %{
9597 "cmpxchgb $res = $mem, $oldval, $newval\t# (byte, weak) if $mem == $oldval then $mem <-- $newval"
9598 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
9599 %}
9600 ins_encode %{
9601 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
9602 Assembler::byte, /*acquire*/ false, /*release*/ true,
9603 /*weak*/ true, noreg);
9604 __ csetw($res$$Register, Assembler::EQ);
9605 %}
9606 ins_pipe(pipe_slow);
9607 %}
9608
9609 // This pattern is generated automatically from cas.m4.
9610 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9611 instruct weakCompareAndSwapS(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
9612
9613 match(Set res (WeakCompareAndSwapS mem (Binary oldval newval)));
9614 ins_cost(2 * VOLATILE_REF_COST);
9615 effect(KILL cr);
9616 format %{
9617 "cmpxchgs $res = $mem, $oldval, $newval\t# (short, weak) if $mem == $oldval then $mem <-- $newval"
9618 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
9619 %}
9620 ins_encode %{
9621 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
9622 Assembler::halfword, /*acquire*/ false, /*release*/ true,
9623 /*weak*/ true, noreg);
9624 __ csetw($res$$Register, Assembler::EQ);
9625 %}
9626 ins_pipe(pipe_slow);
9627 %}
9628
9629 // This pattern is generated automatically from cas.m4.
9630 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9631 instruct weakCompareAndSwapI(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
9632
9633 match(Set res (WeakCompareAndSwapI mem (Binary oldval newval)));
9634 ins_cost(2 * VOLATILE_REF_COST);
9635 effect(KILL cr);
9636 format %{
9637 "cmpxchgw $res = $mem, $oldval, $newval\t# (int, weak) if $mem == $oldval then $mem <-- $newval"
9638 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
9639 %}
9640 ins_encode %{
9641 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
9642 Assembler::word, /*acquire*/ false, /*release*/ true,
9643 /*weak*/ true, noreg);
9644 __ csetw($res$$Register, Assembler::EQ);
9645 %}
9646 ins_pipe(pipe_slow);
9647 %}
9648
9649 // This pattern is generated automatically from cas.m4.
9650 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9651 instruct weakCompareAndSwapL(iRegINoSp res, indirect mem, iRegL oldval, iRegL newval, rFlagsReg cr) %{
9652
9653 match(Set res (WeakCompareAndSwapL mem (Binary oldval newval)));
9654 ins_cost(2 * VOLATILE_REF_COST);
9655 effect(KILL cr);
9656 format %{
9657 "cmpxchg $res = $mem, $oldval, $newval\t# (long, weak) if $mem == $oldval then $mem <-- $newval"
9658 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
9659 %}
9660 ins_encode %{
9661 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
9662 Assembler::xword, /*acquire*/ false, /*release*/ true,
9663 /*weak*/ true, noreg);
9664 __ csetw($res$$Register, Assembler::EQ);
9665 %}
9666 ins_pipe(pipe_slow);
9667 %}
9668
9669 // This pattern is generated automatically from cas.m4.
9670 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9671 instruct weakCompareAndSwapN(iRegINoSp res, indirect mem, iRegN oldval, iRegN newval, rFlagsReg cr) %{
9672
9673 match(Set res (WeakCompareAndSwapN mem (Binary oldval newval)));
9674 ins_cost(2 * VOLATILE_REF_COST);
9675 effect(KILL cr);
9676 format %{
9677 "cmpxchgw $res = $mem, $oldval, $newval\t# (narrow oop, weak) if $mem == $oldval then $mem <-- $newval"
9678 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
9679 %}
9680 ins_encode %{
9681 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
9682 Assembler::word, /*acquire*/ false, /*release*/ true,
9683 /*weak*/ true, noreg);
9684 __ csetw($res$$Register, Assembler::EQ);
9685 %}
9686 ins_pipe(pipe_slow);
9687 %}
9688
9689 // This pattern is generated automatically from cas.m4.
9690 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9691 instruct weakCompareAndSwapP(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
9692 predicate(n->as_LoadStore()->barrier_data() == 0);
9693 match(Set res (WeakCompareAndSwapP mem (Binary oldval newval)));
9694 ins_cost(2 * VOLATILE_REF_COST);
9695 effect(KILL cr);
9696 format %{
9697 "cmpxchg $res = $mem, $oldval, $newval\t# (ptr, weak) if $mem == $oldval then $mem <-- $newval"
9698 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
9699 %}
9700 ins_encode %{
9701 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
9702 Assembler::xword, /*acquire*/ false, /*release*/ true,
9703 /*weak*/ true, noreg);
9704 __ csetw($res$$Register, Assembler::EQ);
9705 %}
9706 ins_pipe(pipe_slow);
9707 %}
9708
9709 // This pattern is generated automatically from cas.m4.
9710 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9711 instruct weakCompareAndSwapBAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
9712 predicate(needs_acquiring_load_exclusive(n));
9713 match(Set res (WeakCompareAndSwapB mem (Binary oldval newval)));
9714 ins_cost(VOLATILE_REF_COST);
9715 effect(KILL cr);
9716 format %{
9717 "cmpxchgb_acq $res = $mem, $oldval, $newval\t# (byte, weak) if $mem == $oldval then $mem <-- $newval"
9718 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
9719 %}
9720 ins_encode %{
9721 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
9722 Assembler::byte, /*acquire*/ true, /*release*/ true,
9723 /*weak*/ true, noreg);
9724 __ csetw($res$$Register, Assembler::EQ);
9725 %}
9726 ins_pipe(pipe_slow);
9727 %}
9728
9729 // This pattern is generated automatically from cas.m4.
9730 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9731 instruct weakCompareAndSwapSAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
9732 predicate(needs_acquiring_load_exclusive(n));
9733 match(Set res (WeakCompareAndSwapS mem (Binary oldval newval)));
9734 ins_cost(VOLATILE_REF_COST);
9735 effect(KILL cr);
9736 format %{
9737 "cmpxchgs_acq $res = $mem, $oldval, $newval\t# (short, weak) if $mem == $oldval then $mem <-- $newval"
9738 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
9739 %}
9740 ins_encode %{
9741 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
9742 Assembler::halfword, /*acquire*/ true, /*release*/ true,
9743 /*weak*/ true, noreg);
9744 __ csetw($res$$Register, Assembler::EQ);
9745 %}
9746 ins_pipe(pipe_slow);
9747 %}
9748
9749 // This pattern is generated automatically from cas.m4.
9750 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9751 instruct weakCompareAndSwapIAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
9752 predicate(needs_acquiring_load_exclusive(n));
9753 match(Set res (WeakCompareAndSwapI mem (Binary oldval newval)));
9754 ins_cost(VOLATILE_REF_COST);
9755 effect(KILL cr);
9756 format %{
9757 "cmpxchgw_acq $res = $mem, $oldval, $newval\t# (int, weak) if $mem == $oldval then $mem <-- $newval"
9758 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
9759 %}
9760 ins_encode %{
9761 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
9762 Assembler::word, /*acquire*/ true, /*release*/ true,
9763 /*weak*/ true, noreg);
9764 __ csetw($res$$Register, Assembler::EQ);
9765 %}
9766 ins_pipe(pipe_slow);
9767 %}
9768
9769 // This pattern is generated automatically from cas.m4.
9770 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9771 instruct weakCompareAndSwapLAcq(iRegINoSp res, indirect mem, iRegL oldval, iRegL newval, rFlagsReg cr) %{
9772 predicate(needs_acquiring_load_exclusive(n));
9773 match(Set res (WeakCompareAndSwapL mem (Binary oldval newval)));
9774 ins_cost(VOLATILE_REF_COST);
9775 effect(KILL cr);
9776 format %{
9777 "cmpxchg_acq $res = $mem, $oldval, $newval\t# (long, weak) if $mem == $oldval then $mem <-- $newval"
9778 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
9779 %}
9780 ins_encode %{
9781 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
9782 Assembler::xword, /*acquire*/ true, /*release*/ true,
9783 /*weak*/ true, noreg);
9784 __ csetw($res$$Register, Assembler::EQ);
9785 %}
9786 ins_pipe(pipe_slow);
9787 %}
9788
9789 // This pattern is generated automatically from cas.m4.
9790 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9791 instruct weakCompareAndSwapNAcq(iRegINoSp res, indirect mem, iRegN oldval, iRegN newval, rFlagsReg cr) %{
9792 predicate(needs_acquiring_load_exclusive(n));
9793 match(Set res (WeakCompareAndSwapN mem (Binary oldval newval)));
9794 ins_cost(VOLATILE_REF_COST);
9795 effect(KILL cr);
9796 format %{
9797 "cmpxchgw_acq $res = $mem, $oldval, $newval\t# (narrow oop, weak) if $mem == $oldval then $mem <-- $newval"
9798 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
9799 %}
9800 ins_encode %{
9801 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
9802 Assembler::word, /*acquire*/ true, /*release*/ true,
9803 /*weak*/ true, noreg);
9804 __ csetw($res$$Register, Assembler::EQ);
9805 %}
9806 ins_pipe(pipe_slow);
9807 %}
9808
9809 // This pattern is generated automatically from cas.m4.
9810 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9811 instruct weakCompareAndSwapPAcq(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
9812 predicate(needs_acquiring_load_exclusive(n) && (n->as_LoadStore()->barrier_data() == 0));
9813 match(Set res (WeakCompareAndSwapP mem (Binary oldval newval)));
9814 ins_cost(VOLATILE_REF_COST);
9815 effect(KILL cr);
9816 format %{
9817 "cmpxchg_acq $res = $mem, $oldval, $newval\t# (ptr, weak) if $mem == $oldval then $mem <-- $newval"
9818 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
9819 %}
9820 ins_encode %{
9821 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
9822 Assembler::xword, /*acquire*/ true, /*release*/ true,
9823 /*weak*/ true, noreg);
9824 __ csetw($res$$Register, Assembler::EQ);
9825 %}
9826 ins_pipe(pipe_slow);
9827 %}
9828
9829 // END This section of the file is automatically generated. Do not edit --------------
9830 // ---------------------------------------------------------------------
9831
9832 instruct get_and_setI(indirect mem, iRegI newv, iRegINoSp prev) %{
9833 match(Set prev (GetAndSetI mem newv));
9834 ins_cost(2 * VOLATILE_REF_COST);
9835 format %{ "atomic_xchgw $prev, $newv, [$mem]" %}
9836 ins_encode %{
9837 __ atomic_xchgw($prev$$Register, $newv$$Register, as_Register($mem$$base));
9838 %}
9839 ins_pipe(pipe_serial);
9840 %}
9841
9842 instruct get_and_setL(indirect mem, iRegL newv, iRegLNoSp prev) %{
9843 match(Set prev (GetAndSetL mem newv));
9844 ins_cost(2 * VOLATILE_REF_COST);
9845 format %{ "atomic_xchg $prev, $newv, [$mem]" %}
9846 ins_encode %{
9847 __ atomic_xchg($prev$$Register, $newv$$Register, as_Register($mem$$base));
9848 %}
9849 ins_pipe(pipe_serial);
9850 %}
9851
9852 instruct get_and_setN(indirect mem, iRegN newv, iRegINoSp prev) %{
9853 match(Set prev (GetAndSetN mem newv));
9854 ins_cost(2 * VOLATILE_REF_COST);
9855 format %{ "atomic_xchgw $prev, $newv, [$mem]" %}
9856 ins_encode %{
9857 __ atomic_xchgw($prev$$Register, $newv$$Register, as_Register($mem$$base));
9858 %}
9859 ins_pipe(pipe_serial);
9860 %}
9861
9862 instruct get_and_setP(indirect mem, iRegP newv, iRegPNoSp prev) %{
9863 predicate(n->as_LoadStore()->barrier_data() == 0);
9864 match(Set prev (GetAndSetP mem newv));
9865 ins_cost(2 * VOLATILE_REF_COST);
9866 format %{ "atomic_xchg $prev, $newv, [$mem]" %}
9867 ins_encode %{
9868 __ atomic_xchg($prev$$Register, $newv$$Register, as_Register($mem$$base));
9869 %}
9870 ins_pipe(pipe_serial);
9871 %}
9872
9873 instruct get_and_setIAcq(indirect mem, iRegI newv, iRegINoSp prev) %{
9874 predicate(needs_acquiring_load_exclusive(n));
9875 match(Set prev (GetAndSetI mem newv));
9876 ins_cost(VOLATILE_REF_COST);
9877 format %{ "atomic_xchgw_acq $prev, $newv, [$mem]" %}
9878 ins_encode %{
9879 __ atomic_xchgalw($prev$$Register, $newv$$Register, as_Register($mem$$base));
9880 %}
9881 ins_pipe(pipe_serial);
9882 %}
9883
9884 instruct get_and_setLAcq(indirect mem, iRegL newv, iRegLNoSp prev) %{
9885 predicate(needs_acquiring_load_exclusive(n));
9886 match(Set prev (GetAndSetL mem newv));
9887 ins_cost(VOLATILE_REF_COST);
9888 format %{ "atomic_xchg_acq $prev, $newv, [$mem]" %}
9889 ins_encode %{
9890 __ atomic_xchgal($prev$$Register, $newv$$Register, as_Register($mem$$base));
9891 %}
9892 ins_pipe(pipe_serial);
9893 %}
9894
9895 instruct get_and_setNAcq(indirect mem, iRegN newv, iRegINoSp prev) %{
9896 predicate(needs_acquiring_load_exclusive(n));
9897 match(Set prev (GetAndSetN mem newv));
9898 ins_cost(VOLATILE_REF_COST);
9899 format %{ "atomic_xchgw_acq $prev, $newv, [$mem]" %}
9900 ins_encode %{
9901 __ atomic_xchgalw($prev$$Register, $newv$$Register, as_Register($mem$$base));
9902 %}
9903 ins_pipe(pipe_serial);
9904 %}
9905
9906 instruct get_and_setPAcq(indirect mem, iRegP newv, iRegPNoSp prev) %{
9907 predicate(needs_acquiring_load_exclusive(n) && (n->as_LoadStore()->barrier_data() == 0));
9908 match(Set prev (GetAndSetP mem newv));
9909 ins_cost(VOLATILE_REF_COST);
9910 format %{ "atomic_xchg_acq $prev, $newv, [$mem]" %}
9911 ins_encode %{
9912 __ atomic_xchgal($prev$$Register, $newv$$Register, as_Register($mem$$base));
9913 %}
9914 ins_pipe(pipe_serial);
9915 %}
9916
9917
9918 instruct get_and_addL(indirect mem, iRegLNoSp newval, iRegL incr) %{
9919 match(Set newval (GetAndAddL mem incr));
9920 ins_cost(2 * VOLATILE_REF_COST + 1);
9921 format %{ "get_and_addL $newval, [$mem], $incr" %}
9922 ins_encode %{
9923 __ atomic_add($newval$$Register, $incr$$Register, as_Register($mem$$base));
9924 %}
9925 ins_pipe(pipe_serial);
9926 %}
9927
9928 instruct get_and_addL_no_res(indirect mem, Universe dummy, iRegL incr) %{
9929 predicate(n->as_LoadStore()->result_not_used());
9930 match(Set dummy (GetAndAddL mem incr));
9931 ins_cost(2 * VOLATILE_REF_COST);
9932 format %{ "get_and_addL [$mem], $incr" %}
9933 ins_encode %{
9934 __ atomic_add(noreg, $incr$$Register, as_Register($mem$$base));
9935 %}
9936 ins_pipe(pipe_serial);
9937 %}
9938
9939 instruct get_and_addLi(indirect mem, iRegLNoSp newval, immLAddSub incr) %{
9940 match(Set newval (GetAndAddL mem incr));
9941 ins_cost(2 * VOLATILE_REF_COST + 1);
9942 format %{ "get_and_addL $newval, [$mem], $incr" %}
9943 ins_encode %{
9944 __ atomic_add($newval$$Register, $incr$$constant, as_Register($mem$$base));
9945 %}
9946 ins_pipe(pipe_serial);
9947 %}
9948
9949 instruct get_and_addLi_no_res(indirect mem, Universe dummy, immLAddSub incr) %{
9950 predicate(n->as_LoadStore()->result_not_used());
9951 match(Set dummy (GetAndAddL mem incr));
9952 ins_cost(2 * VOLATILE_REF_COST);
9953 format %{ "get_and_addL [$mem], $incr" %}
9954 ins_encode %{
9955 __ atomic_add(noreg, $incr$$constant, as_Register($mem$$base));
9956 %}
9957 ins_pipe(pipe_serial);
9958 %}
9959
9960 instruct get_and_addI(indirect mem, iRegINoSp newval, iRegIorL2I incr) %{
9961 match(Set newval (GetAndAddI mem incr));
9962 ins_cost(2 * VOLATILE_REF_COST + 1);
9963 format %{ "get_and_addI $newval, [$mem], $incr" %}
9964 ins_encode %{
9965 __ atomic_addw($newval$$Register, $incr$$Register, as_Register($mem$$base));
9966 %}
9967 ins_pipe(pipe_serial);
9968 %}
9969
9970 instruct get_and_addI_no_res(indirect mem, Universe dummy, iRegIorL2I incr) %{
9971 predicate(n->as_LoadStore()->result_not_used());
9972 match(Set dummy (GetAndAddI mem incr));
9973 ins_cost(2 * VOLATILE_REF_COST);
9974 format %{ "get_and_addI [$mem], $incr" %}
9975 ins_encode %{
9976 __ atomic_addw(noreg, $incr$$Register, as_Register($mem$$base));
9977 %}
9978 ins_pipe(pipe_serial);
9979 %}
9980
9981 instruct get_and_addIi(indirect mem, iRegINoSp newval, immIAddSub incr) %{
9982 match(Set newval (GetAndAddI mem incr));
9983 ins_cost(2 * VOLATILE_REF_COST + 1);
9984 format %{ "get_and_addI $newval, [$mem], $incr" %}
9985 ins_encode %{
9986 __ atomic_addw($newval$$Register, $incr$$constant, as_Register($mem$$base));
9987 %}
9988 ins_pipe(pipe_serial);
9989 %}
9990
9991 instruct get_and_addIi_no_res(indirect mem, Universe dummy, immIAddSub incr) %{
9992 predicate(n->as_LoadStore()->result_not_used());
9993 match(Set dummy (GetAndAddI mem incr));
9994 ins_cost(2 * VOLATILE_REF_COST);
9995 format %{ "get_and_addI [$mem], $incr" %}
9996 ins_encode %{
9997 __ atomic_addw(noreg, $incr$$constant, as_Register($mem$$base));
9998 %}
9999 ins_pipe(pipe_serial);
10000 %}
10001
10002 instruct get_and_addLAcq(indirect mem, iRegLNoSp newval, iRegL incr) %{
10003 predicate(needs_acquiring_load_exclusive(n));
10004 match(Set newval (GetAndAddL mem incr));
10005 ins_cost(VOLATILE_REF_COST + 1);
10006 format %{ "get_and_addL_acq $newval, [$mem], $incr" %}
10007 ins_encode %{
10008 __ atomic_addal($newval$$Register, $incr$$Register, as_Register($mem$$base));
10009 %}
10010 ins_pipe(pipe_serial);
10011 %}
10012
10013 instruct get_and_addL_no_resAcq(indirect mem, Universe dummy, iRegL incr) %{
10014 predicate(n->as_LoadStore()->result_not_used() && needs_acquiring_load_exclusive(n));
10015 match(Set dummy (GetAndAddL mem incr));
10016 ins_cost(VOLATILE_REF_COST);
10017 format %{ "get_and_addL_acq [$mem], $incr" %}
10018 ins_encode %{
10019 __ atomic_addal(noreg, $incr$$Register, as_Register($mem$$base));
10020 %}
10021 ins_pipe(pipe_serial);
10022 %}
10023
10024 instruct get_and_addLiAcq(indirect mem, iRegLNoSp newval, immLAddSub incr) %{
10025 predicate(needs_acquiring_load_exclusive(n));
10026 match(Set newval (GetAndAddL mem incr));
10027 ins_cost(VOLATILE_REF_COST + 1);
10028 format %{ "get_and_addL_acq $newval, [$mem], $incr" %}
10029 ins_encode %{
10030 __ atomic_addal($newval$$Register, $incr$$constant, as_Register($mem$$base));
10031 %}
10032 ins_pipe(pipe_serial);
10033 %}
10034
10035 instruct get_and_addLi_no_resAcq(indirect mem, Universe dummy, immLAddSub incr) %{
10036 predicate(n->as_LoadStore()->result_not_used() && needs_acquiring_load_exclusive(n));
10037 match(Set dummy (GetAndAddL mem incr));
10038 ins_cost(VOLATILE_REF_COST);
10039 format %{ "get_and_addL_acq [$mem], $incr" %}
10040 ins_encode %{
10041 __ atomic_addal(noreg, $incr$$constant, as_Register($mem$$base));
10042 %}
10043 ins_pipe(pipe_serial);
10044 %}
10045
10046 instruct get_and_addIAcq(indirect mem, iRegINoSp newval, iRegIorL2I incr) %{
10047 predicate(needs_acquiring_load_exclusive(n));
10048 match(Set newval (GetAndAddI mem incr));
10049 ins_cost(VOLATILE_REF_COST + 1);
10050 format %{ "get_and_addI_acq $newval, [$mem], $incr" %}
10051 ins_encode %{
10052 __ atomic_addalw($newval$$Register, $incr$$Register, as_Register($mem$$base));
10053 %}
10054 ins_pipe(pipe_serial);
10055 %}
10056
10057 instruct get_and_addI_no_resAcq(indirect mem, Universe dummy, iRegIorL2I incr) %{
10058 predicate(n->as_LoadStore()->result_not_used() && needs_acquiring_load_exclusive(n));
10059 match(Set dummy (GetAndAddI mem incr));
10060 ins_cost(VOLATILE_REF_COST);
10061 format %{ "get_and_addI_acq [$mem], $incr" %}
10062 ins_encode %{
10063 __ atomic_addalw(noreg, $incr$$Register, as_Register($mem$$base));
10064 %}
10065 ins_pipe(pipe_serial);
10066 %}
10067
10068 instruct get_and_addIiAcq(indirect mem, iRegINoSp newval, immIAddSub incr) %{
10069 predicate(needs_acquiring_load_exclusive(n));
10070 match(Set newval (GetAndAddI mem incr));
10071 ins_cost(VOLATILE_REF_COST + 1);
10072 format %{ "get_and_addI_acq $newval, [$mem], $incr" %}
10073 ins_encode %{
10074 __ atomic_addalw($newval$$Register, $incr$$constant, as_Register($mem$$base));
10075 %}
10076 ins_pipe(pipe_serial);
10077 %}
10078
10079 instruct get_and_addIi_no_resAcq(indirect mem, Universe dummy, immIAddSub incr) %{
10080 predicate(n->as_LoadStore()->result_not_used() && needs_acquiring_load_exclusive(n));
10081 match(Set dummy (GetAndAddI mem incr));
10082 ins_cost(VOLATILE_REF_COST);
10083 format %{ "get_and_addI_acq [$mem], $incr" %}
10084 ins_encode %{
10085 __ atomic_addalw(noreg, $incr$$constant, as_Register($mem$$base));
10086 %}
10087 ins_pipe(pipe_serial);
10088 %}
10089
10090 // Manifest a CmpL result in an integer register.
10091 // (src1 < src2) ? -1 : ((src1 > src2) ? 1 : 0)
10092 instruct cmpL3_reg_reg(iRegINoSp dst, iRegL src1, iRegL src2, rFlagsReg flags)
10093 %{
10094 match(Set dst (CmpL3 src1 src2));
10095 effect(KILL flags);
10096
10097 ins_cost(INSN_COST * 6);
10098 format %{
10099 "cmp $src1, $src2"
10100 "csetw $dst, ne"
10101 "cnegw $dst, lt"
10102 %}
10103 // format %{ "CmpL3 $dst, $src1, $src2" %}
10104 ins_encode %{
10105 __ cmp($src1$$Register, $src2$$Register);
10106 __ csetw($dst$$Register, Assembler::NE);
10107 __ cnegw($dst$$Register, $dst$$Register, Assembler::LT);
10108 %}
10109
10110 ins_pipe(pipe_class_default);
10111 %}
10112
10113 instruct cmpL3_reg_imm(iRegINoSp dst, iRegL src1, immLAddSub src2, rFlagsReg flags)
10114 %{
10115 match(Set dst (CmpL3 src1 src2));
10116 effect(KILL flags);
10117
10118 ins_cost(INSN_COST * 6);
10119 format %{
10120 "cmp $src1, $src2"
10121 "csetw $dst, ne"
10122 "cnegw $dst, lt"
10123 %}
10124 ins_encode %{
10125 int32_t con = (int32_t)$src2$$constant;
10126 if (con < 0) {
10127 __ adds(zr, $src1$$Register, -con);
10128 } else {
10129 __ subs(zr, $src1$$Register, con);
10130 }
10131 __ csetw($dst$$Register, Assembler::NE);
10132 __ cnegw($dst$$Register, $dst$$Register, Assembler::LT);
10133 %}
10134
10135 ins_pipe(pipe_class_default);
10136 %}
10137
10138 // ============================================================================
10139 // Conditional Move Instructions
10140
10141 // n.b. we have identical rules for both a signed compare op (cmpOp)
10142 // and an unsigned compare op (cmpOpU). it would be nice if we could
10143 // define an op class which merged both inputs and use it to type the
10144 // argument to a single rule. unfortunatelyt his fails because the
10145 // opclass does not live up to the COND_INTER interface of its
10146 // component operands. When the generic code tries to negate the
10147 // operand it ends up running the generci Machoper::negate method
10148 // which throws a ShouldNotHappen. So, we have to provide two flavours
10149 // of each rule, one for a cmpOp and a second for a cmpOpU (sigh).
10150
10151 instruct cmovI_reg_reg(cmpOp cmp, rFlagsReg cr, iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10152 match(Set dst (CMoveI (Binary cmp cr) (Binary src1 src2)));
10153
10154 ins_cost(INSN_COST * 2);
10155 format %{ "cselw $dst, $src2, $src1 $cmp\t# signed, int" %}
10156
10157 ins_encode %{
10158 __ cselw(as_Register($dst$$reg),
10159 as_Register($src2$$reg),
10160 as_Register($src1$$reg),
10161 (Assembler::Condition)$cmp$$cmpcode);
10162 %}
10163
10164 ins_pipe(icond_reg_reg);
10165 %}
10166
10167 instruct cmovUI_reg_reg(cmpOpU cmp, rFlagsRegU cr, iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10168 match(Set dst (CMoveI (Binary cmp cr) (Binary src1 src2)));
10169
10170 ins_cost(INSN_COST * 2);
10171 format %{ "cselw $dst, $src2, $src1 $cmp\t# unsigned, int" %}
10172
10173 ins_encode %{
10174 __ cselw(as_Register($dst$$reg),
10175 as_Register($src2$$reg),
10176 as_Register($src1$$reg),
10177 (Assembler::Condition)$cmp$$cmpcode);
10178 %}
10179
10180 ins_pipe(icond_reg_reg);
10181 %}
10182
10183 // special cases where one arg is zero
10184
10185 // n.b. this is selected in preference to the rule above because it
10186 // avoids loading constant 0 into a source register
10187
10188 // TODO
10189 // we ought only to be able to cull one of these variants as the ideal
10190 // transforms ought always to order the zero consistently (to left/right?)
10191
10192 instruct cmovI_zero_reg(cmpOp cmp, rFlagsReg cr, iRegINoSp dst, immI0 zero, iRegIorL2I src) %{
10193 match(Set dst (CMoveI (Binary cmp cr) (Binary zero src)));
10194
10195 ins_cost(INSN_COST * 2);
10196 format %{ "cselw $dst, $src, zr $cmp\t# signed, int" %}
10197
10198 ins_encode %{
10199 __ cselw(as_Register($dst$$reg),
10200 as_Register($src$$reg),
10201 zr,
10202 (Assembler::Condition)$cmp$$cmpcode);
10203 %}
10204
10205 ins_pipe(icond_reg);
10206 %}
10207
10208 instruct cmovUI_zero_reg(cmpOpU cmp, rFlagsRegU cr, iRegINoSp dst, immI0 zero, iRegIorL2I src) %{
10209 match(Set dst (CMoveI (Binary cmp cr) (Binary zero src)));
10210
10211 ins_cost(INSN_COST * 2);
10212 format %{ "cselw $dst, $src, zr $cmp\t# unsigned, int" %}
10213
10214 ins_encode %{
10215 __ cselw(as_Register($dst$$reg),
10216 as_Register($src$$reg),
10217 zr,
10218 (Assembler::Condition)$cmp$$cmpcode);
10219 %}
10220
10221 ins_pipe(icond_reg);
10222 %}
10223
10224 instruct cmovI_reg_zero(cmpOp cmp, rFlagsReg cr, iRegINoSp dst, iRegIorL2I src, immI0 zero) %{
10225 match(Set dst (CMoveI (Binary cmp cr) (Binary src zero)));
10226
10227 ins_cost(INSN_COST * 2);
10228 format %{ "cselw $dst, zr, $src $cmp\t# signed, int" %}
10229
10230 ins_encode %{
10231 __ cselw(as_Register($dst$$reg),
10232 zr,
10233 as_Register($src$$reg),
10234 (Assembler::Condition)$cmp$$cmpcode);
10235 %}
10236
10237 ins_pipe(icond_reg);
10238 %}
10239
10240 instruct cmovUI_reg_zero(cmpOpU cmp, rFlagsRegU cr, iRegINoSp dst, iRegIorL2I src, immI0 zero) %{
10241 match(Set dst (CMoveI (Binary cmp cr) (Binary src zero)));
10242
10243 ins_cost(INSN_COST * 2);
10244 format %{ "cselw $dst, zr, $src $cmp\t# unsigned, int" %}
10245
10246 ins_encode %{
10247 __ cselw(as_Register($dst$$reg),
10248 zr,
10249 as_Register($src$$reg),
10250 (Assembler::Condition)$cmp$$cmpcode);
10251 %}
10252
10253 ins_pipe(icond_reg);
10254 %}
10255
10256 // special case for creating a boolean 0 or 1
10257
10258 // n.b. this is selected in preference to the rule above because it
10259 // avoids loading constants 0 and 1 into a source register
10260
10261 instruct cmovI_reg_zero_one(cmpOp cmp, rFlagsReg cr, iRegINoSp dst, immI0 zero, immI_1 one) %{
10262 match(Set dst (CMoveI (Binary cmp cr) (Binary one zero)));
10263
10264 ins_cost(INSN_COST * 2);
10265 format %{ "csincw $dst, zr, zr $cmp\t# signed, int" %}
10266
10267 ins_encode %{
10268 // equivalently
10269 // cset(as_Register($dst$$reg),
10270 // negate_condition((Assembler::Condition)$cmp$$cmpcode));
10271 __ csincw(as_Register($dst$$reg),
10272 zr,
10273 zr,
10274 (Assembler::Condition)$cmp$$cmpcode);
10275 %}
10276
10277 ins_pipe(icond_none);
10278 %}
10279
10280 instruct cmovUI_reg_zero_one(cmpOpU cmp, rFlagsRegU cr, iRegINoSp dst, immI0 zero, immI_1 one) %{
10281 match(Set dst (CMoveI (Binary cmp cr) (Binary one zero)));
10282
10283 ins_cost(INSN_COST * 2);
10284 format %{ "csincw $dst, zr, zr $cmp\t# unsigned, int" %}
10285
10286 ins_encode %{
10287 // equivalently
10288 // cset(as_Register($dst$$reg),
10289 // negate_condition((Assembler::Condition)$cmp$$cmpcode));
10290 __ csincw(as_Register($dst$$reg),
10291 zr,
10292 zr,
10293 (Assembler::Condition)$cmp$$cmpcode);
10294 %}
10295
10296 ins_pipe(icond_none);
10297 %}
10298
10299 instruct cmovL_reg_reg(cmpOp cmp, rFlagsReg cr, iRegLNoSp dst, iRegL src1, iRegL src2) %{
10300 match(Set dst (CMoveL (Binary cmp cr) (Binary src1 src2)));
10301
10302 ins_cost(INSN_COST * 2);
10303 format %{ "csel $dst, $src2, $src1 $cmp\t# signed, long" %}
10304
10305 ins_encode %{
10306 __ csel(as_Register($dst$$reg),
10307 as_Register($src2$$reg),
10308 as_Register($src1$$reg),
10309 (Assembler::Condition)$cmp$$cmpcode);
10310 %}
10311
10312 ins_pipe(icond_reg_reg);
10313 %}
10314
10315 instruct cmovUL_reg_reg(cmpOpU cmp, rFlagsRegU cr, iRegLNoSp dst, iRegL src1, iRegL src2) %{
10316 match(Set dst (CMoveL (Binary cmp cr) (Binary src1 src2)));
10317
10318 ins_cost(INSN_COST * 2);
10319 format %{ "csel $dst, $src2, $src1 $cmp\t# unsigned, long" %}
10320
10321 ins_encode %{
10322 __ csel(as_Register($dst$$reg),
10323 as_Register($src2$$reg),
10324 as_Register($src1$$reg),
10325 (Assembler::Condition)$cmp$$cmpcode);
10326 %}
10327
10328 ins_pipe(icond_reg_reg);
10329 %}
10330
10331 // special cases where one arg is zero
10332
10333 instruct cmovL_reg_zero(cmpOp cmp, rFlagsReg cr, iRegLNoSp dst, iRegL src, immL0 zero) %{
10334 match(Set dst (CMoveL (Binary cmp cr) (Binary src zero)));
10335
10336 ins_cost(INSN_COST * 2);
10337 format %{ "csel $dst, zr, $src $cmp\t# signed, long" %}
10338
10339 ins_encode %{
10340 __ csel(as_Register($dst$$reg),
10341 zr,
10342 as_Register($src$$reg),
10343 (Assembler::Condition)$cmp$$cmpcode);
10344 %}
10345
10346 ins_pipe(icond_reg);
10347 %}
10348
10349 instruct cmovUL_reg_zero(cmpOpU cmp, rFlagsRegU cr, iRegLNoSp dst, iRegL src, immL0 zero) %{
10350 match(Set dst (CMoveL (Binary cmp cr) (Binary src zero)));
10351
10352 ins_cost(INSN_COST * 2);
10353 format %{ "csel $dst, zr, $src $cmp\t# unsigned, long" %}
10354
10355 ins_encode %{
10356 __ csel(as_Register($dst$$reg),
10357 zr,
10358 as_Register($src$$reg),
10359 (Assembler::Condition)$cmp$$cmpcode);
10360 %}
10361
10362 ins_pipe(icond_reg);
10363 %}
10364
10365 instruct cmovL_zero_reg(cmpOp cmp, rFlagsReg cr, iRegLNoSp dst, immL0 zero, iRegL src) %{
10366 match(Set dst (CMoveL (Binary cmp cr) (Binary zero src)));
10367
10368 ins_cost(INSN_COST * 2);
10369 format %{ "csel $dst, $src, zr $cmp\t# signed, long" %}
10370
10371 ins_encode %{
10372 __ csel(as_Register($dst$$reg),
10373 as_Register($src$$reg),
10374 zr,
10375 (Assembler::Condition)$cmp$$cmpcode);
10376 %}
10377
10378 ins_pipe(icond_reg);
10379 %}
10380
10381 instruct cmovUL_zero_reg(cmpOpU cmp, rFlagsRegU cr, iRegLNoSp dst, immL0 zero, iRegL src) %{
10382 match(Set dst (CMoveL (Binary cmp cr) (Binary zero src)));
10383
10384 ins_cost(INSN_COST * 2);
10385 format %{ "csel $dst, $src, zr $cmp\t# unsigned, long" %}
10386
10387 ins_encode %{
10388 __ csel(as_Register($dst$$reg),
10389 as_Register($src$$reg),
10390 zr,
10391 (Assembler::Condition)$cmp$$cmpcode);
10392 %}
10393
10394 ins_pipe(icond_reg);
10395 %}
10396
10397 instruct cmovP_reg_reg(cmpOp cmp, rFlagsReg cr, iRegPNoSp dst, iRegP src1, iRegP src2) %{
10398 match(Set dst (CMoveP (Binary cmp cr) (Binary src1 src2)));
10399
10400 ins_cost(INSN_COST * 2);
10401 format %{ "csel $dst, $src2, $src1 $cmp\t# signed, ptr" %}
10402
10403 ins_encode %{
10404 __ csel(as_Register($dst$$reg),
10405 as_Register($src2$$reg),
10406 as_Register($src1$$reg),
10407 (Assembler::Condition)$cmp$$cmpcode);
10408 %}
10409
10410 ins_pipe(icond_reg_reg);
10411 %}
10412
10413 instruct cmovUP_reg_reg(cmpOpU cmp, rFlagsRegU cr, iRegPNoSp dst, iRegP src1, iRegP src2) %{
10414 match(Set dst (CMoveP (Binary cmp cr) (Binary src1 src2)));
10415
10416 ins_cost(INSN_COST * 2);
10417 format %{ "csel $dst, $src2, $src1 $cmp\t# unsigned, ptr" %}
10418
10419 ins_encode %{
10420 __ csel(as_Register($dst$$reg),
10421 as_Register($src2$$reg),
10422 as_Register($src1$$reg),
10423 (Assembler::Condition)$cmp$$cmpcode);
10424 %}
10425
10426 ins_pipe(icond_reg_reg);
10427 %}
10428
10429 // special cases where one arg is zero
10430
10431 instruct cmovP_reg_zero(cmpOp cmp, rFlagsReg cr, iRegPNoSp dst, iRegP src, immP0 zero) %{
10432 match(Set dst (CMoveP (Binary cmp cr) (Binary src zero)));
10433
10434 ins_cost(INSN_COST * 2);
10435 format %{ "csel $dst, zr, $src $cmp\t# signed, ptr" %}
10436
10437 ins_encode %{
10438 __ csel(as_Register($dst$$reg),
10439 zr,
10440 as_Register($src$$reg),
10441 (Assembler::Condition)$cmp$$cmpcode);
10442 %}
10443
10444 ins_pipe(icond_reg);
10445 %}
10446
10447 instruct cmovUP_reg_zero(cmpOpU cmp, rFlagsRegU cr, iRegPNoSp dst, iRegP src, immP0 zero) %{
10448 match(Set dst (CMoveP (Binary cmp cr) (Binary src zero)));
10449
10450 ins_cost(INSN_COST * 2);
10451 format %{ "csel $dst, zr, $src $cmp\t# unsigned, ptr" %}
10452
10453 ins_encode %{
10454 __ csel(as_Register($dst$$reg),
10455 zr,
10456 as_Register($src$$reg),
10457 (Assembler::Condition)$cmp$$cmpcode);
10458 %}
10459
10460 ins_pipe(icond_reg);
10461 %}
10462
10463 instruct cmovP_zero_reg(cmpOp cmp, rFlagsReg cr, iRegPNoSp dst, immP0 zero, iRegP src) %{
10464 match(Set dst (CMoveP (Binary cmp cr) (Binary zero src)));
10465
10466 ins_cost(INSN_COST * 2);
10467 format %{ "csel $dst, $src, zr $cmp\t# signed, ptr" %}
10468
10469 ins_encode %{
10470 __ csel(as_Register($dst$$reg),
10471 as_Register($src$$reg),
10472 zr,
10473 (Assembler::Condition)$cmp$$cmpcode);
10474 %}
10475
10476 ins_pipe(icond_reg);
10477 %}
10478
10479 instruct cmovUP_zero_reg(cmpOpU cmp, rFlagsRegU cr, iRegPNoSp dst, immP0 zero, iRegP src) %{
10480 match(Set dst (CMoveP (Binary cmp cr) (Binary zero src)));
10481
10482 ins_cost(INSN_COST * 2);
10483 format %{ "csel $dst, $src, zr $cmp\t# unsigned, ptr" %}
10484
10485 ins_encode %{
10486 __ csel(as_Register($dst$$reg),
10487 as_Register($src$$reg),
10488 zr,
10489 (Assembler::Condition)$cmp$$cmpcode);
10490 %}
10491
10492 ins_pipe(icond_reg);
10493 %}
10494
10495 instruct cmovN_reg_reg(cmpOp cmp, rFlagsReg cr, iRegNNoSp dst, iRegN src1, iRegN src2) %{
10496 match(Set dst (CMoveN (Binary cmp cr) (Binary src1 src2)));
10497
10498 ins_cost(INSN_COST * 2);
10499 format %{ "cselw $dst, $src2, $src1 $cmp\t# signed, compressed ptr" %}
10500
10501 ins_encode %{
10502 __ cselw(as_Register($dst$$reg),
10503 as_Register($src2$$reg),
10504 as_Register($src1$$reg),
10505 (Assembler::Condition)$cmp$$cmpcode);
10506 %}
10507
10508 ins_pipe(icond_reg_reg);
10509 %}
10510
10511 instruct cmovUN_reg_reg(cmpOpU cmp, rFlagsRegU cr, iRegNNoSp dst, iRegN src1, iRegN src2) %{
10512 match(Set dst (CMoveN (Binary cmp cr) (Binary src1 src2)));
10513
10514 ins_cost(INSN_COST * 2);
10515 format %{ "cselw $dst, $src2, $src1 $cmp\t# signed, compressed ptr" %}
10516
10517 ins_encode %{
10518 __ cselw(as_Register($dst$$reg),
10519 as_Register($src2$$reg),
10520 as_Register($src1$$reg),
10521 (Assembler::Condition)$cmp$$cmpcode);
10522 %}
10523
10524 ins_pipe(icond_reg_reg);
10525 %}
10526
10527 // special cases where one arg is zero
10528
10529 instruct cmovN_reg_zero(cmpOp cmp, rFlagsReg cr, iRegNNoSp dst, iRegN src, immN0 zero) %{
10530 match(Set dst (CMoveN (Binary cmp cr) (Binary src zero)));
10531
10532 ins_cost(INSN_COST * 2);
10533 format %{ "cselw $dst, zr, $src $cmp\t# signed, compressed ptr" %}
10534
10535 ins_encode %{
10536 __ cselw(as_Register($dst$$reg),
10537 zr,
10538 as_Register($src$$reg),
10539 (Assembler::Condition)$cmp$$cmpcode);
10540 %}
10541
10542 ins_pipe(icond_reg);
10543 %}
10544
10545 instruct cmovUN_reg_zero(cmpOpU cmp, rFlagsRegU cr, iRegNNoSp dst, iRegN src, immN0 zero) %{
10546 match(Set dst (CMoveN (Binary cmp cr) (Binary src zero)));
10547
10548 ins_cost(INSN_COST * 2);
10549 format %{ "cselw $dst, zr, $src $cmp\t# unsigned, compressed ptr" %}
10550
10551 ins_encode %{
10552 __ cselw(as_Register($dst$$reg),
10553 zr,
10554 as_Register($src$$reg),
10555 (Assembler::Condition)$cmp$$cmpcode);
10556 %}
10557
10558 ins_pipe(icond_reg);
10559 %}
10560
10561 instruct cmovN_zero_reg(cmpOp cmp, rFlagsReg cr, iRegNNoSp dst, immN0 zero, iRegN src) %{
10562 match(Set dst (CMoveN (Binary cmp cr) (Binary zero src)));
10563
10564 ins_cost(INSN_COST * 2);
10565 format %{ "cselw $dst, $src, zr $cmp\t# signed, compressed ptr" %}
10566
10567 ins_encode %{
10568 __ cselw(as_Register($dst$$reg),
10569 as_Register($src$$reg),
10570 zr,
10571 (Assembler::Condition)$cmp$$cmpcode);
10572 %}
10573
10574 ins_pipe(icond_reg);
10575 %}
10576
10577 instruct cmovUN_zero_reg(cmpOpU cmp, rFlagsRegU cr, iRegNNoSp dst, immN0 zero, iRegN src) %{
10578 match(Set dst (CMoveN (Binary cmp cr) (Binary zero src)));
10579
10580 ins_cost(INSN_COST * 2);
10581 format %{ "cselw $dst, $src, zr $cmp\t# unsigned, compressed ptr" %}
10582
10583 ins_encode %{
10584 __ cselw(as_Register($dst$$reg),
10585 as_Register($src$$reg),
10586 zr,
10587 (Assembler::Condition)$cmp$$cmpcode);
10588 %}
10589
10590 ins_pipe(icond_reg);
10591 %}
10592
10593 instruct cmovF_reg(cmpOp cmp, rFlagsReg cr, vRegF dst, vRegF src1, vRegF src2)
10594 %{
10595 match(Set dst (CMoveF (Binary cmp cr) (Binary src1 src2)));
10596
10597 ins_cost(INSN_COST * 3);
10598
10599 format %{ "fcsels $dst, $src1, $src2, $cmp\t# signed cmove float\n\t" %}
10600 ins_encode %{
10601 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
10602 __ fcsels(as_FloatRegister($dst$$reg),
10603 as_FloatRegister($src2$$reg),
10604 as_FloatRegister($src1$$reg),
10605 cond);
10606 %}
10607
10608 ins_pipe(fp_cond_reg_reg_s);
10609 %}
10610
10611 instruct cmovUF_reg(cmpOpU cmp, rFlagsRegU cr, vRegF dst, vRegF src1, vRegF src2)
10612 %{
10613 match(Set dst (CMoveF (Binary cmp cr) (Binary src1 src2)));
10614
10615 ins_cost(INSN_COST * 3);
10616
10617 format %{ "fcsels $dst, $src1, $src2, $cmp\t# unsigned cmove float\n\t" %}
10618 ins_encode %{
10619 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
10620 __ fcsels(as_FloatRegister($dst$$reg),
10621 as_FloatRegister($src2$$reg),
10622 as_FloatRegister($src1$$reg),
10623 cond);
10624 %}
10625
10626 ins_pipe(fp_cond_reg_reg_s);
10627 %}
10628
10629 instruct cmovD_reg(cmpOp cmp, rFlagsReg cr, vRegD dst, vRegD src1, vRegD src2)
10630 %{
10631 match(Set dst (CMoveD (Binary cmp cr) (Binary src1 src2)));
10632
10633 ins_cost(INSN_COST * 3);
10634
10635 format %{ "fcseld $dst, $src1, $src2, $cmp\t# signed cmove float\n\t" %}
10636 ins_encode %{
10637 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
10638 __ fcseld(as_FloatRegister($dst$$reg),
10639 as_FloatRegister($src2$$reg),
10640 as_FloatRegister($src1$$reg),
10641 cond);
10642 %}
10643
10644 ins_pipe(fp_cond_reg_reg_d);
10645 %}
10646
10647 instruct cmovUD_reg(cmpOpU cmp, rFlagsRegU cr, vRegD dst, vRegD src1, vRegD src2)
10648 %{
10649 match(Set dst (CMoveD (Binary cmp cr) (Binary src1 src2)));
10650
10651 ins_cost(INSN_COST * 3);
10652
10653 format %{ "fcseld $dst, $src1, $src2, $cmp\t# unsigned cmove float\n\t" %}
10654 ins_encode %{
10655 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
10656 __ fcseld(as_FloatRegister($dst$$reg),
10657 as_FloatRegister($src2$$reg),
10658 as_FloatRegister($src1$$reg),
10659 cond);
10660 %}
10661
10662 ins_pipe(fp_cond_reg_reg_d);
10663 %}
10664
10665 // ============================================================================
10666 // Arithmetic Instructions
10667 //
10668
10669 // Integer Addition
10670
10671 // TODO
10672 // these currently employ operations which do not set CR and hence are
10673 // not flagged as killing CR but we would like to isolate the cases
10674 // where we want to set flags from those where we don't. need to work
10675 // out how to do that.
10676
10677 instruct addI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10678 match(Set dst (AddI src1 src2));
10679
10680 ins_cost(INSN_COST);
10681 format %{ "addw $dst, $src1, $src2" %}
10682
10683 ins_encode %{
10684 __ addw(as_Register($dst$$reg),
10685 as_Register($src1$$reg),
10686 as_Register($src2$$reg));
10687 %}
10688
10689 ins_pipe(ialu_reg_reg);
10690 %}
10691
10692 instruct addI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immIAddSub src2) %{
10693 match(Set dst (AddI src1 src2));
10694
10695 ins_cost(INSN_COST);
10696 format %{ "addw $dst, $src1, $src2" %}
10697
10698 // use opcode to indicate that this is an add not a sub
10699 opcode(0x0);
10700
10701 ins_encode(aarch64_enc_addsubw_imm(dst, src1, src2));
10702
10703 ins_pipe(ialu_reg_imm);
10704 %}
10705
10706 instruct addI_reg_imm_i2l(iRegINoSp dst, iRegL src1, immIAddSub src2) %{
10707 match(Set dst (AddI (ConvL2I src1) src2));
10708
10709 ins_cost(INSN_COST);
10710 format %{ "addw $dst, $src1, $src2" %}
10711
10712 // use opcode to indicate that this is an add not a sub
10713 opcode(0x0);
10714
10715 ins_encode(aarch64_enc_addsubw_imm(dst, src1, src2));
10716
10717 ins_pipe(ialu_reg_imm);
10718 %}
10719
10720 // Pointer Addition
10721 instruct addP_reg_reg(iRegPNoSp dst, iRegP src1, iRegL src2) %{
10722 match(Set dst (AddP src1 src2));
10723
10724 ins_cost(INSN_COST);
10725 format %{ "add $dst, $src1, $src2\t# ptr" %}
10726
10727 ins_encode %{
10728 __ add(as_Register($dst$$reg),
10729 as_Register($src1$$reg),
10730 as_Register($src2$$reg));
10731 %}
10732
10733 ins_pipe(ialu_reg_reg);
10734 %}
10735
10736 instruct addP_reg_reg_ext(iRegPNoSp dst, iRegP src1, iRegIorL2I src2) %{
10737 match(Set dst (AddP src1 (ConvI2L src2)));
10738
10739 ins_cost(1.9 * INSN_COST);
10740 format %{ "add $dst, $src1, $src2, sxtw\t# ptr" %}
10741
10742 ins_encode %{
10743 __ add(as_Register($dst$$reg),
10744 as_Register($src1$$reg),
10745 as_Register($src2$$reg), ext::sxtw);
10746 %}
10747
10748 ins_pipe(ialu_reg_reg);
10749 %}
10750
10751 instruct addP_reg_reg_lsl(iRegPNoSp dst, iRegP src1, iRegL src2, immIScale scale) %{
10752 match(Set dst (AddP src1 (LShiftL src2 scale)));
10753
10754 ins_cost(1.9 * INSN_COST);
10755 format %{ "add $dst, $src1, $src2, LShiftL $scale\t# ptr" %}
10756
10757 ins_encode %{
10758 __ lea(as_Register($dst$$reg),
10759 Address(as_Register($src1$$reg), as_Register($src2$$reg),
10760 Address::lsl($scale$$constant)));
10761 %}
10762
10763 ins_pipe(ialu_reg_reg_shift);
10764 %}
10765
10766 instruct addP_reg_reg_ext_shift(iRegPNoSp dst, iRegP src1, iRegIorL2I src2, immIScale scale) %{
10767 match(Set dst (AddP src1 (LShiftL (ConvI2L src2) scale)));
10768
10769 ins_cost(1.9 * INSN_COST);
10770 format %{ "add $dst, $src1, $src2, I2L $scale\t# ptr" %}
10771
10772 ins_encode %{
10773 __ lea(as_Register($dst$$reg),
10774 Address(as_Register($src1$$reg), as_Register($src2$$reg),
10775 Address::sxtw($scale$$constant)));
10776 %}
10777
10778 ins_pipe(ialu_reg_reg_shift);
10779 %}
10780
10781 instruct lshift_ext(iRegLNoSp dst, iRegIorL2I src, immI scale, rFlagsReg cr) %{
10782 match(Set dst (LShiftL (ConvI2L src) scale));
10783
10784 ins_cost(INSN_COST);
10785 format %{ "sbfiz $dst, $src, $scale & 63, -$scale & 63\t" %}
10786
10787 ins_encode %{
10788 __ sbfiz(as_Register($dst$$reg),
10789 as_Register($src$$reg),
10790 $scale$$constant & 63, MIN2(32, (int)((-$scale$$constant) & 63)));
10791 %}
10792
10793 ins_pipe(ialu_reg_shift);
10794 %}
10795
10796 // Pointer Immediate Addition
10797 // n.b. this needs to be more expensive than using an indirect memory
10798 // operand
10799 instruct addP_reg_imm(iRegPNoSp dst, iRegP src1, immLAddSub src2) %{
10800 match(Set dst (AddP src1 src2));
10801
10802 ins_cost(INSN_COST);
10803 format %{ "add $dst, $src1, $src2\t# ptr" %}
10804
10805 // use opcode to indicate that this is an add not a sub
10806 opcode(0x0);
10807
10808 ins_encode( aarch64_enc_addsub_imm(dst, src1, src2) );
10809
10810 ins_pipe(ialu_reg_imm);
10811 %}
10812
10813 // Long Addition
10814 instruct addL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
10815
10816 match(Set dst (AddL src1 src2));
10817
10818 ins_cost(INSN_COST);
10819 format %{ "add $dst, $src1, $src2" %}
10820
10821 ins_encode %{
10822 __ add(as_Register($dst$$reg),
10823 as_Register($src1$$reg),
10824 as_Register($src2$$reg));
10825 %}
10826
10827 ins_pipe(ialu_reg_reg);
10828 %}
10829
10830 // No constant pool entries requiredLong Immediate Addition.
10831 instruct addL_reg_imm(iRegLNoSp dst, iRegL src1, immLAddSub src2) %{
10832 match(Set dst (AddL src1 src2));
10833
10834 ins_cost(INSN_COST);
10835 format %{ "add $dst, $src1, $src2" %}
10836
10837 // use opcode to indicate that this is an add not a sub
10838 opcode(0x0);
10839
10840 ins_encode( aarch64_enc_addsub_imm(dst, src1, src2) );
10841
10842 ins_pipe(ialu_reg_imm);
10843 %}
10844
10845 // Integer Subtraction
10846 instruct subI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10847 match(Set dst (SubI src1 src2));
10848
10849 ins_cost(INSN_COST);
10850 format %{ "subw $dst, $src1, $src2" %}
10851
10852 ins_encode %{
10853 __ subw(as_Register($dst$$reg),
10854 as_Register($src1$$reg),
10855 as_Register($src2$$reg));
10856 %}
10857
10858 ins_pipe(ialu_reg_reg);
10859 %}
10860
10861 // Immediate Subtraction
10862 instruct subI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immIAddSub src2) %{
10863 match(Set dst (SubI src1 src2));
10864
10865 ins_cost(INSN_COST);
10866 format %{ "subw $dst, $src1, $src2" %}
10867
10868 // use opcode to indicate that this is a sub not an add
10869 opcode(0x1);
10870
10871 ins_encode(aarch64_enc_addsubw_imm(dst, src1, src2));
10872
10873 ins_pipe(ialu_reg_imm);
10874 %}
10875
10876 // Long Subtraction
10877 instruct subL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
10878
10879 match(Set dst (SubL src1 src2));
10880
10881 ins_cost(INSN_COST);
10882 format %{ "sub $dst, $src1, $src2" %}
10883
10884 ins_encode %{
10885 __ sub(as_Register($dst$$reg),
10886 as_Register($src1$$reg),
10887 as_Register($src2$$reg));
10888 %}
10889
10890 ins_pipe(ialu_reg_reg);
10891 %}
10892
10893 // No constant pool entries requiredLong Immediate Subtraction.
10894 instruct subL_reg_imm(iRegLNoSp dst, iRegL src1, immLAddSub src2) %{
10895 match(Set dst (SubL src1 src2));
10896
10897 ins_cost(INSN_COST);
10898 format %{ "sub$dst, $src1, $src2" %}
10899
10900 // use opcode to indicate that this is a sub not an add
10901 opcode(0x1);
10902
10903 ins_encode( aarch64_enc_addsub_imm(dst, src1, src2) );
10904
10905 ins_pipe(ialu_reg_imm);
10906 %}
10907
10908 // Integer Negation (special case for sub)
10909
10910 instruct negI_reg(iRegINoSp dst, iRegIorL2I src, immI0 zero, rFlagsReg cr) %{
10911 match(Set dst (SubI zero src));
10912
10913 ins_cost(INSN_COST);
10914 format %{ "negw $dst, $src\t# int" %}
10915
10916 ins_encode %{
10917 __ negw(as_Register($dst$$reg),
10918 as_Register($src$$reg));
10919 %}
10920
10921 ins_pipe(ialu_reg);
10922 %}
10923
10924 // Long Negation
10925
10926 instruct negL_reg(iRegLNoSp dst, iRegL src, immL0 zero, rFlagsReg cr) %{
10927 match(Set dst (SubL zero src));
10928
10929 ins_cost(INSN_COST);
10930 format %{ "neg $dst, $src\t# long" %}
10931
10932 ins_encode %{
10933 __ neg(as_Register($dst$$reg),
10934 as_Register($src$$reg));
10935 %}
10936
10937 ins_pipe(ialu_reg);
10938 %}
10939
10940 // Integer Multiply
10941
10942 instruct mulI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10943 match(Set dst (MulI src1 src2));
10944
10945 ins_cost(INSN_COST * 3);
10946 format %{ "mulw $dst, $src1, $src2" %}
10947
10948 ins_encode %{
10949 __ mulw(as_Register($dst$$reg),
10950 as_Register($src1$$reg),
10951 as_Register($src2$$reg));
10952 %}
10953
10954 ins_pipe(imul_reg_reg);
10955 %}
10956
10957 instruct smulI(iRegLNoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10958 match(Set dst (MulL (ConvI2L src1) (ConvI2L src2)));
10959
10960 ins_cost(INSN_COST * 3);
10961 format %{ "smull $dst, $src1, $src2" %}
10962
10963 ins_encode %{
10964 __ smull(as_Register($dst$$reg),
10965 as_Register($src1$$reg),
10966 as_Register($src2$$reg));
10967 %}
10968
10969 ins_pipe(imul_reg_reg);
10970 %}
10971
10972 // Long Multiply
10973
10974 instruct mulL(iRegLNoSp dst, iRegL src1, iRegL src2) %{
10975 match(Set dst (MulL src1 src2));
10976
10977 ins_cost(INSN_COST * 5);
10978 format %{ "mul $dst, $src1, $src2" %}
10979
10980 ins_encode %{
10981 __ mul(as_Register($dst$$reg),
10982 as_Register($src1$$reg),
10983 as_Register($src2$$reg));
10984 %}
10985
10986 ins_pipe(lmul_reg_reg);
10987 %}
10988
10989 instruct mulHiL_rReg(iRegLNoSp dst, iRegL src1, iRegL src2, rFlagsReg cr)
10990 %{
10991 match(Set dst (MulHiL src1 src2));
10992
10993 ins_cost(INSN_COST * 7);
10994 format %{ "smulh $dst, $src1, $src2, \t# mulhi" %}
10995
10996 ins_encode %{
10997 __ smulh(as_Register($dst$$reg),
10998 as_Register($src1$$reg),
10999 as_Register($src2$$reg));
11000 %}
11001
11002 ins_pipe(lmul_reg_reg);
11003 %}
11004
11005 // Combined Integer Multiply & Add/Sub
11006
11007 instruct maddI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegIorL2I src3) %{
11008 match(Set dst (AddI src3 (MulI src1 src2)));
11009
11010 ins_cost(INSN_COST * 3);
11011 format %{ "madd $dst, $src1, $src2, $src3" %}
11012
11013 ins_encode %{
11014 __ maddw(as_Register($dst$$reg),
11015 as_Register($src1$$reg),
11016 as_Register($src2$$reg),
11017 as_Register($src3$$reg));
11018 %}
11019
11020 ins_pipe(imac_reg_reg);
11021 %}
11022
11023 instruct msubI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegIorL2I src3) %{
11024 match(Set dst (SubI src3 (MulI src1 src2)));
11025
11026 ins_cost(INSN_COST * 3);
11027 format %{ "msub $dst, $src1, $src2, $src3" %}
11028
11029 ins_encode %{
11030 __ msubw(as_Register($dst$$reg),
11031 as_Register($src1$$reg),
11032 as_Register($src2$$reg),
11033 as_Register($src3$$reg));
11034 %}
11035
11036 ins_pipe(imac_reg_reg);
11037 %}
11038
11039 // Combined Integer Multiply & Neg
11040
11041 instruct mnegI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI0 zero) %{
11042 match(Set dst (MulI (SubI zero src1) src2));
11043 match(Set dst (MulI src1 (SubI zero src2)));
11044
11045 ins_cost(INSN_COST * 3);
11046 format %{ "mneg $dst, $src1, $src2" %}
11047
11048 ins_encode %{
11049 __ mnegw(as_Register($dst$$reg),
11050 as_Register($src1$$reg),
11051 as_Register($src2$$reg));
11052 %}
11053
11054 ins_pipe(imac_reg_reg);
11055 %}
11056
11057 // Combined Long Multiply & Add/Sub
11058
11059 instruct maddL(iRegLNoSp dst, iRegL src1, iRegL src2, iRegL src3) %{
11060 match(Set dst (AddL src3 (MulL src1 src2)));
11061
11062 ins_cost(INSN_COST * 5);
11063 format %{ "madd $dst, $src1, $src2, $src3" %}
11064
11065 ins_encode %{
11066 __ madd(as_Register($dst$$reg),
11067 as_Register($src1$$reg),
11068 as_Register($src2$$reg),
11069 as_Register($src3$$reg));
11070 %}
11071
11072 ins_pipe(lmac_reg_reg);
11073 %}
11074
11075 instruct msubL(iRegLNoSp dst, iRegL src1, iRegL src2, iRegL src3) %{
11076 match(Set dst (SubL src3 (MulL src1 src2)));
11077
11078 ins_cost(INSN_COST * 5);
11079 format %{ "msub $dst, $src1, $src2, $src3" %}
11080
11081 ins_encode %{
11082 __ msub(as_Register($dst$$reg),
11083 as_Register($src1$$reg),
11084 as_Register($src2$$reg),
11085 as_Register($src3$$reg));
11086 %}
11087
11088 ins_pipe(lmac_reg_reg);
11089 %}
11090
11091 // Combined Long Multiply & Neg
11092
11093 instruct mnegL(iRegLNoSp dst, iRegL src1, iRegL src2, immL0 zero) %{
11094 match(Set dst (MulL (SubL zero src1) src2));
11095 match(Set dst (MulL src1 (SubL zero src2)));
11096
11097 ins_cost(INSN_COST * 5);
11098 format %{ "mneg $dst, $src1, $src2" %}
11099
11100 ins_encode %{
11101 __ mneg(as_Register($dst$$reg),
11102 as_Register($src1$$reg),
11103 as_Register($src2$$reg));
11104 %}
11105
11106 ins_pipe(lmac_reg_reg);
11107 %}
11108
11109 // Combine Integer Signed Multiply & Add/Sub/Neg Long
11110
11111 instruct smaddL(iRegLNoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegLNoSp src3) %{
11112 match(Set dst (AddL src3 (MulL (ConvI2L src1) (ConvI2L src2))));
11113
11114 ins_cost(INSN_COST * 3);
11115 format %{ "smaddl $dst, $src1, $src2, $src3" %}
11116
11117 ins_encode %{
11118 __ smaddl(as_Register($dst$$reg),
11119 as_Register($src1$$reg),
11120 as_Register($src2$$reg),
11121 as_Register($src3$$reg));
11122 %}
11123
11124 ins_pipe(imac_reg_reg);
11125 %}
11126
11127 instruct smsubL(iRegLNoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegLNoSp src3) %{
11128 match(Set dst (SubL src3 (MulL (ConvI2L src1) (ConvI2L src2))));
11129
11130 ins_cost(INSN_COST * 3);
11131 format %{ "smsubl $dst, $src1, $src2, $src3" %}
11132
11133 ins_encode %{
11134 __ smsubl(as_Register($dst$$reg),
11135 as_Register($src1$$reg),
11136 as_Register($src2$$reg),
11137 as_Register($src3$$reg));
11138 %}
11139
11140 ins_pipe(imac_reg_reg);
11141 %}
11142
11143 instruct smnegL(iRegLNoSp dst, iRegIorL2I src1, iRegIorL2I src2, immL0 zero) %{
11144 match(Set dst (MulL (SubL zero (ConvI2L src1)) (ConvI2L src2)));
11145 match(Set dst (MulL (ConvI2L src1) (SubL zero (ConvI2L src2))));
11146
11147 ins_cost(INSN_COST * 3);
11148 format %{ "smnegl $dst, $src1, $src2" %}
11149
11150 ins_encode %{
11151 __ smnegl(as_Register($dst$$reg),
11152 as_Register($src1$$reg),
11153 as_Register($src2$$reg));
11154 %}
11155
11156 ins_pipe(imac_reg_reg);
11157 %}
11158
11159 // Combined Multiply-Add Shorts into Integer (dst = src1 * src2 + src3 * src4)
11160
11161 instruct muladdS2I(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegIorL2I src3, iRegIorL2I src4) %{
11162 match(Set dst (MulAddS2I (Binary src1 src2) (Binary src3 src4)));
11163
11164 ins_cost(INSN_COST * 5);
11165 format %{ "mulw rscratch1, $src1, $src2\n\t"
11166 "maddw $dst, $src3, $src4, rscratch1" %}
11167
11168 ins_encode %{
11169 __ mulw(rscratch1, as_Register($src1$$reg), as_Register($src2$$reg));
11170 __ maddw(as_Register($dst$$reg), as_Register($src3$$reg), as_Register($src4$$reg), rscratch1); %}
11171
11172 ins_pipe(imac_reg_reg);
11173 %}
11174
11175 // Integer Divide
11176
11177 instruct divI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
11178 match(Set dst (DivI src1 src2));
11179
11180 ins_cost(INSN_COST * 19);
11181 format %{ "sdivw $dst, $src1, $src2" %}
11182
11183 ins_encode(aarch64_enc_divw(dst, src1, src2));
11184 ins_pipe(idiv_reg_reg);
11185 %}
11186
11187 // Long Divide
11188
11189 instruct divL(iRegLNoSp dst, iRegL src1, iRegL src2) %{
11190 match(Set dst (DivL src1 src2));
11191
11192 ins_cost(INSN_COST * 35);
11193 format %{ "sdiv $dst, $src1, $src2" %}
11194
11195 ins_encode(aarch64_enc_div(dst, src1, src2));
11196 ins_pipe(ldiv_reg_reg);
11197 %}
11198
11199 // Integer Remainder
11200
11201 instruct modI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
11202 match(Set dst (ModI src1 src2));
11203
11204 ins_cost(INSN_COST * 22);
11205 format %{ "sdivw rscratch1, $src1, $src2\n\t"
11206 "msubw($dst, rscratch1, $src2, $src1" %}
11207
11208 ins_encode(aarch64_enc_modw(dst, src1, src2));
11209 ins_pipe(idiv_reg_reg);
11210 %}
11211
11212 // Long Remainder
11213
11214 instruct modL(iRegLNoSp dst, iRegL src1, iRegL src2) %{
11215 match(Set dst (ModL src1 src2));
11216
11217 ins_cost(INSN_COST * 38);
11218 format %{ "sdiv rscratch1, $src1, $src2\n"
11219 "msub($dst, rscratch1, $src2, $src1" %}
11220
11221 ins_encode(aarch64_enc_mod(dst, src1, src2));
11222 ins_pipe(ldiv_reg_reg);
11223 %}
11224
11225 // Integer Shifts
11226
11227 // Shift Left Register
11228 instruct lShiftI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
11229 match(Set dst (LShiftI src1 src2));
11230
11231 ins_cost(INSN_COST * 2);
11232 format %{ "lslvw $dst, $src1, $src2" %}
11233
11234 ins_encode %{
11235 __ lslvw(as_Register($dst$$reg),
11236 as_Register($src1$$reg),
11237 as_Register($src2$$reg));
11238 %}
11239
11240 ins_pipe(ialu_reg_reg_vshift);
11241 %}
11242
11243 // Shift Left Immediate
11244 instruct lShiftI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immI src2) %{
11245 match(Set dst (LShiftI src1 src2));
11246
11247 ins_cost(INSN_COST);
11248 format %{ "lslw $dst, $src1, ($src2 & 0x1f)" %}
11249
11250 ins_encode %{
11251 __ lslw(as_Register($dst$$reg),
11252 as_Register($src1$$reg),
11253 $src2$$constant & 0x1f);
11254 %}
11255
11256 ins_pipe(ialu_reg_shift);
11257 %}
11258
11259 // Shift Right Logical Register
11260 instruct urShiftI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
11261 match(Set dst (URShiftI src1 src2));
11262
11263 ins_cost(INSN_COST * 2);
11264 format %{ "lsrvw $dst, $src1, $src2" %}
11265
11266 ins_encode %{
11267 __ lsrvw(as_Register($dst$$reg),
11268 as_Register($src1$$reg),
11269 as_Register($src2$$reg));
11270 %}
11271
11272 ins_pipe(ialu_reg_reg_vshift);
11273 %}
11274
11275 // Shift Right Logical Immediate
11276 instruct urShiftI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immI src2) %{
11277 match(Set dst (URShiftI src1 src2));
11278
11279 ins_cost(INSN_COST);
11280 format %{ "lsrw $dst, $src1, ($src2 & 0x1f)" %}
11281
11282 ins_encode %{
11283 __ lsrw(as_Register($dst$$reg),
11284 as_Register($src1$$reg),
11285 $src2$$constant & 0x1f);
11286 %}
11287
11288 ins_pipe(ialu_reg_shift);
11289 %}
11290
11291 // Shift Right Arithmetic Register
11292 instruct rShiftI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
11293 match(Set dst (RShiftI src1 src2));
11294
11295 ins_cost(INSN_COST * 2);
11296 format %{ "asrvw $dst, $src1, $src2" %}
11297
11298 ins_encode %{
11299 __ asrvw(as_Register($dst$$reg),
11300 as_Register($src1$$reg),
11301 as_Register($src2$$reg));
11302 %}
11303
11304 ins_pipe(ialu_reg_reg_vshift);
11305 %}
11306
11307 // Shift Right Arithmetic Immediate
11308 instruct rShiftI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immI src2) %{
11309 match(Set dst (RShiftI src1 src2));
11310
11311 ins_cost(INSN_COST);
11312 format %{ "asrw $dst, $src1, ($src2 & 0x1f)" %}
11313
11314 ins_encode %{
11315 __ asrw(as_Register($dst$$reg),
11316 as_Register($src1$$reg),
11317 $src2$$constant & 0x1f);
11318 %}
11319
11320 ins_pipe(ialu_reg_shift);
11321 %}
11322
11323 // Combined Int Mask and Right Shift (using UBFM)
11324 // TODO
11325
11326 // Long Shifts
11327
11328 // Shift Left Register
11329 instruct lShiftL_reg_reg(iRegLNoSp dst, iRegL src1, iRegIorL2I src2) %{
11330 match(Set dst (LShiftL src1 src2));
11331
11332 ins_cost(INSN_COST * 2);
11333 format %{ "lslv $dst, $src1, $src2" %}
11334
11335 ins_encode %{
11336 __ lslv(as_Register($dst$$reg),
11337 as_Register($src1$$reg),
11338 as_Register($src2$$reg));
11339 %}
11340
11341 ins_pipe(ialu_reg_reg_vshift);
11342 %}
11343
11344 // Shift Left Immediate
11345 instruct lShiftL_reg_imm(iRegLNoSp dst, iRegL src1, immI src2) %{
11346 match(Set dst (LShiftL src1 src2));
11347
11348 ins_cost(INSN_COST);
11349 format %{ "lsl $dst, $src1, ($src2 & 0x3f)" %}
11350
11351 ins_encode %{
11352 __ lsl(as_Register($dst$$reg),
11353 as_Register($src1$$reg),
11354 $src2$$constant & 0x3f);
11355 %}
11356
11357 ins_pipe(ialu_reg_shift);
11358 %}
11359
11360 // Shift Right Logical Register
11361 instruct urShiftL_reg_reg(iRegLNoSp dst, iRegL src1, iRegIorL2I src2) %{
11362 match(Set dst (URShiftL src1 src2));
11363
11364 ins_cost(INSN_COST * 2);
11365 format %{ "lsrv $dst, $src1, $src2" %}
11366
11367 ins_encode %{
11368 __ lsrv(as_Register($dst$$reg),
11369 as_Register($src1$$reg),
11370 as_Register($src2$$reg));
11371 %}
11372
11373 ins_pipe(ialu_reg_reg_vshift);
11374 %}
11375
11376 // Shift Right Logical Immediate
11377 instruct urShiftL_reg_imm(iRegLNoSp dst, iRegL src1, immI src2) %{
11378 match(Set dst (URShiftL src1 src2));
11379
11380 ins_cost(INSN_COST);
11381 format %{ "lsr $dst, $src1, ($src2 & 0x3f)" %}
11382
11383 ins_encode %{
11384 __ lsr(as_Register($dst$$reg),
11385 as_Register($src1$$reg),
11386 $src2$$constant & 0x3f);
11387 %}
11388
11389 ins_pipe(ialu_reg_shift);
11390 %}
11391
11392 // A special-case pattern for card table stores.
11393 instruct urShiftP_reg_imm(iRegLNoSp dst, iRegP src1, immI src2) %{
11394 match(Set dst (URShiftL (CastP2X src1) src2));
11395
11396 ins_cost(INSN_COST);
11397 format %{ "lsr $dst, p2x($src1), ($src2 & 0x3f)" %}
11398
11399 ins_encode %{
11400 __ lsr(as_Register($dst$$reg),
11401 as_Register($src1$$reg),
11402 $src2$$constant & 0x3f);
11403 %}
11404
11405 ins_pipe(ialu_reg_shift);
11406 %}
11407
11408 // Shift Right Arithmetic Register
11409 instruct rShiftL_reg_reg(iRegLNoSp dst, iRegL src1, iRegIorL2I src2) %{
11410 match(Set dst (RShiftL src1 src2));
11411
11412 ins_cost(INSN_COST * 2);
11413 format %{ "asrv $dst, $src1, $src2" %}
11414
11415 ins_encode %{
11416 __ asrv(as_Register($dst$$reg),
11417 as_Register($src1$$reg),
11418 as_Register($src2$$reg));
11419 %}
11420
11421 ins_pipe(ialu_reg_reg_vshift);
11422 %}
11423
11424 // Shift Right Arithmetic Immediate
11425 instruct rShiftL_reg_imm(iRegLNoSp dst, iRegL src1, immI src2) %{
11426 match(Set dst (RShiftL src1 src2));
11427
11428 ins_cost(INSN_COST);
11429 format %{ "asr $dst, $src1, ($src2 & 0x3f)" %}
11430
11431 ins_encode %{
11432 __ asr(as_Register($dst$$reg),
11433 as_Register($src1$$reg),
11434 $src2$$constant & 0x3f);
11435 %}
11436
11437 ins_pipe(ialu_reg_shift);
11438 %}
11439
11440 // BEGIN This section of the file is automatically generated. Do not edit --------------
11441 // This section is generated from aarch64_ad.m4
11442
11443
11444 // This pattern is automatically generated from aarch64_ad.m4
11445 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11446 instruct regL_not_reg(iRegLNoSp dst,
11447 iRegL src1, immL_M1 m1,
11448 rFlagsReg cr) %{
11449 match(Set dst (XorL src1 m1));
11450 ins_cost(INSN_COST);
11451 format %{ "eon $dst, $src1, zr" %}
11452
11453 ins_encode %{
11454 __ eon(as_Register($dst$$reg),
11455 as_Register($src1$$reg),
11456 zr,
11457 Assembler::LSL, 0);
11458 %}
11459
11460 ins_pipe(ialu_reg);
11461 %}
11462
11463 // This pattern is automatically generated from aarch64_ad.m4
11464 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11465 instruct regI_not_reg(iRegINoSp dst,
11466 iRegIorL2I src1, immI_M1 m1,
11467 rFlagsReg cr) %{
11468 match(Set dst (XorI src1 m1));
11469 ins_cost(INSN_COST);
11470 format %{ "eonw $dst, $src1, zr" %}
11471
11472 ins_encode %{
11473 __ eonw(as_Register($dst$$reg),
11474 as_Register($src1$$reg),
11475 zr,
11476 Assembler::LSL, 0);
11477 %}
11478
11479 ins_pipe(ialu_reg);
11480 %}
11481
11482 // This pattern is automatically generated from aarch64_ad.m4
11483 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11484 instruct NegI_reg_URShift_reg(iRegINoSp dst,
11485 immI0 zero, iRegIorL2I src1, immI src2) %{
11486 match(Set dst (SubI zero (URShiftI src1 src2)));
11487
11488 ins_cost(1.9 * INSN_COST);
11489 format %{ "negw $dst, $src1, LSR $src2" %}
11490
11491 ins_encode %{
11492 __ negw(as_Register($dst$$reg), as_Register($src1$$reg),
11493 Assembler::LSR, $src2$$constant & 0x1f);
11494 %}
11495
11496 ins_pipe(ialu_reg_shift);
11497 %}
11498
11499 // This pattern is automatically generated from aarch64_ad.m4
11500 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11501 instruct NegI_reg_RShift_reg(iRegINoSp dst,
11502 immI0 zero, iRegIorL2I src1, immI src2) %{
11503 match(Set dst (SubI zero (RShiftI src1 src2)));
11504
11505 ins_cost(1.9 * INSN_COST);
11506 format %{ "negw $dst, $src1, ASR $src2" %}
11507
11508 ins_encode %{
11509 __ negw(as_Register($dst$$reg), as_Register($src1$$reg),
11510 Assembler::ASR, $src2$$constant & 0x1f);
11511 %}
11512
11513 ins_pipe(ialu_reg_shift);
11514 %}
11515
11516 // This pattern is automatically generated from aarch64_ad.m4
11517 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11518 instruct NegI_reg_LShift_reg(iRegINoSp dst,
11519 immI0 zero, iRegIorL2I src1, immI src2) %{
11520 match(Set dst (SubI zero (LShiftI src1 src2)));
11521
11522 ins_cost(1.9 * INSN_COST);
11523 format %{ "negw $dst, $src1, LSL $src2" %}
11524
11525 ins_encode %{
11526 __ negw(as_Register($dst$$reg), as_Register($src1$$reg),
11527 Assembler::LSL, $src2$$constant & 0x1f);
11528 %}
11529
11530 ins_pipe(ialu_reg_shift);
11531 %}
11532
11533 // This pattern is automatically generated from aarch64_ad.m4
11534 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11535 instruct NegL_reg_URShift_reg(iRegLNoSp dst,
11536 immL0 zero, iRegL src1, immI src2) %{
11537 match(Set dst (SubL zero (URShiftL src1 src2)));
11538
11539 ins_cost(1.9 * INSN_COST);
11540 format %{ "neg $dst, $src1, LSR $src2" %}
11541
11542 ins_encode %{
11543 __ neg(as_Register($dst$$reg), as_Register($src1$$reg),
11544 Assembler::LSR, $src2$$constant & 0x3f);
11545 %}
11546
11547 ins_pipe(ialu_reg_shift);
11548 %}
11549
11550 // This pattern is automatically generated from aarch64_ad.m4
11551 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11552 instruct NegL_reg_RShift_reg(iRegLNoSp dst,
11553 immL0 zero, iRegL src1, immI src2) %{
11554 match(Set dst (SubL zero (RShiftL src1 src2)));
11555
11556 ins_cost(1.9 * INSN_COST);
11557 format %{ "neg $dst, $src1, ASR $src2" %}
11558
11559 ins_encode %{
11560 __ neg(as_Register($dst$$reg), as_Register($src1$$reg),
11561 Assembler::ASR, $src2$$constant & 0x3f);
11562 %}
11563
11564 ins_pipe(ialu_reg_shift);
11565 %}
11566
11567 // This pattern is automatically generated from aarch64_ad.m4
11568 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11569 instruct NegL_reg_LShift_reg(iRegLNoSp dst,
11570 immL0 zero, iRegL src1, immI src2) %{
11571 match(Set dst (SubL zero (LShiftL src1 src2)));
11572
11573 ins_cost(1.9 * INSN_COST);
11574 format %{ "neg $dst, $src1, LSL $src2" %}
11575
11576 ins_encode %{
11577 __ neg(as_Register($dst$$reg), as_Register($src1$$reg),
11578 Assembler::LSL, $src2$$constant & 0x3f);
11579 %}
11580
11581 ins_pipe(ialu_reg_shift);
11582 %}
11583
11584 // This pattern is automatically generated from aarch64_ad.m4
11585 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11586 instruct AndI_reg_not_reg(iRegINoSp dst,
11587 iRegIorL2I src1, iRegIorL2I src2, immI_M1 m1) %{
11588 match(Set dst (AndI src1 (XorI src2 m1)));
11589 ins_cost(INSN_COST);
11590 format %{ "bicw $dst, $src1, $src2" %}
11591
11592 ins_encode %{
11593 __ bicw(as_Register($dst$$reg),
11594 as_Register($src1$$reg),
11595 as_Register($src2$$reg),
11596 Assembler::LSL, 0);
11597 %}
11598
11599 ins_pipe(ialu_reg_reg);
11600 %}
11601
11602 // This pattern is automatically generated from aarch64_ad.m4
11603 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11604 instruct AndL_reg_not_reg(iRegLNoSp dst,
11605 iRegL src1, iRegL src2, immL_M1 m1) %{
11606 match(Set dst (AndL src1 (XorL src2 m1)));
11607 ins_cost(INSN_COST);
11608 format %{ "bic $dst, $src1, $src2" %}
11609
11610 ins_encode %{
11611 __ bic(as_Register($dst$$reg),
11612 as_Register($src1$$reg),
11613 as_Register($src2$$reg),
11614 Assembler::LSL, 0);
11615 %}
11616
11617 ins_pipe(ialu_reg_reg);
11618 %}
11619
11620 // This pattern is automatically generated from aarch64_ad.m4
11621 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11622 instruct OrI_reg_not_reg(iRegINoSp dst,
11623 iRegIorL2I src1, iRegIorL2I src2, immI_M1 m1) %{
11624 match(Set dst (OrI src1 (XorI src2 m1)));
11625 ins_cost(INSN_COST);
11626 format %{ "ornw $dst, $src1, $src2" %}
11627
11628 ins_encode %{
11629 __ ornw(as_Register($dst$$reg),
11630 as_Register($src1$$reg),
11631 as_Register($src2$$reg),
11632 Assembler::LSL, 0);
11633 %}
11634
11635 ins_pipe(ialu_reg_reg);
11636 %}
11637
11638 // This pattern is automatically generated from aarch64_ad.m4
11639 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11640 instruct OrL_reg_not_reg(iRegLNoSp dst,
11641 iRegL src1, iRegL src2, immL_M1 m1) %{
11642 match(Set dst (OrL src1 (XorL src2 m1)));
11643 ins_cost(INSN_COST);
11644 format %{ "orn $dst, $src1, $src2" %}
11645
11646 ins_encode %{
11647 __ orn(as_Register($dst$$reg),
11648 as_Register($src1$$reg),
11649 as_Register($src2$$reg),
11650 Assembler::LSL, 0);
11651 %}
11652
11653 ins_pipe(ialu_reg_reg);
11654 %}
11655
11656 // This pattern is automatically generated from aarch64_ad.m4
11657 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11658 instruct XorI_reg_not_reg(iRegINoSp dst,
11659 iRegIorL2I src1, iRegIorL2I src2, immI_M1 m1) %{
11660 match(Set dst (XorI m1 (XorI src2 src1)));
11661 ins_cost(INSN_COST);
11662 format %{ "eonw $dst, $src1, $src2" %}
11663
11664 ins_encode %{
11665 __ eonw(as_Register($dst$$reg),
11666 as_Register($src1$$reg),
11667 as_Register($src2$$reg),
11668 Assembler::LSL, 0);
11669 %}
11670
11671 ins_pipe(ialu_reg_reg);
11672 %}
11673
11674 // This pattern is automatically generated from aarch64_ad.m4
11675 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11676 instruct XorL_reg_not_reg(iRegLNoSp dst,
11677 iRegL src1, iRegL src2, immL_M1 m1) %{
11678 match(Set dst (XorL m1 (XorL src2 src1)));
11679 ins_cost(INSN_COST);
11680 format %{ "eon $dst, $src1, $src2" %}
11681
11682 ins_encode %{
11683 __ eon(as_Register($dst$$reg),
11684 as_Register($src1$$reg),
11685 as_Register($src2$$reg),
11686 Assembler::LSL, 0);
11687 %}
11688
11689 ins_pipe(ialu_reg_reg);
11690 %}
11691
11692 // This pattern is automatically generated from aarch64_ad.m4
11693 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11694 // val & (-1 ^ (val >>> shift)) ==> bicw
11695 instruct AndI_reg_URShift_not_reg(iRegINoSp dst,
11696 iRegIorL2I src1, iRegIorL2I src2,
11697 immI src3, immI_M1 src4) %{
11698 match(Set dst (AndI src1 (XorI(URShiftI src2 src3) src4)));
11699 ins_cost(1.9 * INSN_COST);
11700 format %{ "bicw $dst, $src1, $src2, LSR $src3" %}
11701
11702 ins_encode %{
11703 __ bicw(as_Register($dst$$reg),
11704 as_Register($src1$$reg),
11705 as_Register($src2$$reg),
11706 Assembler::LSR,
11707 $src3$$constant & 0x1f);
11708 %}
11709
11710 ins_pipe(ialu_reg_reg_shift);
11711 %}
11712
11713 // This pattern is automatically generated from aarch64_ad.m4
11714 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11715 // val & (-1 ^ (val >>> shift)) ==> bic
11716 instruct AndL_reg_URShift_not_reg(iRegLNoSp dst,
11717 iRegL src1, iRegL src2,
11718 immI src3, immL_M1 src4) %{
11719 match(Set dst (AndL src1 (XorL(URShiftL src2 src3) src4)));
11720 ins_cost(1.9 * INSN_COST);
11721 format %{ "bic $dst, $src1, $src2, LSR $src3" %}
11722
11723 ins_encode %{
11724 __ bic(as_Register($dst$$reg),
11725 as_Register($src1$$reg),
11726 as_Register($src2$$reg),
11727 Assembler::LSR,
11728 $src3$$constant & 0x3f);
11729 %}
11730
11731 ins_pipe(ialu_reg_reg_shift);
11732 %}
11733
11734 // This pattern is automatically generated from aarch64_ad.m4
11735 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11736 // val & (-1 ^ (val >> shift)) ==> bicw
11737 instruct AndI_reg_RShift_not_reg(iRegINoSp dst,
11738 iRegIorL2I src1, iRegIorL2I src2,
11739 immI src3, immI_M1 src4) %{
11740 match(Set dst (AndI src1 (XorI(RShiftI src2 src3) src4)));
11741 ins_cost(1.9 * INSN_COST);
11742 format %{ "bicw $dst, $src1, $src2, ASR $src3" %}
11743
11744 ins_encode %{
11745 __ bicw(as_Register($dst$$reg),
11746 as_Register($src1$$reg),
11747 as_Register($src2$$reg),
11748 Assembler::ASR,
11749 $src3$$constant & 0x1f);
11750 %}
11751
11752 ins_pipe(ialu_reg_reg_shift);
11753 %}
11754
11755 // This pattern is automatically generated from aarch64_ad.m4
11756 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11757 // val & (-1 ^ (val >> shift)) ==> bic
11758 instruct AndL_reg_RShift_not_reg(iRegLNoSp dst,
11759 iRegL src1, iRegL src2,
11760 immI src3, immL_M1 src4) %{
11761 match(Set dst (AndL src1 (XorL(RShiftL src2 src3) src4)));
11762 ins_cost(1.9 * INSN_COST);
11763 format %{ "bic $dst, $src1, $src2, ASR $src3" %}
11764
11765 ins_encode %{
11766 __ bic(as_Register($dst$$reg),
11767 as_Register($src1$$reg),
11768 as_Register($src2$$reg),
11769 Assembler::ASR,
11770 $src3$$constant & 0x3f);
11771 %}
11772
11773 ins_pipe(ialu_reg_reg_shift);
11774 %}
11775
11776 // This pattern is automatically generated from aarch64_ad.m4
11777 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11778 // val & (-1 ^ (val ror shift)) ==> bicw
11779 instruct AndI_reg_RotateRight_not_reg(iRegINoSp dst,
11780 iRegIorL2I src1, iRegIorL2I src2,
11781 immI src3, immI_M1 src4) %{
11782 match(Set dst (AndI src1 (XorI(RotateRight src2 src3) src4)));
11783 ins_cost(1.9 * INSN_COST);
11784 format %{ "bicw $dst, $src1, $src2, ROR $src3" %}
11785
11786 ins_encode %{
11787 __ bicw(as_Register($dst$$reg),
11788 as_Register($src1$$reg),
11789 as_Register($src2$$reg),
11790 Assembler::ROR,
11791 $src3$$constant & 0x1f);
11792 %}
11793
11794 ins_pipe(ialu_reg_reg_shift);
11795 %}
11796
11797 // This pattern is automatically generated from aarch64_ad.m4
11798 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11799 // val & (-1 ^ (val ror shift)) ==> bic
11800 instruct AndL_reg_RotateRight_not_reg(iRegLNoSp dst,
11801 iRegL src1, iRegL src2,
11802 immI src3, immL_M1 src4) %{
11803 match(Set dst (AndL src1 (XorL(RotateRight src2 src3) src4)));
11804 ins_cost(1.9 * INSN_COST);
11805 format %{ "bic $dst, $src1, $src2, ROR $src3" %}
11806
11807 ins_encode %{
11808 __ bic(as_Register($dst$$reg),
11809 as_Register($src1$$reg),
11810 as_Register($src2$$reg),
11811 Assembler::ROR,
11812 $src3$$constant & 0x3f);
11813 %}
11814
11815 ins_pipe(ialu_reg_reg_shift);
11816 %}
11817
11818 // This pattern is automatically generated from aarch64_ad.m4
11819 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11820 // val & (-1 ^ (val << shift)) ==> bicw
11821 instruct AndI_reg_LShift_not_reg(iRegINoSp dst,
11822 iRegIorL2I src1, iRegIorL2I src2,
11823 immI src3, immI_M1 src4) %{
11824 match(Set dst (AndI src1 (XorI(LShiftI src2 src3) src4)));
11825 ins_cost(1.9 * INSN_COST);
11826 format %{ "bicw $dst, $src1, $src2, LSL $src3" %}
11827
11828 ins_encode %{
11829 __ bicw(as_Register($dst$$reg),
11830 as_Register($src1$$reg),
11831 as_Register($src2$$reg),
11832 Assembler::LSL,
11833 $src3$$constant & 0x1f);
11834 %}
11835
11836 ins_pipe(ialu_reg_reg_shift);
11837 %}
11838
11839 // This pattern is automatically generated from aarch64_ad.m4
11840 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11841 // val & (-1 ^ (val << shift)) ==> bic
11842 instruct AndL_reg_LShift_not_reg(iRegLNoSp dst,
11843 iRegL src1, iRegL src2,
11844 immI src3, immL_M1 src4) %{
11845 match(Set dst (AndL src1 (XorL(LShiftL src2 src3) src4)));
11846 ins_cost(1.9 * INSN_COST);
11847 format %{ "bic $dst, $src1, $src2, LSL $src3" %}
11848
11849 ins_encode %{
11850 __ bic(as_Register($dst$$reg),
11851 as_Register($src1$$reg),
11852 as_Register($src2$$reg),
11853 Assembler::LSL,
11854 $src3$$constant & 0x3f);
11855 %}
11856
11857 ins_pipe(ialu_reg_reg_shift);
11858 %}
11859
11860 // This pattern is automatically generated from aarch64_ad.m4
11861 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11862 // val ^ (-1 ^ (val >>> shift)) ==> eonw
11863 instruct XorI_reg_URShift_not_reg(iRegINoSp dst,
11864 iRegIorL2I src1, iRegIorL2I src2,
11865 immI src3, immI_M1 src4) %{
11866 match(Set dst (XorI src4 (XorI(URShiftI src2 src3) src1)));
11867 ins_cost(1.9 * INSN_COST);
11868 format %{ "eonw $dst, $src1, $src2, LSR $src3" %}
11869
11870 ins_encode %{
11871 __ eonw(as_Register($dst$$reg),
11872 as_Register($src1$$reg),
11873 as_Register($src2$$reg),
11874 Assembler::LSR,
11875 $src3$$constant & 0x1f);
11876 %}
11877
11878 ins_pipe(ialu_reg_reg_shift);
11879 %}
11880
11881 // This pattern is automatically generated from aarch64_ad.m4
11882 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11883 // val ^ (-1 ^ (val >>> shift)) ==> eon
11884 instruct XorL_reg_URShift_not_reg(iRegLNoSp dst,
11885 iRegL src1, iRegL src2,
11886 immI src3, immL_M1 src4) %{
11887 match(Set dst (XorL src4 (XorL(URShiftL src2 src3) src1)));
11888 ins_cost(1.9 * INSN_COST);
11889 format %{ "eon $dst, $src1, $src2, LSR $src3" %}
11890
11891 ins_encode %{
11892 __ eon(as_Register($dst$$reg),
11893 as_Register($src1$$reg),
11894 as_Register($src2$$reg),
11895 Assembler::LSR,
11896 $src3$$constant & 0x3f);
11897 %}
11898
11899 ins_pipe(ialu_reg_reg_shift);
11900 %}
11901
11902 // This pattern is automatically generated from aarch64_ad.m4
11903 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11904 // val ^ (-1 ^ (val >> shift)) ==> eonw
11905 instruct XorI_reg_RShift_not_reg(iRegINoSp dst,
11906 iRegIorL2I src1, iRegIorL2I src2,
11907 immI src3, immI_M1 src4) %{
11908 match(Set dst (XorI src4 (XorI(RShiftI src2 src3) src1)));
11909 ins_cost(1.9 * INSN_COST);
11910 format %{ "eonw $dst, $src1, $src2, ASR $src3" %}
11911
11912 ins_encode %{
11913 __ eonw(as_Register($dst$$reg),
11914 as_Register($src1$$reg),
11915 as_Register($src2$$reg),
11916 Assembler::ASR,
11917 $src3$$constant & 0x1f);
11918 %}
11919
11920 ins_pipe(ialu_reg_reg_shift);
11921 %}
11922
11923 // This pattern is automatically generated from aarch64_ad.m4
11924 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11925 // val ^ (-1 ^ (val >> shift)) ==> eon
11926 instruct XorL_reg_RShift_not_reg(iRegLNoSp dst,
11927 iRegL src1, iRegL src2,
11928 immI src3, immL_M1 src4) %{
11929 match(Set dst (XorL src4 (XorL(RShiftL src2 src3) src1)));
11930 ins_cost(1.9 * INSN_COST);
11931 format %{ "eon $dst, $src1, $src2, ASR $src3" %}
11932
11933 ins_encode %{
11934 __ eon(as_Register($dst$$reg),
11935 as_Register($src1$$reg),
11936 as_Register($src2$$reg),
11937 Assembler::ASR,
11938 $src3$$constant & 0x3f);
11939 %}
11940
11941 ins_pipe(ialu_reg_reg_shift);
11942 %}
11943
11944 // This pattern is automatically generated from aarch64_ad.m4
11945 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11946 // val ^ (-1 ^ (val ror shift)) ==> eonw
11947 instruct XorI_reg_RotateRight_not_reg(iRegINoSp dst,
11948 iRegIorL2I src1, iRegIorL2I src2,
11949 immI src3, immI_M1 src4) %{
11950 match(Set dst (XorI src4 (XorI(RotateRight src2 src3) src1)));
11951 ins_cost(1.9 * INSN_COST);
11952 format %{ "eonw $dst, $src1, $src2, ROR $src3" %}
11953
11954 ins_encode %{
11955 __ eonw(as_Register($dst$$reg),
11956 as_Register($src1$$reg),
11957 as_Register($src2$$reg),
11958 Assembler::ROR,
11959 $src3$$constant & 0x1f);
11960 %}
11961
11962 ins_pipe(ialu_reg_reg_shift);
11963 %}
11964
11965 // This pattern is automatically generated from aarch64_ad.m4
11966 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11967 // val ^ (-1 ^ (val ror shift)) ==> eon
11968 instruct XorL_reg_RotateRight_not_reg(iRegLNoSp dst,
11969 iRegL src1, iRegL src2,
11970 immI src3, immL_M1 src4) %{
11971 match(Set dst (XorL src4 (XorL(RotateRight src2 src3) src1)));
11972 ins_cost(1.9 * INSN_COST);
11973 format %{ "eon $dst, $src1, $src2, ROR $src3" %}
11974
11975 ins_encode %{
11976 __ eon(as_Register($dst$$reg),
11977 as_Register($src1$$reg),
11978 as_Register($src2$$reg),
11979 Assembler::ROR,
11980 $src3$$constant & 0x3f);
11981 %}
11982
11983 ins_pipe(ialu_reg_reg_shift);
11984 %}
11985
11986 // This pattern is automatically generated from aarch64_ad.m4
11987 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11988 // val ^ (-1 ^ (val << shift)) ==> eonw
11989 instruct XorI_reg_LShift_not_reg(iRegINoSp dst,
11990 iRegIorL2I src1, iRegIorL2I src2,
11991 immI src3, immI_M1 src4) %{
11992 match(Set dst (XorI src4 (XorI(LShiftI src2 src3) src1)));
11993 ins_cost(1.9 * INSN_COST);
11994 format %{ "eonw $dst, $src1, $src2, LSL $src3" %}
11995
11996 ins_encode %{
11997 __ eonw(as_Register($dst$$reg),
11998 as_Register($src1$$reg),
11999 as_Register($src2$$reg),
12000 Assembler::LSL,
12001 $src3$$constant & 0x1f);
12002 %}
12003
12004 ins_pipe(ialu_reg_reg_shift);
12005 %}
12006
12007 // This pattern is automatically generated from aarch64_ad.m4
12008 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12009 // val ^ (-1 ^ (val << shift)) ==> eon
12010 instruct XorL_reg_LShift_not_reg(iRegLNoSp dst,
12011 iRegL src1, iRegL src2,
12012 immI src3, immL_M1 src4) %{
12013 match(Set dst (XorL src4 (XorL(LShiftL src2 src3) src1)));
12014 ins_cost(1.9 * INSN_COST);
12015 format %{ "eon $dst, $src1, $src2, LSL $src3" %}
12016
12017 ins_encode %{
12018 __ eon(as_Register($dst$$reg),
12019 as_Register($src1$$reg),
12020 as_Register($src2$$reg),
12021 Assembler::LSL,
12022 $src3$$constant & 0x3f);
12023 %}
12024
12025 ins_pipe(ialu_reg_reg_shift);
12026 %}
12027
12028 // This pattern is automatically generated from aarch64_ad.m4
12029 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12030 // val | (-1 ^ (val >>> shift)) ==> ornw
12031 instruct OrI_reg_URShift_not_reg(iRegINoSp dst,
12032 iRegIorL2I src1, iRegIorL2I src2,
12033 immI src3, immI_M1 src4) %{
12034 match(Set dst (OrI src1 (XorI(URShiftI src2 src3) src4)));
12035 ins_cost(1.9 * INSN_COST);
12036 format %{ "ornw $dst, $src1, $src2, LSR $src3" %}
12037
12038 ins_encode %{
12039 __ ornw(as_Register($dst$$reg),
12040 as_Register($src1$$reg),
12041 as_Register($src2$$reg),
12042 Assembler::LSR,
12043 $src3$$constant & 0x1f);
12044 %}
12045
12046 ins_pipe(ialu_reg_reg_shift);
12047 %}
12048
12049 // This pattern is automatically generated from aarch64_ad.m4
12050 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12051 // val | (-1 ^ (val >>> shift)) ==> orn
12052 instruct OrL_reg_URShift_not_reg(iRegLNoSp dst,
12053 iRegL src1, iRegL src2,
12054 immI src3, immL_M1 src4) %{
12055 match(Set dst (OrL src1 (XorL(URShiftL src2 src3) src4)));
12056 ins_cost(1.9 * INSN_COST);
12057 format %{ "orn $dst, $src1, $src2, LSR $src3" %}
12058
12059 ins_encode %{
12060 __ orn(as_Register($dst$$reg),
12061 as_Register($src1$$reg),
12062 as_Register($src2$$reg),
12063 Assembler::LSR,
12064 $src3$$constant & 0x3f);
12065 %}
12066
12067 ins_pipe(ialu_reg_reg_shift);
12068 %}
12069
12070 // This pattern is automatically generated from aarch64_ad.m4
12071 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12072 // val | (-1 ^ (val >> shift)) ==> ornw
12073 instruct OrI_reg_RShift_not_reg(iRegINoSp dst,
12074 iRegIorL2I src1, iRegIorL2I src2,
12075 immI src3, immI_M1 src4) %{
12076 match(Set dst (OrI src1 (XorI(RShiftI src2 src3) src4)));
12077 ins_cost(1.9 * INSN_COST);
12078 format %{ "ornw $dst, $src1, $src2, ASR $src3" %}
12079
12080 ins_encode %{
12081 __ ornw(as_Register($dst$$reg),
12082 as_Register($src1$$reg),
12083 as_Register($src2$$reg),
12084 Assembler::ASR,
12085 $src3$$constant & 0x1f);
12086 %}
12087
12088 ins_pipe(ialu_reg_reg_shift);
12089 %}
12090
12091 // This pattern is automatically generated from aarch64_ad.m4
12092 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12093 // val | (-1 ^ (val >> shift)) ==> orn
12094 instruct OrL_reg_RShift_not_reg(iRegLNoSp dst,
12095 iRegL src1, iRegL src2,
12096 immI src3, immL_M1 src4) %{
12097 match(Set dst (OrL src1 (XorL(RShiftL src2 src3) src4)));
12098 ins_cost(1.9 * INSN_COST);
12099 format %{ "orn $dst, $src1, $src2, ASR $src3" %}
12100
12101 ins_encode %{
12102 __ orn(as_Register($dst$$reg),
12103 as_Register($src1$$reg),
12104 as_Register($src2$$reg),
12105 Assembler::ASR,
12106 $src3$$constant & 0x3f);
12107 %}
12108
12109 ins_pipe(ialu_reg_reg_shift);
12110 %}
12111
12112 // This pattern is automatically generated from aarch64_ad.m4
12113 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12114 // val | (-1 ^ (val ror shift)) ==> ornw
12115 instruct OrI_reg_RotateRight_not_reg(iRegINoSp dst,
12116 iRegIorL2I src1, iRegIorL2I src2,
12117 immI src3, immI_M1 src4) %{
12118 match(Set dst (OrI src1 (XorI(RotateRight src2 src3) src4)));
12119 ins_cost(1.9 * INSN_COST);
12120 format %{ "ornw $dst, $src1, $src2, ROR $src3" %}
12121
12122 ins_encode %{
12123 __ ornw(as_Register($dst$$reg),
12124 as_Register($src1$$reg),
12125 as_Register($src2$$reg),
12126 Assembler::ROR,
12127 $src3$$constant & 0x1f);
12128 %}
12129
12130 ins_pipe(ialu_reg_reg_shift);
12131 %}
12132
12133 // This pattern is automatically generated from aarch64_ad.m4
12134 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12135 // val | (-1 ^ (val ror shift)) ==> orn
12136 instruct OrL_reg_RotateRight_not_reg(iRegLNoSp dst,
12137 iRegL src1, iRegL src2,
12138 immI src3, immL_M1 src4) %{
12139 match(Set dst (OrL src1 (XorL(RotateRight src2 src3) src4)));
12140 ins_cost(1.9 * INSN_COST);
12141 format %{ "orn $dst, $src1, $src2, ROR $src3" %}
12142
12143 ins_encode %{
12144 __ orn(as_Register($dst$$reg),
12145 as_Register($src1$$reg),
12146 as_Register($src2$$reg),
12147 Assembler::ROR,
12148 $src3$$constant & 0x3f);
12149 %}
12150
12151 ins_pipe(ialu_reg_reg_shift);
12152 %}
12153
12154 // This pattern is automatically generated from aarch64_ad.m4
12155 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12156 // val | (-1 ^ (val << shift)) ==> ornw
12157 instruct OrI_reg_LShift_not_reg(iRegINoSp dst,
12158 iRegIorL2I src1, iRegIorL2I src2,
12159 immI src3, immI_M1 src4) %{
12160 match(Set dst (OrI src1 (XorI(LShiftI src2 src3) src4)));
12161 ins_cost(1.9 * INSN_COST);
12162 format %{ "ornw $dst, $src1, $src2, LSL $src3" %}
12163
12164 ins_encode %{
12165 __ ornw(as_Register($dst$$reg),
12166 as_Register($src1$$reg),
12167 as_Register($src2$$reg),
12168 Assembler::LSL,
12169 $src3$$constant & 0x1f);
12170 %}
12171
12172 ins_pipe(ialu_reg_reg_shift);
12173 %}
12174
12175 // This pattern is automatically generated from aarch64_ad.m4
12176 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12177 // val | (-1 ^ (val << shift)) ==> orn
12178 instruct OrL_reg_LShift_not_reg(iRegLNoSp dst,
12179 iRegL src1, iRegL src2,
12180 immI src3, immL_M1 src4) %{
12181 match(Set dst (OrL src1 (XorL(LShiftL src2 src3) src4)));
12182 ins_cost(1.9 * INSN_COST);
12183 format %{ "orn $dst, $src1, $src2, LSL $src3" %}
12184
12185 ins_encode %{
12186 __ orn(as_Register($dst$$reg),
12187 as_Register($src1$$reg),
12188 as_Register($src2$$reg),
12189 Assembler::LSL,
12190 $src3$$constant & 0x3f);
12191 %}
12192
12193 ins_pipe(ialu_reg_reg_shift);
12194 %}
12195
12196 // This pattern is automatically generated from aarch64_ad.m4
12197 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12198 instruct AndI_reg_URShift_reg(iRegINoSp dst,
12199 iRegIorL2I src1, iRegIorL2I src2,
12200 immI src3) %{
12201 match(Set dst (AndI src1 (URShiftI src2 src3)));
12202
12203 ins_cost(1.9 * INSN_COST);
12204 format %{ "andw $dst, $src1, $src2, LSR $src3" %}
12205
12206 ins_encode %{
12207 __ andw(as_Register($dst$$reg),
12208 as_Register($src1$$reg),
12209 as_Register($src2$$reg),
12210 Assembler::LSR,
12211 $src3$$constant & 0x1f);
12212 %}
12213
12214 ins_pipe(ialu_reg_reg_shift);
12215 %}
12216
12217 // This pattern is automatically generated from aarch64_ad.m4
12218 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12219 instruct AndL_reg_URShift_reg(iRegLNoSp dst,
12220 iRegL src1, iRegL src2,
12221 immI src3) %{
12222 match(Set dst (AndL src1 (URShiftL src2 src3)));
12223
12224 ins_cost(1.9 * INSN_COST);
12225 format %{ "andr $dst, $src1, $src2, LSR $src3" %}
12226
12227 ins_encode %{
12228 __ andr(as_Register($dst$$reg),
12229 as_Register($src1$$reg),
12230 as_Register($src2$$reg),
12231 Assembler::LSR,
12232 $src3$$constant & 0x3f);
12233 %}
12234
12235 ins_pipe(ialu_reg_reg_shift);
12236 %}
12237
12238 // This pattern is automatically generated from aarch64_ad.m4
12239 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12240 instruct AndI_reg_RShift_reg(iRegINoSp dst,
12241 iRegIorL2I src1, iRegIorL2I src2,
12242 immI src3) %{
12243 match(Set dst (AndI src1 (RShiftI src2 src3)));
12244
12245 ins_cost(1.9 * INSN_COST);
12246 format %{ "andw $dst, $src1, $src2, ASR $src3" %}
12247
12248 ins_encode %{
12249 __ andw(as_Register($dst$$reg),
12250 as_Register($src1$$reg),
12251 as_Register($src2$$reg),
12252 Assembler::ASR,
12253 $src3$$constant & 0x1f);
12254 %}
12255
12256 ins_pipe(ialu_reg_reg_shift);
12257 %}
12258
12259 // This pattern is automatically generated from aarch64_ad.m4
12260 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12261 instruct AndL_reg_RShift_reg(iRegLNoSp dst,
12262 iRegL src1, iRegL src2,
12263 immI src3) %{
12264 match(Set dst (AndL src1 (RShiftL src2 src3)));
12265
12266 ins_cost(1.9 * INSN_COST);
12267 format %{ "andr $dst, $src1, $src2, ASR $src3" %}
12268
12269 ins_encode %{
12270 __ andr(as_Register($dst$$reg),
12271 as_Register($src1$$reg),
12272 as_Register($src2$$reg),
12273 Assembler::ASR,
12274 $src3$$constant & 0x3f);
12275 %}
12276
12277 ins_pipe(ialu_reg_reg_shift);
12278 %}
12279
12280 // This pattern is automatically generated from aarch64_ad.m4
12281 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12282 instruct AndI_reg_LShift_reg(iRegINoSp dst,
12283 iRegIorL2I src1, iRegIorL2I src2,
12284 immI src3) %{
12285 match(Set dst (AndI src1 (LShiftI src2 src3)));
12286
12287 ins_cost(1.9 * INSN_COST);
12288 format %{ "andw $dst, $src1, $src2, LSL $src3" %}
12289
12290 ins_encode %{
12291 __ andw(as_Register($dst$$reg),
12292 as_Register($src1$$reg),
12293 as_Register($src2$$reg),
12294 Assembler::LSL,
12295 $src3$$constant & 0x1f);
12296 %}
12297
12298 ins_pipe(ialu_reg_reg_shift);
12299 %}
12300
12301 // This pattern is automatically generated from aarch64_ad.m4
12302 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12303 instruct AndL_reg_LShift_reg(iRegLNoSp dst,
12304 iRegL src1, iRegL src2,
12305 immI src3) %{
12306 match(Set dst (AndL src1 (LShiftL src2 src3)));
12307
12308 ins_cost(1.9 * INSN_COST);
12309 format %{ "andr $dst, $src1, $src2, LSL $src3" %}
12310
12311 ins_encode %{
12312 __ andr(as_Register($dst$$reg),
12313 as_Register($src1$$reg),
12314 as_Register($src2$$reg),
12315 Assembler::LSL,
12316 $src3$$constant & 0x3f);
12317 %}
12318
12319 ins_pipe(ialu_reg_reg_shift);
12320 %}
12321
12322 // This pattern is automatically generated from aarch64_ad.m4
12323 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12324 instruct AndI_reg_RotateRight_reg(iRegINoSp dst,
12325 iRegIorL2I src1, iRegIorL2I src2,
12326 immI src3) %{
12327 match(Set dst (AndI src1 (RotateRight src2 src3)));
12328
12329 ins_cost(1.9 * INSN_COST);
12330 format %{ "andw $dst, $src1, $src2, ROR $src3" %}
12331
12332 ins_encode %{
12333 __ andw(as_Register($dst$$reg),
12334 as_Register($src1$$reg),
12335 as_Register($src2$$reg),
12336 Assembler::ROR,
12337 $src3$$constant & 0x1f);
12338 %}
12339
12340 ins_pipe(ialu_reg_reg_shift);
12341 %}
12342
12343 // This pattern is automatically generated from aarch64_ad.m4
12344 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12345 instruct AndL_reg_RotateRight_reg(iRegLNoSp dst,
12346 iRegL src1, iRegL src2,
12347 immI src3) %{
12348 match(Set dst (AndL src1 (RotateRight src2 src3)));
12349
12350 ins_cost(1.9 * INSN_COST);
12351 format %{ "andr $dst, $src1, $src2, ROR $src3" %}
12352
12353 ins_encode %{
12354 __ andr(as_Register($dst$$reg),
12355 as_Register($src1$$reg),
12356 as_Register($src2$$reg),
12357 Assembler::ROR,
12358 $src3$$constant & 0x3f);
12359 %}
12360
12361 ins_pipe(ialu_reg_reg_shift);
12362 %}
12363
12364 // This pattern is automatically generated from aarch64_ad.m4
12365 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12366 instruct XorI_reg_URShift_reg(iRegINoSp dst,
12367 iRegIorL2I src1, iRegIorL2I src2,
12368 immI src3) %{
12369 match(Set dst (XorI src1 (URShiftI src2 src3)));
12370
12371 ins_cost(1.9 * INSN_COST);
12372 format %{ "eorw $dst, $src1, $src2, LSR $src3" %}
12373
12374 ins_encode %{
12375 __ eorw(as_Register($dst$$reg),
12376 as_Register($src1$$reg),
12377 as_Register($src2$$reg),
12378 Assembler::LSR,
12379 $src3$$constant & 0x1f);
12380 %}
12381
12382 ins_pipe(ialu_reg_reg_shift);
12383 %}
12384
12385 // This pattern is automatically generated from aarch64_ad.m4
12386 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12387 instruct XorL_reg_URShift_reg(iRegLNoSp dst,
12388 iRegL src1, iRegL src2,
12389 immI src3) %{
12390 match(Set dst (XorL src1 (URShiftL src2 src3)));
12391
12392 ins_cost(1.9 * INSN_COST);
12393 format %{ "eor $dst, $src1, $src2, LSR $src3" %}
12394
12395 ins_encode %{
12396 __ eor(as_Register($dst$$reg),
12397 as_Register($src1$$reg),
12398 as_Register($src2$$reg),
12399 Assembler::LSR,
12400 $src3$$constant & 0x3f);
12401 %}
12402
12403 ins_pipe(ialu_reg_reg_shift);
12404 %}
12405
12406 // This pattern is automatically generated from aarch64_ad.m4
12407 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12408 instruct XorI_reg_RShift_reg(iRegINoSp dst,
12409 iRegIorL2I src1, iRegIorL2I src2,
12410 immI src3) %{
12411 match(Set dst (XorI src1 (RShiftI src2 src3)));
12412
12413 ins_cost(1.9 * INSN_COST);
12414 format %{ "eorw $dst, $src1, $src2, ASR $src3" %}
12415
12416 ins_encode %{
12417 __ eorw(as_Register($dst$$reg),
12418 as_Register($src1$$reg),
12419 as_Register($src2$$reg),
12420 Assembler::ASR,
12421 $src3$$constant & 0x1f);
12422 %}
12423
12424 ins_pipe(ialu_reg_reg_shift);
12425 %}
12426
12427 // This pattern is automatically generated from aarch64_ad.m4
12428 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12429 instruct XorL_reg_RShift_reg(iRegLNoSp dst,
12430 iRegL src1, iRegL src2,
12431 immI src3) %{
12432 match(Set dst (XorL src1 (RShiftL src2 src3)));
12433
12434 ins_cost(1.9 * INSN_COST);
12435 format %{ "eor $dst, $src1, $src2, ASR $src3" %}
12436
12437 ins_encode %{
12438 __ eor(as_Register($dst$$reg),
12439 as_Register($src1$$reg),
12440 as_Register($src2$$reg),
12441 Assembler::ASR,
12442 $src3$$constant & 0x3f);
12443 %}
12444
12445 ins_pipe(ialu_reg_reg_shift);
12446 %}
12447
12448 // This pattern is automatically generated from aarch64_ad.m4
12449 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12450 instruct XorI_reg_LShift_reg(iRegINoSp dst,
12451 iRegIorL2I src1, iRegIorL2I src2,
12452 immI src3) %{
12453 match(Set dst (XorI src1 (LShiftI src2 src3)));
12454
12455 ins_cost(1.9 * INSN_COST);
12456 format %{ "eorw $dst, $src1, $src2, LSL $src3" %}
12457
12458 ins_encode %{
12459 __ eorw(as_Register($dst$$reg),
12460 as_Register($src1$$reg),
12461 as_Register($src2$$reg),
12462 Assembler::LSL,
12463 $src3$$constant & 0x1f);
12464 %}
12465
12466 ins_pipe(ialu_reg_reg_shift);
12467 %}
12468
12469 // This pattern is automatically generated from aarch64_ad.m4
12470 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12471 instruct XorL_reg_LShift_reg(iRegLNoSp dst,
12472 iRegL src1, iRegL src2,
12473 immI src3) %{
12474 match(Set dst (XorL src1 (LShiftL src2 src3)));
12475
12476 ins_cost(1.9 * INSN_COST);
12477 format %{ "eor $dst, $src1, $src2, LSL $src3" %}
12478
12479 ins_encode %{
12480 __ eor(as_Register($dst$$reg),
12481 as_Register($src1$$reg),
12482 as_Register($src2$$reg),
12483 Assembler::LSL,
12484 $src3$$constant & 0x3f);
12485 %}
12486
12487 ins_pipe(ialu_reg_reg_shift);
12488 %}
12489
12490 // This pattern is automatically generated from aarch64_ad.m4
12491 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12492 instruct XorI_reg_RotateRight_reg(iRegINoSp dst,
12493 iRegIorL2I src1, iRegIorL2I src2,
12494 immI src3) %{
12495 match(Set dst (XorI src1 (RotateRight src2 src3)));
12496
12497 ins_cost(1.9 * INSN_COST);
12498 format %{ "eorw $dst, $src1, $src2, ROR $src3" %}
12499
12500 ins_encode %{
12501 __ eorw(as_Register($dst$$reg),
12502 as_Register($src1$$reg),
12503 as_Register($src2$$reg),
12504 Assembler::ROR,
12505 $src3$$constant & 0x1f);
12506 %}
12507
12508 ins_pipe(ialu_reg_reg_shift);
12509 %}
12510
12511 // This pattern is automatically generated from aarch64_ad.m4
12512 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12513 instruct XorL_reg_RotateRight_reg(iRegLNoSp dst,
12514 iRegL src1, iRegL src2,
12515 immI src3) %{
12516 match(Set dst (XorL src1 (RotateRight src2 src3)));
12517
12518 ins_cost(1.9 * INSN_COST);
12519 format %{ "eor $dst, $src1, $src2, ROR $src3" %}
12520
12521 ins_encode %{
12522 __ eor(as_Register($dst$$reg),
12523 as_Register($src1$$reg),
12524 as_Register($src2$$reg),
12525 Assembler::ROR,
12526 $src3$$constant & 0x3f);
12527 %}
12528
12529 ins_pipe(ialu_reg_reg_shift);
12530 %}
12531
12532 // This pattern is automatically generated from aarch64_ad.m4
12533 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12534 instruct OrI_reg_URShift_reg(iRegINoSp dst,
12535 iRegIorL2I src1, iRegIorL2I src2,
12536 immI src3) %{
12537 match(Set dst (OrI src1 (URShiftI src2 src3)));
12538
12539 ins_cost(1.9 * INSN_COST);
12540 format %{ "orrw $dst, $src1, $src2, LSR $src3" %}
12541
12542 ins_encode %{
12543 __ orrw(as_Register($dst$$reg),
12544 as_Register($src1$$reg),
12545 as_Register($src2$$reg),
12546 Assembler::LSR,
12547 $src3$$constant & 0x1f);
12548 %}
12549
12550 ins_pipe(ialu_reg_reg_shift);
12551 %}
12552
12553 // This pattern is automatically generated from aarch64_ad.m4
12554 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12555 instruct OrL_reg_URShift_reg(iRegLNoSp dst,
12556 iRegL src1, iRegL src2,
12557 immI src3) %{
12558 match(Set dst (OrL src1 (URShiftL src2 src3)));
12559
12560 ins_cost(1.9 * INSN_COST);
12561 format %{ "orr $dst, $src1, $src2, LSR $src3" %}
12562
12563 ins_encode %{
12564 __ orr(as_Register($dst$$reg),
12565 as_Register($src1$$reg),
12566 as_Register($src2$$reg),
12567 Assembler::LSR,
12568 $src3$$constant & 0x3f);
12569 %}
12570
12571 ins_pipe(ialu_reg_reg_shift);
12572 %}
12573
12574 // This pattern is automatically generated from aarch64_ad.m4
12575 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12576 instruct OrI_reg_RShift_reg(iRegINoSp dst,
12577 iRegIorL2I src1, iRegIorL2I src2,
12578 immI src3) %{
12579 match(Set dst (OrI src1 (RShiftI src2 src3)));
12580
12581 ins_cost(1.9 * INSN_COST);
12582 format %{ "orrw $dst, $src1, $src2, ASR $src3" %}
12583
12584 ins_encode %{
12585 __ orrw(as_Register($dst$$reg),
12586 as_Register($src1$$reg),
12587 as_Register($src2$$reg),
12588 Assembler::ASR,
12589 $src3$$constant & 0x1f);
12590 %}
12591
12592 ins_pipe(ialu_reg_reg_shift);
12593 %}
12594
12595 // This pattern is automatically generated from aarch64_ad.m4
12596 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12597 instruct OrL_reg_RShift_reg(iRegLNoSp dst,
12598 iRegL src1, iRegL src2,
12599 immI src3) %{
12600 match(Set dst (OrL src1 (RShiftL src2 src3)));
12601
12602 ins_cost(1.9 * INSN_COST);
12603 format %{ "orr $dst, $src1, $src2, ASR $src3" %}
12604
12605 ins_encode %{
12606 __ orr(as_Register($dst$$reg),
12607 as_Register($src1$$reg),
12608 as_Register($src2$$reg),
12609 Assembler::ASR,
12610 $src3$$constant & 0x3f);
12611 %}
12612
12613 ins_pipe(ialu_reg_reg_shift);
12614 %}
12615
12616 // This pattern is automatically generated from aarch64_ad.m4
12617 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12618 instruct OrI_reg_LShift_reg(iRegINoSp dst,
12619 iRegIorL2I src1, iRegIorL2I src2,
12620 immI src3) %{
12621 match(Set dst (OrI src1 (LShiftI src2 src3)));
12622
12623 ins_cost(1.9 * INSN_COST);
12624 format %{ "orrw $dst, $src1, $src2, LSL $src3" %}
12625
12626 ins_encode %{
12627 __ orrw(as_Register($dst$$reg),
12628 as_Register($src1$$reg),
12629 as_Register($src2$$reg),
12630 Assembler::LSL,
12631 $src3$$constant & 0x1f);
12632 %}
12633
12634 ins_pipe(ialu_reg_reg_shift);
12635 %}
12636
12637 // This pattern is automatically generated from aarch64_ad.m4
12638 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12639 instruct OrL_reg_LShift_reg(iRegLNoSp dst,
12640 iRegL src1, iRegL src2,
12641 immI src3) %{
12642 match(Set dst (OrL src1 (LShiftL src2 src3)));
12643
12644 ins_cost(1.9 * INSN_COST);
12645 format %{ "orr $dst, $src1, $src2, LSL $src3" %}
12646
12647 ins_encode %{
12648 __ orr(as_Register($dst$$reg),
12649 as_Register($src1$$reg),
12650 as_Register($src2$$reg),
12651 Assembler::LSL,
12652 $src3$$constant & 0x3f);
12653 %}
12654
12655 ins_pipe(ialu_reg_reg_shift);
12656 %}
12657
12658 // This pattern is automatically generated from aarch64_ad.m4
12659 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12660 instruct OrI_reg_RotateRight_reg(iRegINoSp dst,
12661 iRegIorL2I src1, iRegIorL2I src2,
12662 immI src3) %{
12663 match(Set dst (OrI src1 (RotateRight src2 src3)));
12664
12665 ins_cost(1.9 * INSN_COST);
12666 format %{ "orrw $dst, $src1, $src2, ROR $src3" %}
12667
12668 ins_encode %{
12669 __ orrw(as_Register($dst$$reg),
12670 as_Register($src1$$reg),
12671 as_Register($src2$$reg),
12672 Assembler::ROR,
12673 $src3$$constant & 0x1f);
12674 %}
12675
12676 ins_pipe(ialu_reg_reg_shift);
12677 %}
12678
12679 // This pattern is automatically generated from aarch64_ad.m4
12680 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12681 instruct OrL_reg_RotateRight_reg(iRegLNoSp dst,
12682 iRegL src1, iRegL src2,
12683 immI src3) %{
12684 match(Set dst (OrL src1 (RotateRight src2 src3)));
12685
12686 ins_cost(1.9 * INSN_COST);
12687 format %{ "orr $dst, $src1, $src2, ROR $src3" %}
12688
12689 ins_encode %{
12690 __ orr(as_Register($dst$$reg),
12691 as_Register($src1$$reg),
12692 as_Register($src2$$reg),
12693 Assembler::ROR,
12694 $src3$$constant & 0x3f);
12695 %}
12696
12697 ins_pipe(ialu_reg_reg_shift);
12698 %}
12699
12700 // This pattern is automatically generated from aarch64_ad.m4
12701 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12702 instruct AddI_reg_URShift_reg(iRegINoSp dst,
12703 iRegIorL2I src1, iRegIorL2I src2,
12704 immI src3) %{
12705 match(Set dst (AddI src1 (URShiftI src2 src3)));
12706
12707 ins_cost(1.9 * INSN_COST);
12708 format %{ "addw $dst, $src1, $src2, LSR $src3" %}
12709
12710 ins_encode %{
12711 __ addw(as_Register($dst$$reg),
12712 as_Register($src1$$reg),
12713 as_Register($src2$$reg),
12714 Assembler::LSR,
12715 $src3$$constant & 0x1f);
12716 %}
12717
12718 ins_pipe(ialu_reg_reg_shift);
12719 %}
12720
12721 // This pattern is automatically generated from aarch64_ad.m4
12722 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12723 instruct AddL_reg_URShift_reg(iRegLNoSp dst,
12724 iRegL src1, iRegL src2,
12725 immI src3) %{
12726 match(Set dst (AddL src1 (URShiftL src2 src3)));
12727
12728 ins_cost(1.9 * INSN_COST);
12729 format %{ "add $dst, $src1, $src2, LSR $src3" %}
12730
12731 ins_encode %{
12732 __ add(as_Register($dst$$reg),
12733 as_Register($src1$$reg),
12734 as_Register($src2$$reg),
12735 Assembler::LSR,
12736 $src3$$constant & 0x3f);
12737 %}
12738
12739 ins_pipe(ialu_reg_reg_shift);
12740 %}
12741
12742 // This pattern is automatically generated from aarch64_ad.m4
12743 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12744 instruct AddI_reg_RShift_reg(iRegINoSp dst,
12745 iRegIorL2I src1, iRegIorL2I src2,
12746 immI src3) %{
12747 match(Set dst (AddI src1 (RShiftI src2 src3)));
12748
12749 ins_cost(1.9 * INSN_COST);
12750 format %{ "addw $dst, $src1, $src2, ASR $src3" %}
12751
12752 ins_encode %{
12753 __ addw(as_Register($dst$$reg),
12754 as_Register($src1$$reg),
12755 as_Register($src2$$reg),
12756 Assembler::ASR,
12757 $src3$$constant & 0x1f);
12758 %}
12759
12760 ins_pipe(ialu_reg_reg_shift);
12761 %}
12762
12763 // This pattern is automatically generated from aarch64_ad.m4
12764 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12765 instruct AddL_reg_RShift_reg(iRegLNoSp dst,
12766 iRegL src1, iRegL src2,
12767 immI src3) %{
12768 match(Set dst (AddL src1 (RShiftL src2 src3)));
12769
12770 ins_cost(1.9 * INSN_COST);
12771 format %{ "add $dst, $src1, $src2, ASR $src3" %}
12772
12773 ins_encode %{
12774 __ add(as_Register($dst$$reg),
12775 as_Register($src1$$reg),
12776 as_Register($src2$$reg),
12777 Assembler::ASR,
12778 $src3$$constant & 0x3f);
12779 %}
12780
12781 ins_pipe(ialu_reg_reg_shift);
12782 %}
12783
12784 // This pattern is automatically generated from aarch64_ad.m4
12785 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12786 instruct AddI_reg_LShift_reg(iRegINoSp dst,
12787 iRegIorL2I src1, iRegIorL2I src2,
12788 immI src3) %{
12789 match(Set dst (AddI src1 (LShiftI src2 src3)));
12790
12791 ins_cost(1.9 * INSN_COST);
12792 format %{ "addw $dst, $src1, $src2, LSL $src3" %}
12793
12794 ins_encode %{
12795 __ addw(as_Register($dst$$reg),
12796 as_Register($src1$$reg),
12797 as_Register($src2$$reg),
12798 Assembler::LSL,
12799 $src3$$constant & 0x1f);
12800 %}
12801
12802 ins_pipe(ialu_reg_reg_shift);
12803 %}
12804
12805 // This pattern is automatically generated from aarch64_ad.m4
12806 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12807 instruct AddL_reg_LShift_reg(iRegLNoSp dst,
12808 iRegL src1, iRegL src2,
12809 immI src3) %{
12810 match(Set dst (AddL src1 (LShiftL src2 src3)));
12811
12812 ins_cost(1.9 * INSN_COST);
12813 format %{ "add $dst, $src1, $src2, LSL $src3" %}
12814
12815 ins_encode %{
12816 __ add(as_Register($dst$$reg),
12817 as_Register($src1$$reg),
12818 as_Register($src2$$reg),
12819 Assembler::LSL,
12820 $src3$$constant & 0x3f);
12821 %}
12822
12823 ins_pipe(ialu_reg_reg_shift);
12824 %}
12825
12826 // This pattern is automatically generated from aarch64_ad.m4
12827 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12828 instruct SubI_reg_URShift_reg(iRegINoSp dst,
12829 iRegIorL2I src1, iRegIorL2I src2,
12830 immI src3) %{
12831 match(Set dst (SubI src1 (URShiftI src2 src3)));
12832
12833 ins_cost(1.9 * INSN_COST);
12834 format %{ "subw $dst, $src1, $src2, LSR $src3" %}
12835
12836 ins_encode %{
12837 __ subw(as_Register($dst$$reg),
12838 as_Register($src1$$reg),
12839 as_Register($src2$$reg),
12840 Assembler::LSR,
12841 $src3$$constant & 0x1f);
12842 %}
12843
12844 ins_pipe(ialu_reg_reg_shift);
12845 %}
12846
12847 // This pattern is automatically generated from aarch64_ad.m4
12848 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12849 instruct SubL_reg_URShift_reg(iRegLNoSp dst,
12850 iRegL src1, iRegL src2,
12851 immI src3) %{
12852 match(Set dst (SubL src1 (URShiftL src2 src3)));
12853
12854 ins_cost(1.9 * INSN_COST);
12855 format %{ "sub $dst, $src1, $src2, LSR $src3" %}
12856
12857 ins_encode %{
12858 __ sub(as_Register($dst$$reg),
12859 as_Register($src1$$reg),
12860 as_Register($src2$$reg),
12861 Assembler::LSR,
12862 $src3$$constant & 0x3f);
12863 %}
12864
12865 ins_pipe(ialu_reg_reg_shift);
12866 %}
12867
12868 // This pattern is automatically generated from aarch64_ad.m4
12869 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12870 instruct SubI_reg_RShift_reg(iRegINoSp dst,
12871 iRegIorL2I src1, iRegIorL2I src2,
12872 immI src3) %{
12873 match(Set dst (SubI src1 (RShiftI src2 src3)));
12874
12875 ins_cost(1.9 * INSN_COST);
12876 format %{ "subw $dst, $src1, $src2, ASR $src3" %}
12877
12878 ins_encode %{
12879 __ subw(as_Register($dst$$reg),
12880 as_Register($src1$$reg),
12881 as_Register($src2$$reg),
12882 Assembler::ASR,
12883 $src3$$constant & 0x1f);
12884 %}
12885
12886 ins_pipe(ialu_reg_reg_shift);
12887 %}
12888
12889 // This pattern is automatically generated from aarch64_ad.m4
12890 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12891 instruct SubL_reg_RShift_reg(iRegLNoSp dst,
12892 iRegL src1, iRegL src2,
12893 immI src3) %{
12894 match(Set dst (SubL src1 (RShiftL src2 src3)));
12895
12896 ins_cost(1.9 * INSN_COST);
12897 format %{ "sub $dst, $src1, $src2, ASR $src3" %}
12898
12899 ins_encode %{
12900 __ sub(as_Register($dst$$reg),
12901 as_Register($src1$$reg),
12902 as_Register($src2$$reg),
12903 Assembler::ASR,
12904 $src3$$constant & 0x3f);
12905 %}
12906
12907 ins_pipe(ialu_reg_reg_shift);
12908 %}
12909
12910 // This pattern is automatically generated from aarch64_ad.m4
12911 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12912 instruct SubI_reg_LShift_reg(iRegINoSp dst,
12913 iRegIorL2I src1, iRegIorL2I src2,
12914 immI src3) %{
12915 match(Set dst (SubI src1 (LShiftI src2 src3)));
12916
12917 ins_cost(1.9 * INSN_COST);
12918 format %{ "subw $dst, $src1, $src2, LSL $src3" %}
12919
12920 ins_encode %{
12921 __ subw(as_Register($dst$$reg),
12922 as_Register($src1$$reg),
12923 as_Register($src2$$reg),
12924 Assembler::LSL,
12925 $src3$$constant & 0x1f);
12926 %}
12927
12928 ins_pipe(ialu_reg_reg_shift);
12929 %}
12930
12931 // This pattern is automatically generated from aarch64_ad.m4
12932 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12933 instruct SubL_reg_LShift_reg(iRegLNoSp dst,
12934 iRegL src1, iRegL src2,
12935 immI src3) %{
12936 match(Set dst (SubL src1 (LShiftL src2 src3)));
12937
12938 ins_cost(1.9 * INSN_COST);
12939 format %{ "sub $dst, $src1, $src2, LSL $src3" %}
12940
12941 ins_encode %{
12942 __ sub(as_Register($dst$$reg),
12943 as_Register($src1$$reg),
12944 as_Register($src2$$reg),
12945 Assembler::LSL,
12946 $src3$$constant & 0x3f);
12947 %}
12948
12949 ins_pipe(ialu_reg_reg_shift);
12950 %}
12951
12952 // This pattern is automatically generated from aarch64_ad.m4
12953 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12954
12955 // Shift Left followed by Shift Right.
12956 // This idiom is used by the compiler for the i2b bytecode etc.
12957 instruct sbfmL(iRegLNoSp dst, iRegL src, immI lshift_count, immI rshift_count)
12958 %{
12959 match(Set dst (RShiftL (LShiftL src lshift_count) rshift_count));
12960 ins_cost(INSN_COST * 2);
12961 format %{ "sbfm $dst, $src, $rshift_count - $lshift_count, #63 - $lshift_count" %}
12962 ins_encode %{
12963 int lshift = $lshift_count$$constant & 63;
12964 int rshift = $rshift_count$$constant & 63;
12965 int s = 63 - lshift;
12966 int r = (rshift - lshift) & 63;
12967 __ sbfm(as_Register($dst$$reg),
12968 as_Register($src$$reg),
12969 r, s);
12970 %}
12971
12972 ins_pipe(ialu_reg_shift);
12973 %}
12974
12975 // This pattern is automatically generated from aarch64_ad.m4
12976 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12977
12978 // Shift Left followed by Shift Right.
12979 // This idiom is used by the compiler for the i2b bytecode etc.
12980 instruct sbfmwI(iRegINoSp dst, iRegIorL2I src, immI lshift_count, immI rshift_count)
12981 %{
12982 match(Set dst (RShiftI (LShiftI src lshift_count) rshift_count));
12983 ins_cost(INSN_COST * 2);
12984 format %{ "sbfmw $dst, $src, $rshift_count - $lshift_count, #31 - $lshift_count" %}
12985 ins_encode %{
12986 int lshift = $lshift_count$$constant & 31;
12987 int rshift = $rshift_count$$constant & 31;
12988 int s = 31 - lshift;
12989 int r = (rshift - lshift) & 31;
12990 __ sbfmw(as_Register($dst$$reg),
12991 as_Register($src$$reg),
12992 r, s);
12993 %}
12994
12995 ins_pipe(ialu_reg_shift);
12996 %}
12997
12998 // This pattern is automatically generated from aarch64_ad.m4
12999 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13000
13001 // Shift Left followed by Shift Right.
13002 // This idiom is used by the compiler for the i2b bytecode etc.
13003 instruct ubfmL(iRegLNoSp dst, iRegL src, immI lshift_count, immI rshift_count)
13004 %{
13005 match(Set dst (URShiftL (LShiftL src lshift_count) rshift_count));
13006 ins_cost(INSN_COST * 2);
13007 format %{ "ubfm $dst, $src, $rshift_count - $lshift_count, #63 - $lshift_count" %}
13008 ins_encode %{
13009 int lshift = $lshift_count$$constant & 63;
13010 int rshift = $rshift_count$$constant & 63;
13011 int s = 63 - lshift;
13012 int r = (rshift - lshift) & 63;
13013 __ ubfm(as_Register($dst$$reg),
13014 as_Register($src$$reg),
13015 r, s);
13016 %}
13017
13018 ins_pipe(ialu_reg_shift);
13019 %}
13020
13021 // This pattern is automatically generated from aarch64_ad.m4
13022 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13023
13024 // Shift Left followed by Shift Right.
13025 // This idiom is used by the compiler for the i2b bytecode etc.
13026 instruct ubfmwI(iRegINoSp dst, iRegIorL2I src, immI lshift_count, immI rshift_count)
13027 %{
13028 match(Set dst (URShiftI (LShiftI src lshift_count) rshift_count));
13029 ins_cost(INSN_COST * 2);
13030 format %{ "ubfmw $dst, $src, $rshift_count - $lshift_count, #31 - $lshift_count" %}
13031 ins_encode %{
13032 int lshift = $lshift_count$$constant & 31;
13033 int rshift = $rshift_count$$constant & 31;
13034 int s = 31 - lshift;
13035 int r = (rshift - lshift) & 31;
13036 __ ubfmw(as_Register($dst$$reg),
13037 as_Register($src$$reg),
13038 r, s);
13039 %}
13040
13041 ins_pipe(ialu_reg_shift);
13042 %}
13043
13044 // Bitfield extract with shift & mask
13045
13046 // This pattern is automatically generated from aarch64_ad.m4
13047 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13048 instruct ubfxwI(iRegINoSp dst, iRegIorL2I src, immI rshift, immI_bitmask mask)
13049 %{
13050 match(Set dst (AndI (URShiftI src rshift) mask));
13051 // Make sure we are not going to exceed what ubfxw can do.
13052 predicate((exact_log2(n->in(2)->get_int() + 1) + (n->in(1)->in(2)->get_int() & 31)) <= (31 + 1));
13053
13054 ins_cost(INSN_COST);
13055 format %{ "ubfxw $dst, $src, $rshift, $mask" %}
13056 ins_encode %{
13057 int rshift = $rshift$$constant & 31;
13058 intptr_t mask = $mask$$constant;
13059 int width = exact_log2(mask+1);
13060 __ ubfxw(as_Register($dst$$reg),
13061 as_Register($src$$reg), rshift, width);
13062 %}
13063 ins_pipe(ialu_reg_shift);
13064 %}
13065
13066 // This pattern is automatically generated from aarch64_ad.m4
13067 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13068 instruct ubfxL(iRegLNoSp dst, iRegL src, immI rshift, immL_bitmask mask)
13069 %{
13070 match(Set dst (AndL (URShiftL src rshift) mask));
13071 // Make sure we are not going to exceed what ubfx can do.
13072 predicate((exact_log2_long(n->in(2)->get_long() + 1) + (n->in(1)->in(2)->get_int() & 63)) <= (63 + 1));
13073
13074 ins_cost(INSN_COST);
13075 format %{ "ubfx $dst, $src, $rshift, $mask" %}
13076 ins_encode %{
13077 int rshift = $rshift$$constant & 63;
13078 intptr_t mask = $mask$$constant;
13079 int width = exact_log2_long(mask+1);
13080 __ ubfx(as_Register($dst$$reg),
13081 as_Register($src$$reg), rshift, width);
13082 %}
13083 ins_pipe(ialu_reg_shift);
13084 %}
13085
13086
13087 // This pattern is automatically generated from aarch64_ad.m4
13088 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13089
13090 // We can use ubfx when extending an And with a mask when we know mask
13091 // is positive. We know that because immI_bitmask guarantees it.
13092 instruct ubfxIConvI2L(iRegLNoSp dst, iRegIorL2I src, immI rshift, immI_bitmask mask)
13093 %{
13094 match(Set dst (ConvI2L (AndI (URShiftI src rshift) mask)));
13095 // Make sure we are not going to exceed what ubfxw can do.
13096 predicate((exact_log2(n->in(1)->in(2)->get_int() + 1) + (n->in(1)->in(1)->in(2)->get_int() & 31)) <= (31 + 1));
13097
13098 ins_cost(INSN_COST * 2);
13099 format %{ "ubfx $dst, $src, $rshift, $mask" %}
13100 ins_encode %{
13101 int rshift = $rshift$$constant & 31;
13102 intptr_t mask = $mask$$constant;
13103 int width = exact_log2(mask+1);
13104 __ ubfx(as_Register($dst$$reg),
13105 as_Register($src$$reg), rshift, width);
13106 %}
13107 ins_pipe(ialu_reg_shift);
13108 %}
13109
13110
13111 // This pattern is automatically generated from aarch64_ad.m4
13112 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13113
13114 // We can use ubfiz when masking by a positive number and then left shifting the result.
13115 // We know that the mask is positive because immI_bitmask guarantees it.
13116 instruct ubfizwI(iRegINoSp dst, iRegIorL2I src, immI lshift, immI_bitmask mask)
13117 %{
13118 match(Set dst (LShiftI (AndI src mask) lshift));
13119 predicate((exact_log2(n->in(1)->in(2)->get_int() + 1) + (n->in(2)->get_int() & 31)) <= (31 + 1));
13120
13121 ins_cost(INSN_COST);
13122 format %{ "ubfizw $dst, $src, $lshift, $mask" %}
13123 ins_encode %{
13124 int lshift = $lshift$$constant & 31;
13125 intptr_t mask = $mask$$constant;
13126 int width = exact_log2(mask+1);
13127 __ ubfizw(as_Register($dst$$reg),
13128 as_Register($src$$reg), lshift, width);
13129 %}
13130 ins_pipe(ialu_reg_shift);
13131 %}
13132
13133 // This pattern is automatically generated from aarch64_ad.m4
13134 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13135
13136 // We can use ubfiz when masking by a positive number and then left shifting the result.
13137 // We know that the mask is positive because immL_bitmask guarantees it.
13138 instruct ubfizL(iRegLNoSp dst, iRegL src, immI lshift, immL_bitmask mask)
13139 %{
13140 match(Set dst (LShiftL (AndL src mask) lshift));
13141 predicate((exact_log2_long(n->in(1)->in(2)->get_long() + 1) + (n->in(2)->get_int() & 63)) <= (63 + 1));
13142
13143 ins_cost(INSN_COST);
13144 format %{ "ubfiz $dst, $src, $lshift, $mask" %}
13145 ins_encode %{
13146 int lshift = $lshift$$constant & 63;
13147 intptr_t mask = $mask$$constant;
13148 int width = exact_log2_long(mask+1);
13149 __ ubfiz(as_Register($dst$$reg),
13150 as_Register($src$$reg), lshift, width);
13151 %}
13152 ins_pipe(ialu_reg_shift);
13153 %}
13154
13155 // This pattern is automatically generated from aarch64_ad.m4
13156 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13157
13158 // We can use ubfiz when masking by a positive number and then left shifting the result.
13159 // We know that the mask is positive because immI_bitmask guarantees it.
13160 instruct ubfizwIConvI2L(iRegLNoSp dst, iRegIorL2I src, immI lshift, immI_bitmask mask)
13161 %{
13162 match(Set dst (ConvI2L (LShiftI (AndI src mask) lshift)));
13163 predicate((exact_log2(n->in(1)->in(1)->in(2)->get_int() + 1) + (n->in(1)->in(2)->get_int() & 31)) <= 31);
13164
13165 ins_cost(INSN_COST);
13166 format %{ "ubfizw $dst, $src, $lshift, $mask" %}
13167 ins_encode %{
13168 int lshift = $lshift$$constant & 31;
13169 intptr_t mask = $mask$$constant;
13170 int width = exact_log2(mask+1);
13171 __ ubfizw(as_Register($dst$$reg),
13172 as_Register($src$$reg), lshift, width);
13173 %}
13174 ins_pipe(ialu_reg_shift);
13175 %}
13176
13177 // This pattern is automatically generated from aarch64_ad.m4
13178 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13179
13180 // We can use ubfiz when masking by a positive number and then left shifting the result.
13181 // We know that the mask is positive because immL_bitmask guarantees it.
13182 instruct ubfizLConvL2I(iRegINoSp dst, iRegL src, immI lshift, immL_positive_bitmaskI mask)
13183 %{
13184 match(Set dst (ConvL2I (LShiftL (AndL src mask) lshift)));
13185 predicate((exact_log2_long(n->in(1)->in(1)->in(2)->get_long() + 1) + (n->in(1)->in(2)->get_int() & 63)) <= 31);
13186
13187 ins_cost(INSN_COST);
13188 format %{ "ubfiz $dst, $src, $lshift, $mask" %}
13189 ins_encode %{
13190 int lshift = $lshift$$constant & 63;
13191 intptr_t mask = $mask$$constant;
13192 int width = exact_log2_long(mask+1);
13193 __ ubfiz(as_Register($dst$$reg),
13194 as_Register($src$$reg), lshift, width);
13195 %}
13196 ins_pipe(ialu_reg_shift);
13197 %}
13198
13199
13200 // This pattern is automatically generated from aarch64_ad.m4
13201 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13202
13203 // If there is a convert I to L block between and AndI and a LShiftL, we can also match ubfiz
13204 instruct ubfizIConvI2L(iRegLNoSp dst, iRegIorL2I src, immI lshift, immI_bitmask mask)
13205 %{
13206 match(Set dst (LShiftL (ConvI2L (AndI src mask)) lshift));
13207 predicate((exact_log2(n->in(1)->in(1)->in(2)->get_int() + 1) + (n->in(2)->get_int() & 63)) <= (63 + 1));
13208
13209 ins_cost(INSN_COST);
13210 format %{ "ubfiz $dst, $src, $lshift, $mask" %}
13211 ins_encode %{
13212 int lshift = $lshift$$constant & 63;
13213 intptr_t mask = $mask$$constant;
13214 int width = exact_log2(mask+1);
13215 __ ubfiz(as_Register($dst$$reg),
13216 as_Register($src$$reg), lshift, width);
13217 %}
13218 ins_pipe(ialu_reg_shift);
13219 %}
13220
13221 // This pattern is automatically generated from aarch64_ad.m4
13222 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13223
13224 // If there is a convert L to I block between and AndL and a LShiftI, we can also match ubfiz
13225 instruct ubfizLConvL2Ix(iRegINoSp dst, iRegL src, immI lshift, immL_positive_bitmaskI mask)
13226 %{
13227 match(Set dst (LShiftI (ConvL2I (AndL src mask)) lshift));
13228 predicate((exact_log2_long(n->in(1)->in(1)->in(2)->get_long() + 1) + (n->in(2)->get_int() & 31)) <= 31);
13229
13230 ins_cost(INSN_COST);
13231 format %{ "ubfiz $dst, $src, $lshift, $mask" %}
13232 ins_encode %{
13233 int lshift = $lshift$$constant & 31;
13234 intptr_t mask = $mask$$constant;
13235 int width = exact_log2(mask+1);
13236 __ ubfiz(as_Register($dst$$reg),
13237 as_Register($src$$reg), lshift, width);
13238 %}
13239 ins_pipe(ialu_reg_shift);
13240 %}
13241
13242 // This pattern is automatically generated from aarch64_ad.m4
13243 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13244
13245 // Can skip int2long conversions after AND with small bitmask
13246 instruct ubfizIConvI2LAndI(iRegLNoSp dst, iRegI src, immI_bitmask msk)
13247 %{
13248 match(Set dst (ConvI2L (AndI src msk)));
13249 ins_cost(INSN_COST);
13250 format %{ "ubfiz $dst, $src, 0, exact_log2($msk + 1) " %}
13251 ins_encode %{
13252 __ ubfiz(as_Register($dst$$reg), as_Register($src$$reg), 0, exact_log2($msk$$constant + 1));
13253 %}
13254 ins_pipe(ialu_reg_shift);
13255 %}
13256
13257
13258 // Rotations
13259 // This pattern is automatically generated from aarch64_ad.m4
13260 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13261 instruct extrOrL(iRegLNoSp dst, iRegL src1, iRegL src2, immI lshift, immI rshift, rFlagsReg cr)
13262 %{
13263 match(Set dst (OrL (LShiftL src1 lshift) (URShiftL src2 rshift)));
13264 predicate(0 == (((n->in(1)->in(2)->get_int() & 63) + (n->in(2)->in(2)->get_int() & 63)) & 63));
13265
13266 ins_cost(INSN_COST);
13267 format %{ "extr $dst, $src1, $src2, #$rshift" %}
13268
13269 ins_encode %{
13270 __ extr(as_Register($dst$$reg), as_Register($src1$$reg), as_Register($src2$$reg),
13271 $rshift$$constant & 63);
13272 %}
13273 ins_pipe(ialu_reg_reg_extr);
13274 %}
13275
13276
13277 // This pattern is automatically generated from aarch64_ad.m4
13278 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13279 instruct extrOrI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI lshift, immI rshift, rFlagsReg cr)
13280 %{
13281 match(Set dst (OrI (LShiftI src1 lshift) (URShiftI src2 rshift)));
13282 predicate(0 == (((n->in(1)->in(2)->get_int() & 31) + (n->in(2)->in(2)->get_int() & 31)) & 31));
13283
13284 ins_cost(INSN_COST);
13285 format %{ "extr $dst, $src1, $src2, #$rshift" %}
13286
13287 ins_encode %{
13288 __ extrw(as_Register($dst$$reg), as_Register($src1$$reg), as_Register($src2$$reg),
13289 $rshift$$constant & 31);
13290 %}
13291 ins_pipe(ialu_reg_reg_extr);
13292 %}
13293
13294
13295 // This pattern is automatically generated from aarch64_ad.m4
13296 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13297 instruct extrAddL(iRegLNoSp dst, iRegL src1, iRegL src2, immI lshift, immI rshift, rFlagsReg cr)
13298 %{
13299 match(Set dst (AddL (LShiftL src1 lshift) (URShiftL src2 rshift)));
13300 predicate(0 == (((n->in(1)->in(2)->get_int() & 63) + (n->in(2)->in(2)->get_int() & 63)) & 63));
13301
13302 ins_cost(INSN_COST);
13303 format %{ "extr $dst, $src1, $src2, #$rshift" %}
13304
13305 ins_encode %{
13306 __ extr(as_Register($dst$$reg), as_Register($src1$$reg), as_Register($src2$$reg),
13307 $rshift$$constant & 63);
13308 %}
13309 ins_pipe(ialu_reg_reg_extr);
13310 %}
13311
13312
13313 // This pattern is automatically generated from aarch64_ad.m4
13314 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13315 instruct extrAddI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI lshift, immI rshift, rFlagsReg cr)
13316 %{
13317 match(Set dst (AddI (LShiftI src1 lshift) (URShiftI src2 rshift)));
13318 predicate(0 == (((n->in(1)->in(2)->get_int() & 31) + (n->in(2)->in(2)->get_int() & 31)) & 31));
13319
13320 ins_cost(INSN_COST);
13321 format %{ "extr $dst, $src1, $src2, #$rshift" %}
13322
13323 ins_encode %{
13324 __ extrw(as_Register($dst$$reg), as_Register($src1$$reg), as_Register($src2$$reg),
13325 $rshift$$constant & 31);
13326 %}
13327 ins_pipe(ialu_reg_reg_extr);
13328 %}
13329
13330
13331 // This pattern is automatically generated from aarch64_ad.m4
13332 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13333 instruct rorI_imm(iRegINoSp dst, iRegI src, immI shift)
13334 %{
13335 match(Set dst (RotateRight src shift));
13336
13337 ins_cost(INSN_COST);
13338 format %{ "ror $dst, $src, $shift" %}
13339
13340 ins_encode %{
13341 __ extrw(as_Register($dst$$reg), as_Register($src$$reg), as_Register($src$$reg),
13342 $shift$$constant & 0x1f);
13343 %}
13344 ins_pipe(ialu_reg_reg_vshift);
13345 %}
13346
13347 // This pattern is automatically generated from aarch64_ad.m4
13348 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13349 instruct rorL_imm(iRegLNoSp dst, iRegL src, immI shift)
13350 %{
13351 match(Set dst (RotateRight src shift));
13352
13353 ins_cost(INSN_COST);
13354 format %{ "ror $dst, $src, $shift" %}
13355
13356 ins_encode %{
13357 __ extr(as_Register($dst$$reg), as_Register($src$$reg), as_Register($src$$reg),
13358 $shift$$constant & 0x3f);
13359 %}
13360 ins_pipe(ialu_reg_reg_vshift);
13361 %}
13362
13363 // This pattern is automatically generated from aarch64_ad.m4
13364 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13365 instruct rorI_reg(iRegINoSp dst, iRegI src, iRegI shift)
13366 %{
13367 match(Set dst (RotateRight src shift));
13368
13369 ins_cost(INSN_COST);
13370 format %{ "ror $dst, $src, $shift" %}
13371
13372 ins_encode %{
13373 __ rorvw(as_Register($dst$$reg), as_Register($src$$reg), as_Register($shift$$reg));
13374 %}
13375 ins_pipe(ialu_reg_reg_vshift);
13376 %}
13377
13378 // This pattern is automatically generated from aarch64_ad.m4
13379 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13380 instruct rorL_reg(iRegLNoSp dst, iRegL src, iRegI shift)
13381 %{
13382 match(Set dst (RotateRight src shift));
13383
13384 ins_cost(INSN_COST);
13385 format %{ "ror $dst, $src, $shift" %}
13386
13387 ins_encode %{
13388 __ rorv(as_Register($dst$$reg), as_Register($src$$reg), as_Register($shift$$reg));
13389 %}
13390 ins_pipe(ialu_reg_reg_vshift);
13391 %}
13392
13393 // This pattern is automatically generated from aarch64_ad.m4
13394 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13395 instruct rolI_reg(iRegINoSp dst, iRegI src, iRegI shift)
13396 %{
13397 match(Set dst (RotateLeft src shift));
13398
13399 ins_cost(INSN_COST);
13400 format %{ "rol $dst, $src, $shift" %}
13401
13402 ins_encode %{
13403 __ subw(rscratch1, zr, as_Register($shift$$reg));
13404 __ rorvw(as_Register($dst$$reg), as_Register($src$$reg), rscratch1);
13405 %}
13406 ins_pipe(ialu_reg_reg_vshift);
13407 %}
13408
13409 // This pattern is automatically generated from aarch64_ad.m4
13410 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13411 instruct rolL_reg(iRegLNoSp dst, iRegL src, iRegI shift)
13412 %{
13413 match(Set dst (RotateLeft src shift));
13414
13415 ins_cost(INSN_COST);
13416 format %{ "rol $dst, $src, $shift" %}
13417
13418 ins_encode %{
13419 __ subw(rscratch1, zr, as_Register($shift$$reg));
13420 __ rorv(as_Register($dst$$reg), as_Register($src$$reg), rscratch1);
13421 %}
13422 ins_pipe(ialu_reg_reg_vshift);
13423 %}
13424
13425
13426 // Add/subtract (extended)
13427
13428 // This pattern is automatically generated from aarch64_ad.m4
13429 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13430 instruct AddExtI(iRegLNoSp dst, iRegL src1, iRegIorL2I src2, rFlagsReg cr)
13431 %{
13432 match(Set dst (AddL src1 (ConvI2L src2)));
13433 ins_cost(INSN_COST);
13434 format %{ "add $dst, $src1, $src2, sxtw" %}
13435
13436 ins_encode %{
13437 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13438 as_Register($src2$$reg), ext::sxtw);
13439 %}
13440 ins_pipe(ialu_reg_reg);
13441 %}
13442
13443 // This pattern is automatically generated from aarch64_ad.m4
13444 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13445 instruct SubExtI(iRegLNoSp dst, iRegL src1, iRegIorL2I src2, rFlagsReg cr)
13446 %{
13447 match(Set dst (SubL src1 (ConvI2L src2)));
13448 ins_cost(INSN_COST);
13449 format %{ "sub $dst, $src1, $src2, sxtw" %}
13450
13451 ins_encode %{
13452 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13453 as_Register($src2$$reg), ext::sxtw);
13454 %}
13455 ins_pipe(ialu_reg_reg);
13456 %}
13457
13458 // This pattern is automatically generated from aarch64_ad.m4
13459 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13460 instruct AddExtI_sxth(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_16 lshift, immI_16 rshift, rFlagsReg cr)
13461 %{
13462 match(Set dst (AddI src1 (RShiftI (LShiftI src2 lshift) rshift)));
13463 ins_cost(INSN_COST);
13464 format %{ "add $dst, $src1, $src2, sxth" %}
13465
13466 ins_encode %{
13467 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13468 as_Register($src2$$reg), ext::sxth);
13469 %}
13470 ins_pipe(ialu_reg_reg);
13471 %}
13472
13473 // This pattern is automatically generated from aarch64_ad.m4
13474 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13475 instruct AddExtI_sxtb(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_24 lshift, immI_24 rshift, rFlagsReg cr)
13476 %{
13477 match(Set dst (AddI src1 (RShiftI (LShiftI src2 lshift) rshift)));
13478 ins_cost(INSN_COST);
13479 format %{ "add $dst, $src1, $src2, sxtb" %}
13480
13481 ins_encode %{
13482 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13483 as_Register($src2$$reg), ext::sxtb);
13484 %}
13485 ins_pipe(ialu_reg_reg);
13486 %}
13487
13488 // This pattern is automatically generated from aarch64_ad.m4
13489 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13490 instruct AddExtI_uxtb(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_24 lshift, immI_24 rshift, rFlagsReg cr)
13491 %{
13492 match(Set dst (AddI src1 (URShiftI (LShiftI src2 lshift) rshift)));
13493 ins_cost(INSN_COST);
13494 format %{ "add $dst, $src1, $src2, uxtb" %}
13495
13496 ins_encode %{
13497 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13498 as_Register($src2$$reg), ext::uxtb);
13499 %}
13500 ins_pipe(ialu_reg_reg);
13501 %}
13502
13503 // This pattern is automatically generated from aarch64_ad.m4
13504 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13505 instruct AddExtL_sxth(iRegLNoSp dst, iRegL src1, iRegL src2, immI_48 lshift, immI_48 rshift, rFlagsReg cr)
13506 %{
13507 match(Set dst (AddL src1 (RShiftL (LShiftL src2 lshift) rshift)));
13508 ins_cost(INSN_COST);
13509 format %{ "add $dst, $src1, $src2, sxth" %}
13510
13511 ins_encode %{
13512 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13513 as_Register($src2$$reg), ext::sxth);
13514 %}
13515 ins_pipe(ialu_reg_reg);
13516 %}
13517
13518 // This pattern is automatically generated from aarch64_ad.m4
13519 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13520 instruct AddExtL_sxtw(iRegLNoSp dst, iRegL src1, iRegL src2, immI_32 lshift, immI_32 rshift, rFlagsReg cr)
13521 %{
13522 match(Set dst (AddL src1 (RShiftL (LShiftL src2 lshift) rshift)));
13523 ins_cost(INSN_COST);
13524 format %{ "add $dst, $src1, $src2, sxtw" %}
13525
13526 ins_encode %{
13527 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13528 as_Register($src2$$reg), ext::sxtw);
13529 %}
13530 ins_pipe(ialu_reg_reg);
13531 %}
13532
13533 // This pattern is automatically generated from aarch64_ad.m4
13534 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13535 instruct AddExtL_sxtb(iRegLNoSp dst, iRegL src1, iRegL src2, immI_56 lshift, immI_56 rshift, rFlagsReg cr)
13536 %{
13537 match(Set dst (AddL src1 (RShiftL (LShiftL src2 lshift) rshift)));
13538 ins_cost(INSN_COST);
13539 format %{ "add $dst, $src1, $src2, sxtb" %}
13540
13541 ins_encode %{
13542 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13543 as_Register($src2$$reg), ext::sxtb);
13544 %}
13545 ins_pipe(ialu_reg_reg);
13546 %}
13547
13548 // This pattern is automatically generated from aarch64_ad.m4
13549 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13550 instruct AddExtL_uxtb(iRegLNoSp dst, iRegL src1, iRegL src2, immI_56 lshift, immI_56 rshift, rFlagsReg cr)
13551 %{
13552 match(Set dst (AddL src1 (URShiftL (LShiftL src2 lshift) rshift)));
13553 ins_cost(INSN_COST);
13554 format %{ "add $dst, $src1, $src2, uxtb" %}
13555
13556 ins_encode %{
13557 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13558 as_Register($src2$$reg), ext::uxtb);
13559 %}
13560 ins_pipe(ialu_reg_reg);
13561 %}
13562
13563 // This pattern is automatically generated from aarch64_ad.m4
13564 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13565 instruct AddExtI_uxtb_and(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_255 mask, rFlagsReg cr)
13566 %{
13567 match(Set dst (AddI src1 (AndI src2 mask)));
13568 ins_cost(INSN_COST);
13569 format %{ "addw $dst, $src1, $src2, uxtb" %}
13570
13571 ins_encode %{
13572 __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
13573 as_Register($src2$$reg), ext::uxtb);
13574 %}
13575 ins_pipe(ialu_reg_reg);
13576 %}
13577
13578 // This pattern is automatically generated from aarch64_ad.m4
13579 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13580 instruct AddExtI_uxth_and(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_65535 mask, rFlagsReg cr)
13581 %{
13582 match(Set dst (AddI src1 (AndI src2 mask)));
13583 ins_cost(INSN_COST);
13584 format %{ "addw $dst, $src1, $src2, uxth" %}
13585
13586 ins_encode %{
13587 __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
13588 as_Register($src2$$reg), ext::uxth);
13589 %}
13590 ins_pipe(ialu_reg_reg);
13591 %}
13592
13593 // This pattern is automatically generated from aarch64_ad.m4
13594 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13595 instruct AddExtL_uxtb_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_255 mask, rFlagsReg cr)
13596 %{
13597 match(Set dst (AddL src1 (AndL src2 mask)));
13598 ins_cost(INSN_COST);
13599 format %{ "add $dst, $src1, $src2, uxtb" %}
13600
13601 ins_encode %{
13602 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13603 as_Register($src2$$reg), ext::uxtb);
13604 %}
13605 ins_pipe(ialu_reg_reg);
13606 %}
13607
13608 // This pattern is automatically generated from aarch64_ad.m4
13609 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13610 instruct AddExtL_uxth_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_65535 mask, rFlagsReg cr)
13611 %{
13612 match(Set dst (AddL src1 (AndL src2 mask)));
13613 ins_cost(INSN_COST);
13614 format %{ "add $dst, $src1, $src2, uxth" %}
13615
13616 ins_encode %{
13617 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13618 as_Register($src2$$reg), ext::uxth);
13619 %}
13620 ins_pipe(ialu_reg_reg);
13621 %}
13622
13623 // This pattern is automatically generated from aarch64_ad.m4
13624 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13625 instruct AddExtL_uxtw_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_4294967295 mask, rFlagsReg cr)
13626 %{
13627 match(Set dst (AddL src1 (AndL src2 mask)));
13628 ins_cost(INSN_COST);
13629 format %{ "add $dst, $src1, $src2, uxtw" %}
13630
13631 ins_encode %{
13632 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13633 as_Register($src2$$reg), ext::uxtw);
13634 %}
13635 ins_pipe(ialu_reg_reg);
13636 %}
13637
13638 // This pattern is automatically generated from aarch64_ad.m4
13639 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13640 instruct SubExtI_uxtb_and(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_255 mask, rFlagsReg cr)
13641 %{
13642 match(Set dst (SubI src1 (AndI src2 mask)));
13643 ins_cost(INSN_COST);
13644 format %{ "subw $dst, $src1, $src2, uxtb" %}
13645
13646 ins_encode %{
13647 __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
13648 as_Register($src2$$reg), ext::uxtb);
13649 %}
13650 ins_pipe(ialu_reg_reg);
13651 %}
13652
13653 // This pattern is automatically generated from aarch64_ad.m4
13654 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13655 instruct SubExtI_uxth_and(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_65535 mask, rFlagsReg cr)
13656 %{
13657 match(Set dst (SubI src1 (AndI src2 mask)));
13658 ins_cost(INSN_COST);
13659 format %{ "subw $dst, $src1, $src2, uxth" %}
13660
13661 ins_encode %{
13662 __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
13663 as_Register($src2$$reg), ext::uxth);
13664 %}
13665 ins_pipe(ialu_reg_reg);
13666 %}
13667
13668 // This pattern is automatically generated from aarch64_ad.m4
13669 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13670 instruct SubExtL_uxtb_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_255 mask, rFlagsReg cr)
13671 %{
13672 match(Set dst (SubL src1 (AndL src2 mask)));
13673 ins_cost(INSN_COST);
13674 format %{ "sub $dst, $src1, $src2, uxtb" %}
13675
13676 ins_encode %{
13677 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13678 as_Register($src2$$reg), ext::uxtb);
13679 %}
13680 ins_pipe(ialu_reg_reg);
13681 %}
13682
13683 // This pattern is automatically generated from aarch64_ad.m4
13684 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13685 instruct SubExtL_uxth_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_65535 mask, rFlagsReg cr)
13686 %{
13687 match(Set dst (SubL src1 (AndL src2 mask)));
13688 ins_cost(INSN_COST);
13689 format %{ "sub $dst, $src1, $src2, uxth" %}
13690
13691 ins_encode %{
13692 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13693 as_Register($src2$$reg), ext::uxth);
13694 %}
13695 ins_pipe(ialu_reg_reg);
13696 %}
13697
13698 // This pattern is automatically generated from aarch64_ad.m4
13699 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13700 instruct SubExtL_uxtw_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_4294967295 mask, rFlagsReg cr)
13701 %{
13702 match(Set dst (SubL src1 (AndL src2 mask)));
13703 ins_cost(INSN_COST);
13704 format %{ "sub $dst, $src1, $src2, uxtw" %}
13705
13706 ins_encode %{
13707 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13708 as_Register($src2$$reg), ext::uxtw);
13709 %}
13710 ins_pipe(ialu_reg_reg);
13711 %}
13712
13713
13714 // This pattern is automatically generated from aarch64_ad.m4
13715 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13716 instruct AddExtL_sxtb_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_56 lshift1, immI_56 rshift1, rFlagsReg cr)
13717 %{
13718 match(Set dst (AddL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
13719 ins_cost(1.9 * INSN_COST);
13720 format %{ "add $dst, $src1, $src2, sxtb #lshift2" %}
13721
13722 ins_encode %{
13723 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13724 as_Register($src2$$reg), ext::sxtb, ($lshift2$$constant));
13725 %}
13726 ins_pipe(ialu_reg_reg_shift);
13727 %}
13728
13729 // This pattern is automatically generated from aarch64_ad.m4
13730 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13731 instruct AddExtL_sxth_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_48 lshift1, immI_48 rshift1, rFlagsReg cr)
13732 %{
13733 match(Set dst (AddL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
13734 ins_cost(1.9 * INSN_COST);
13735 format %{ "add $dst, $src1, $src2, sxth #lshift2" %}
13736
13737 ins_encode %{
13738 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13739 as_Register($src2$$reg), ext::sxth, ($lshift2$$constant));
13740 %}
13741 ins_pipe(ialu_reg_reg_shift);
13742 %}
13743
13744 // This pattern is automatically generated from aarch64_ad.m4
13745 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13746 instruct AddExtL_sxtw_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_32 lshift1, immI_32 rshift1, rFlagsReg cr)
13747 %{
13748 match(Set dst (AddL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
13749 ins_cost(1.9 * INSN_COST);
13750 format %{ "add $dst, $src1, $src2, sxtw #lshift2" %}
13751
13752 ins_encode %{
13753 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13754 as_Register($src2$$reg), ext::sxtw, ($lshift2$$constant));
13755 %}
13756 ins_pipe(ialu_reg_reg_shift);
13757 %}
13758
13759 // This pattern is automatically generated from aarch64_ad.m4
13760 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13761 instruct SubExtL_sxtb_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_56 lshift1, immI_56 rshift1, rFlagsReg cr)
13762 %{
13763 match(Set dst (SubL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
13764 ins_cost(1.9 * INSN_COST);
13765 format %{ "sub $dst, $src1, $src2, sxtb #lshift2" %}
13766
13767 ins_encode %{
13768 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13769 as_Register($src2$$reg), ext::sxtb, ($lshift2$$constant));
13770 %}
13771 ins_pipe(ialu_reg_reg_shift);
13772 %}
13773
13774 // This pattern is automatically generated from aarch64_ad.m4
13775 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13776 instruct SubExtL_sxth_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_48 lshift1, immI_48 rshift1, rFlagsReg cr)
13777 %{
13778 match(Set dst (SubL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
13779 ins_cost(1.9 * INSN_COST);
13780 format %{ "sub $dst, $src1, $src2, sxth #lshift2" %}
13781
13782 ins_encode %{
13783 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13784 as_Register($src2$$reg), ext::sxth, ($lshift2$$constant));
13785 %}
13786 ins_pipe(ialu_reg_reg_shift);
13787 %}
13788
13789 // This pattern is automatically generated from aarch64_ad.m4
13790 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13791 instruct SubExtL_sxtw_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_32 lshift1, immI_32 rshift1, rFlagsReg cr)
13792 %{
13793 match(Set dst (SubL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
13794 ins_cost(1.9 * INSN_COST);
13795 format %{ "sub $dst, $src1, $src2, sxtw #lshift2" %}
13796
13797 ins_encode %{
13798 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13799 as_Register($src2$$reg), ext::sxtw, ($lshift2$$constant));
13800 %}
13801 ins_pipe(ialu_reg_reg_shift);
13802 %}
13803
13804 // This pattern is automatically generated from aarch64_ad.m4
13805 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13806 instruct AddExtI_sxtb_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immIExt lshift2, immI_24 lshift1, immI_24 rshift1, rFlagsReg cr)
13807 %{
13808 match(Set dst (AddI src1 (LShiftI (RShiftI (LShiftI src2 lshift1) rshift1) lshift2)));
13809 ins_cost(1.9 * INSN_COST);
13810 format %{ "addw $dst, $src1, $src2, sxtb #lshift2" %}
13811
13812 ins_encode %{
13813 __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
13814 as_Register($src2$$reg), ext::sxtb, ($lshift2$$constant));
13815 %}
13816 ins_pipe(ialu_reg_reg_shift);
13817 %}
13818
13819 // This pattern is automatically generated from aarch64_ad.m4
13820 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13821 instruct AddExtI_sxth_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immIExt lshift2, immI_16 lshift1, immI_16 rshift1, rFlagsReg cr)
13822 %{
13823 match(Set dst (AddI src1 (LShiftI (RShiftI (LShiftI src2 lshift1) rshift1) lshift2)));
13824 ins_cost(1.9 * INSN_COST);
13825 format %{ "addw $dst, $src1, $src2, sxth #lshift2" %}
13826
13827 ins_encode %{
13828 __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
13829 as_Register($src2$$reg), ext::sxth, ($lshift2$$constant));
13830 %}
13831 ins_pipe(ialu_reg_reg_shift);
13832 %}
13833
13834 // This pattern is automatically generated from aarch64_ad.m4
13835 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13836 instruct SubExtI_sxtb_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immIExt lshift2, immI_24 lshift1, immI_24 rshift1, rFlagsReg cr)
13837 %{
13838 match(Set dst (SubI src1 (LShiftI (RShiftI (LShiftI src2 lshift1) rshift1) lshift2)));
13839 ins_cost(1.9 * INSN_COST);
13840 format %{ "subw $dst, $src1, $src2, sxtb #lshift2" %}
13841
13842 ins_encode %{
13843 __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
13844 as_Register($src2$$reg), ext::sxtb, ($lshift2$$constant));
13845 %}
13846 ins_pipe(ialu_reg_reg_shift);
13847 %}
13848
13849 // This pattern is automatically generated from aarch64_ad.m4
13850 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13851 instruct SubExtI_sxth_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immIExt lshift2, immI_16 lshift1, immI_16 rshift1, rFlagsReg cr)
13852 %{
13853 match(Set dst (SubI src1 (LShiftI (RShiftI (LShiftI src2 lshift1) rshift1) lshift2)));
13854 ins_cost(1.9 * INSN_COST);
13855 format %{ "subw $dst, $src1, $src2, sxth #lshift2" %}
13856
13857 ins_encode %{
13858 __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
13859 as_Register($src2$$reg), ext::sxth, ($lshift2$$constant));
13860 %}
13861 ins_pipe(ialu_reg_reg_shift);
13862 %}
13863
13864 // This pattern is automatically generated from aarch64_ad.m4
13865 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13866 instruct AddExtI_shift(iRegLNoSp dst, iRegL src1, iRegIorL2I src2, immIExt lshift, rFlagsReg cr)
13867 %{
13868 match(Set dst (AddL src1 (LShiftL (ConvI2L src2) lshift)));
13869 ins_cost(1.9 * INSN_COST);
13870 format %{ "add $dst, $src1, $src2, sxtw #lshift" %}
13871
13872 ins_encode %{
13873 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13874 as_Register($src2$$reg), ext::sxtw, ($lshift$$constant));
13875 %}
13876 ins_pipe(ialu_reg_reg_shift);
13877 %}
13878
13879 // This pattern is automatically generated from aarch64_ad.m4
13880 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13881 instruct SubExtI_shift(iRegLNoSp dst, iRegL src1, iRegIorL2I src2, immIExt lshift, rFlagsReg cr)
13882 %{
13883 match(Set dst (SubL src1 (LShiftL (ConvI2L src2) lshift)));
13884 ins_cost(1.9 * INSN_COST);
13885 format %{ "sub $dst, $src1, $src2, sxtw #lshift" %}
13886
13887 ins_encode %{
13888 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13889 as_Register($src2$$reg), ext::sxtw, ($lshift$$constant));
13890 %}
13891 ins_pipe(ialu_reg_reg_shift);
13892 %}
13893
13894 // This pattern is automatically generated from aarch64_ad.m4
13895 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13896 instruct AddExtL_uxtb_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_255 mask, immIExt lshift, rFlagsReg cr)
13897 %{
13898 match(Set dst (AddL src1 (LShiftL (AndL src2 mask) lshift)));
13899 ins_cost(1.9 * INSN_COST);
13900 format %{ "add $dst, $src1, $src2, uxtb #lshift" %}
13901
13902 ins_encode %{
13903 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13904 as_Register($src2$$reg), ext::uxtb, ($lshift$$constant));
13905 %}
13906 ins_pipe(ialu_reg_reg_shift);
13907 %}
13908
13909 // This pattern is automatically generated from aarch64_ad.m4
13910 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13911 instruct AddExtL_uxth_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_65535 mask, immIExt lshift, rFlagsReg cr)
13912 %{
13913 match(Set dst (AddL src1 (LShiftL (AndL src2 mask) lshift)));
13914 ins_cost(1.9 * INSN_COST);
13915 format %{ "add $dst, $src1, $src2, uxth #lshift" %}
13916
13917 ins_encode %{
13918 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13919 as_Register($src2$$reg), ext::uxth, ($lshift$$constant));
13920 %}
13921 ins_pipe(ialu_reg_reg_shift);
13922 %}
13923
13924 // This pattern is automatically generated from aarch64_ad.m4
13925 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13926 instruct AddExtL_uxtw_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_4294967295 mask, immIExt lshift, rFlagsReg cr)
13927 %{
13928 match(Set dst (AddL src1 (LShiftL (AndL src2 mask) lshift)));
13929 ins_cost(1.9 * INSN_COST);
13930 format %{ "add $dst, $src1, $src2, uxtw #lshift" %}
13931
13932 ins_encode %{
13933 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13934 as_Register($src2$$reg), ext::uxtw, ($lshift$$constant));
13935 %}
13936 ins_pipe(ialu_reg_reg_shift);
13937 %}
13938
13939 // This pattern is automatically generated from aarch64_ad.m4
13940 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13941 instruct SubExtL_uxtb_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_255 mask, immIExt lshift, rFlagsReg cr)
13942 %{
13943 match(Set dst (SubL src1 (LShiftL (AndL src2 mask) lshift)));
13944 ins_cost(1.9 * INSN_COST);
13945 format %{ "sub $dst, $src1, $src2, uxtb #lshift" %}
13946
13947 ins_encode %{
13948 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13949 as_Register($src2$$reg), ext::uxtb, ($lshift$$constant));
13950 %}
13951 ins_pipe(ialu_reg_reg_shift);
13952 %}
13953
13954 // This pattern is automatically generated from aarch64_ad.m4
13955 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13956 instruct SubExtL_uxth_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_65535 mask, immIExt lshift, rFlagsReg cr)
13957 %{
13958 match(Set dst (SubL src1 (LShiftL (AndL src2 mask) lshift)));
13959 ins_cost(1.9 * INSN_COST);
13960 format %{ "sub $dst, $src1, $src2, uxth #lshift" %}
13961
13962 ins_encode %{
13963 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13964 as_Register($src2$$reg), ext::uxth, ($lshift$$constant));
13965 %}
13966 ins_pipe(ialu_reg_reg_shift);
13967 %}
13968
13969 // This pattern is automatically generated from aarch64_ad.m4
13970 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13971 instruct SubExtL_uxtw_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_4294967295 mask, immIExt lshift, rFlagsReg cr)
13972 %{
13973 match(Set dst (SubL src1 (LShiftL (AndL src2 mask) lshift)));
13974 ins_cost(1.9 * INSN_COST);
13975 format %{ "sub $dst, $src1, $src2, uxtw #lshift" %}
13976
13977 ins_encode %{
13978 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13979 as_Register($src2$$reg), ext::uxtw, ($lshift$$constant));
13980 %}
13981 ins_pipe(ialu_reg_reg_shift);
13982 %}
13983
13984 // This pattern is automatically generated from aarch64_ad.m4
13985 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13986 instruct AddExtI_uxtb_and_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_255 mask, immIExt lshift, rFlagsReg cr)
13987 %{
13988 match(Set dst (AddI src1 (LShiftI (AndI src2 mask) lshift)));
13989 ins_cost(1.9 * INSN_COST);
13990 format %{ "addw $dst, $src1, $src2, uxtb #lshift" %}
13991
13992 ins_encode %{
13993 __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
13994 as_Register($src2$$reg), ext::uxtb, ($lshift$$constant));
13995 %}
13996 ins_pipe(ialu_reg_reg_shift);
13997 %}
13998
13999 // This pattern is automatically generated from aarch64_ad.m4
14000 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
14001 instruct AddExtI_uxth_and_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_65535 mask, immIExt lshift, rFlagsReg cr)
14002 %{
14003 match(Set dst (AddI src1 (LShiftI (AndI src2 mask) lshift)));
14004 ins_cost(1.9 * INSN_COST);
14005 format %{ "addw $dst, $src1, $src2, uxth #lshift" %}
14006
14007 ins_encode %{
14008 __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
14009 as_Register($src2$$reg), ext::uxth, ($lshift$$constant));
14010 %}
14011 ins_pipe(ialu_reg_reg_shift);
14012 %}
14013
14014 // This pattern is automatically generated from aarch64_ad.m4
14015 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
14016 instruct SubExtI_uxtb_and_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_255 mask, immIExt lshift, rFlagsReg cr)
14017 %{
14018 match(Set dst (SubI src1 (LShiftI (AndI src2 mask) lshift)));
14019 ins_cost(1.9 * INSN_COST);
14020 format %{ "subw $dst, $src1, $src2, uxtb #lshift" %}
14021
14022 ins_encode %{
14023 __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
14024 as_Register($src2$$reg), ext::uxtb, ($lshift$$constant));
14025 %}
14026 ins_pipe(ialu_reg_reg_shift);
14027 %}
14028
14029 // This pattern is automatically generated from aarch64_ad.m4
14030 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
14031 instruct SubExtI_uxth_and_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_65535 mask, immIExt lshift, rFlagsReg cr)
14032 %{
14033 match(Set dst (SubI src1 (LShiftI (AndI src2 mask) lshift)));
14034 ins_cost(1.9 * INSN_COST);
14035 format %{ "subw $dst, $src1, $src2, uxth #lshift" %}
14036
14037 ins_encode %{
14038 __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
14039 as_Register($src2$$reg), ext::uxth, ($lshift$$constant));
14040 %}
14041 ins_pipe(ialu_reg_reg_shift);
14042 %}
14043
14044
14045
14046 // END This section of the file is automatically generated. Do not edit --------------
14047
14048
14049 // ============================================================================
14050 // Floating Point Arithmetic Instructions
14051
14052 instruct addF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
14053 match(Set dst (AddF src1 src2));
14054
14055 ins_cost(INSN_COST * 5);
14056 format %{ "fadds $dst, $src1, $src2" %}
14057
14058 ins_encode %{
14059 __ fadds(as_FloatRegister($dst$$reg),
14060 as_FloatRegister($src1$$reg),
14061 as_FloatRegister($src2$$reg));
14062 %}
14063
14064 ins_pipe(fp_dop_reg_reg_s);
14065 %}
14066
14067 instruct addD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
14068 match(Set dst (AddD src1 src2));
14069
14070 ins_cost(INSN_COST * 5);
14071 format %{ "faddd $dst, $src1, $src2" %}
14072
14073 ins_encode %{
14074 __ faddd(as_FloatRegister($dst$$reg),
14075 as_FloatRegister($src1$$reg),
14076 as_FloatRegister($src2$$reg));
14077 %}
14078
14079 ins_pipe(fp_dop_reg_reg_d);
14080 %}
14081
14082 instruct subF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
14083 match(Set dst (SubF src1 src2));
14084
14085 ins_cost(INSN_COST * 5);
14086 format %{ "fsubs $dst, $src1, $src2" %}
14087
14088 ins_encode %{
14089 __ fsubs(as_FloatRegister($dst$$reg),
14090 as_FloatRegister($src1$$reg),
14091 as_FloatRegister($src2$$reg));
14092 %}
14093
14094 ins_pipe(fp_dop_reg_reg_s);
14095 %}
14096
14097 instruct subD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
14098 match(Set dst (SubD src1 src2));
14099
14100 ins_cost(INSN_COST * 5);
14101 format %{ "fsubd $dst, $src1, $src2" %}
14102
14103 ins_encode %{
14104 __ fsubd(as_FloatRegister($dst$$reg),
14105 as_FloatRegister($src1$$reg),
14106 as_FloatRegister($src2$$reg));
14107 %}
14108
14109 ins_pipe(fp_dop_reg_reg_d);
14110 %}
14111
14112 instruct mulF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
14113 match(Set dst (MulF src1 src2));
14114
14115 ins_cost(INSN_COST * 6);
14116 format %{ "fmuls $dst, $src1, $src2" %}
14117
14118 ins_encode %{
14119 __ fmuls(as_FloatRegister($dst$$reg),
14120 as_FloatRegister($src1$$reg),
14121 as_FloatRegister($src2$$reg));
14122 %}
14123
14124 ins_pipe(fp_dop_reg_reg_s);
14125 %}
14126
14127 instruct mulD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
14128 match(Set dst (MulD src1 src2));
14129
14130 ins_cost(INSN_COST * 6);
14131 format %{ "fmuld $dst, $src1, $src2" %}
14132
14133 ins_encode %{
14134 __ fmuld(as_FloatRegister($dst$$reg),
14135 as_FloatRegister($src1$$reg),
14136 as_FloatRegister($src2$$reg));
14137 %}
14138
14139 ins_pipe(fp_dop_reg_reg_d);
14140 %}
14141
14142 // src1 * src2 + src3
14143 instruct maddF_reg_reg(vRegF dst, vRegF src1, vRegF src2, vRegF src3) %{
14144 predicate(UseFMA);
14145 match(Set dst (FmaF src3 (Binary src1 src2)));
14146
14147 format %{ "fmadds $dst, $src1, $src2, $src3" %}
14148
14149 ins_encode %{
14150 __ fmadds(as_FloatRegister($dst$$reg),
14151 as_FloatRegister($src1$$reg),
14152 as_FloatRegister($src2$$reg),
14153 as_FloatRegister($src3$$reg));
14154 %}
14155
14156 ins_pipe(pipe_class_default);
14157 %}
14158
14159 // src1 * src2 + src3
14160 instruct maddD_reg_reg(vRegD dst, vRegD src1, vRegD src2, vRegD src3) %{
14161 predicate(UseFMA);
14162 match(Set dst (FmaD src3 (Binary src1 src2)));
14163
14164 format %{ "fmaddd $dst, $src1, $src2, $src3" %}
14165
14166 ins_encode %{
14167 __ fmaddd(as_FloatRegister($dst$$reg),
14168 as_FloatRegister($src1$$reg),
14169 as_FloatRegister($src2$$reg),
14170 as_FloatRegister($src3$$reg));
14171 %}
14172
14173 ins_pipe(pipe_class_default);
14174 %}
14175
14176 // -src1 * src2 + src3
14177 instruct msubF_reg_reg(vRegF dst, vRegF src1, vRegF src2, vRegF src3) %{
14178 predicate(UseFMA);
14179 match(Set dst (FmaF src3 (Binary (NegF src1) src2)));
14180 match(Set dst (FmaF src3 (Binary src1 (NegF src2))));
14181
14182 format %{ "fmsubs $dst, $src1, $src2, $src3" %}
14183
14184 ins_encode %{
14185 __ fmsubs(as_FloatRegister($dst$$reg),
14186 as_FloatRegister($src1$$reg),
14187 as_FloatRegister($src2$$reg),
14188 as_FloatRegister($src3$$reg));
14189 %}
14190
14191 ins_pipe(pipe_class_default);
14192 %}
14193
14194 // -src1 * src2 + src3
14195 instruct msubD_reg_reg(vRegD dst, vRegD src1, vRegD src2, vRegD src3) %{
14196 predicate(UseFMA);
14197 match(Set dst (FmaD src3 (Binary (NegD src1) src2)));
14198 match(Set dst (FmaD src3 (Binary src1 (NegD src2))));
14199
14200 format %{ "fmsubd $dst, $src1, $src2, $src3" %}
14201
14202 ins_encode %{
14203 __ fmsubd(as_FloatRegister($dst$$reg),
14204 as_FloatRegister($src1$$reg),
14205 as_FloatRegister($src2$$reg),
14206 as_FloatRegister($src3$$reg));
14207 %}
14208
14209 ins_pipe(pipe_class_default);
14210 %}
14211
14212 // -src1 * src2 - src3
14213 instruct mnaddF_reg_reg(vRegF dst, vRegF src1, vRegF src2, vRegF src3) %{
14214 predicate(UseFMA);
14215 match(Set dst (FmaF (NegF src3) (Binary (NegF src1) src2)));
14216 match(Set dst (FmaF (NegF src3) (Binary src1 (NegF src2))));
14217
14218 format %{ "fnmadds $dst, $src1, $src2, $src3" %}
14219
14220 ins_encode %{
14221 __ fnmadds(as_FloatRegister($dst$$reg),
14222 as_FloatRegister($src1$$reg),
14223 as_FloatRegister($src2$$reg),
14224 as_FloatRegister($src3$$reg));
14225 %}
14226
14227 ins_pipe(pipe_class_default);
14228 %}
14229
14230 // -src1 * src2 - src3
14231 instruct mnaddD_reg_reg(vRegD dst, vRegD src1, vRegD src2, vRegD src3) %{
14232 predicate(UseFMA);
14233 match(Set dst (FmaD (NegD src3) (Binary (NegD src1) src2)));
14234 match(Set dst (FmaD (NegD src3) (Binary src1 (NegD src2))));
14235
14236 format %{ "fnmaddd $dst, $src1, $src2, $src3" %}
14237
14238 ins_encode %{
14239 __ fnmaddd(as_FloatRegister($dst$$reg),
14240 as_FloatRegister($src1$$reg),
14241 as_FloatRegister($src2$$reg),
14242 as_FloatRegister($src3$$reg));
14243 %}
14244
14245 ins_pipe(pipe_class_default);
14246 %}
14247
14248 // src1 * src2 - src3
14249 instruct mnsubF_reg_reg(vRegF dst, vRegF src1, vRegF src2, vRegF src3, immF0 zero) %{
14250 predicate(UseFMA);
14251 match(Set dst (FmaF (NegF src3) (Binary src1 src2)));
14252
14253 format %{ "fnmsubs $dst, $src1, $src2, $src3" %}
14254
14255 ins_encode %{
14256 __ fnmsubs(as_FloatRegister($dst$$reg),
14257 as_FloatRegister($src1$$reg),
14258 as_FloatRegister($src2$$reg),
14259 as_FloatRegister($src3$$reg));
14260 %}
14261
14262 ins_pipe(pipe_class_default);
14263 %}
14264
14265 // src1 * src2 - src3
14266 instruct mnsubD_reg_reg(vRegD dst, vRegD src1, vRegD src2, vRegD src3, immD0 zero) %{
14267 predicate(UseFMA);
14268 match(Set dst (FmaD (NegD src3) (Binary src1 src2)));
14269
14270 format %{ "fnmsubd $dst, $src1, $src2, $src3" %}
14271
14272 ins_encode %{
14273 // n.b. insn name should be fnmsubd
14274 __ fnmsub(as_FloatRegister($dst$$reg),
14275 as_FloatRegister($src1$$reg),
14276 as_FloatRegister($src2$$reg),
14277 as_FloatRegister($src3$$reg));
14278 %}
14279
14280 ins_pipe(pipe_class_default);
14281 %}
14282
14283
14284 // Math.max(FF)F
14285 instruct maxF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
14286 match(Set dst (MaxF src1 src2));
14287
14288 format %{ "fmaxs $dst, $src1, $src2" %}
14289 ins_encode %{
14290 __ fmaxs(as_FloatRegister($dst$$reg),
14291 as_FloatRegister($src1$$reg),
14292 as_FloatRegister($src2$$reg));
14293 %}
14294
14295 ins_pipe(fp_dop_reg_reg_s);
14296 %}
14297
14298 // Math.min(FF)F
14299 instruct minF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
14300 match(Set dst (MinF src1 src2));
14301
14302 format %{ "fmins $dst, $src1, $src2" %}
14303 ins_encode %{
14304 __ fmins(as_FloatRegister($dst$$reg),
14305 as_FloatRegister($src1$$reg),
14306 as_FloatRegister($src2$$reg));
14307 %}
14308
14309 ins_pipe(fp_dop_reg_reg_s);
14310 %}
14311
14312 // Math.max(DD)D
14313 instruct maxD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
14314 match(Set dst (MaxD src1 src2));
14315
14316 format %{ "fmaxd $dst, $src1, $src2" %}
14317 ins_encode %{
14318 __ fmaxd(as_FloatRegister($dst$$reg),
14319 as_FloatRegister($src1$$reg),
14320 as_FloatRegister($src2$$reg));
14321 %}
14322
14323 ins_pipe(fp_dop_reg_reg_d);
14324 %}
14325
14326 // Math.min(DD)D
14327 instruct minD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
14328 match(Set dst (MinD src1 src2));
14329
14330 format %{ "fmind $dst, $src1, $src2" %}
14331 ins_encode %{
14332 __ fmind(as_FloatRegister($dst$$reg),
14333 as_FloatRegister($src1$$reg),
14334 as_FloatRegister($src2$$reg));
14335 %}
14336
14337 ins_pipe(fp_dop_reg_reg_d);
14338 %}
14339
14340
14341 instruct divF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
14342 match(Set dst (DivF src1 src2));
14343
14344 ins_cost(INSN_COST * 18);
14345 format %{ "fdivs $dst, $src1, $src2" %}
14346
14347 ins_encode %{
14348 __ fdivs(as_FloatRegister($dst$$reg),
14349 as_FloatRegister($src1$$reg),
14350 as_FloatRegister($src2$$reg));
14351 %}
14352
14353 ins_pipe(fp_div_s);
14354 %}
14355
14356 instruct divD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
14357 match(Set dst (DivD src1 src2));
14358
14359 ins_cost(INSN_COST * 32);
14360 format %{ "fdivd $dst, $src1, $src2" %}
14361
14362 ins_encode %{
14363 __ fdivd(as_FloatRegister($dst$$reg),
14364 as_FloatRegister($src1$$reg),
14365 as_FloatRegister($src2$$reg));
14366 %}
14367
14368 ins_pipe(fp_div_d);
14369 %}
14370
14371 instruct negF_reg_reg(vRegF dst, vRegF src) %{
14372 match(Set dst (NegF src));
14373
14374 ins_cost(INSN_COST * 3);
14375 format %{ "fneg $dst, $src" %}
14376
14377 ins_encode %{
14378 __ fnegs(as_FloatRegister($dst$$reg),
14379 as_FloatRegister($src$$reg));
14380 %}
14381
14382 ins_pipe(fp_uop_s);
14383 %}
14384
14385 instruct negD_reg_reg(vRegD dst, vRegD src) %{
14386 match(Set dst (NegD src));
14387
14388 ins_cost(INSN_COST * 3);
14389 format %{ "fnegd $dst, $src" %}
14390
14391 ins_encode %{
14392 __ fnegd(as_FloatRegister($dst$$reg),
14393 as_FloatRegister($src$$reg));
14394 %}
14395
14396 ins_pipe(fp_uop_d);
14397 %}
14398
14399 instruct absI_reg(iRegINoSp dst, iRegIorL2I src, rFlagsReg cr)
14400 %{
14401 match(Set dst (AbsI src));
14402
14403 effect(KILL cr);
14404 ins_cost(INSN_COST * 2);
14405 format %{ "cmpw $src, zr\n\t"
14406 "cnegw $dst, $src, Assembler::LT\t# int abs"
14407 %}
14408
14409 ins_encode %{
14410 __ cmpw(as_Register($src$$reg), zr);
14411 __ cnegw(as_Register($dst$$reg), as_Register($src$$reg), Assembler::LT);
14412 %}
14413 ins_pipe(pipe_class_default);
14414 %}
14415
14416 instruct absL_reg(iRegLNoSp dst, iRegL src, rFlagsReg cr)
14417 %{
14418 match(Set dst (AbsL src));
14419
14420 effect(KILL cr);
14421 ins_cost(INSN_COST * 2);
14422 format %{ "cmp $src, zr\n\t"
14423 "cneg $dst, $src, Assembler::LT\t# long abs"
14424 %}
14425
14426 ins_encode %{
14427 __ cmp(as_Register($src$$reg), zr);
14428 __ cneg(as_Register($dst$$reg), as_Register($src$$reg), Assembler::LT);
14429 %}
14430 ins_pipe(pipe_class_default);
14431 %}
14432
14433 instruct absF_reg(vRegF dst, vRegF src) %{
14434 match(Set dst (AbsF src));
14435
14436 ins_cost(INSN_COST * 3);
14437 format %{ "fabss $dst, $src" %}
14438 ins_encode %{
14439 __ fabss(as_FloatRegister($dst$$reg),
14440 as_FloatRegister($src$$reg));
14441 %}
14442
14443 ins_pipe(fp_uop_s);
14444 %}
14445
14446 instruct absD_reg(vRegD dst, vRegD src) %{
14447 match(Set dst (AbsD src));
14448
14449 ins_cost(INSN_COST * 3);
14450 format %{ "fabsd $dst, $src" %}
14451 ins_encode %{
14452 __ fabsd(as_FloatRegister($dst$$reg),
14453 as_FloatRegister($src$$reg));
14454 %}
14455
14456 ins_pipe(fp_uop_d);
14457 %}
14458
14459 instruct absdF_reg(vRegF dst, vRegF src1, vRegF src2) %{
14460 match(Set dst (AbsF (SubF src1 src2)));
14461
14462 ins_cost(INSN_COST * 3);
14463 format %{ "fabds $dst, $src1, $src2" %}
14464 ins_encode %{
14465 __ fabds(as_FloatRegister($dst$$reg),
14466 as_FloatRegister($src1$$reg),
14467 as_FloatRegister($src2$$reg));
14468 %}
14469
14470 ins_pipe(fp_uop_s);
14471 %}
14472
14473 instruct absdD_reg(vRegD dst, vRegD src1, vRegD src2) %{
14474 match(Set dst (AbsD (SubD src1 src2)));
14475
14476 ins_cost(INSN_COST * 3);
14477 format %{ "fabdd $dst, $src1, $src2" %}
14478 ins_encode %{
14479 __ fabdd(as_FloatRegister($dst$$reg),
14480 as_FloatRegister($src1$$reg),
14481 as_FloatRegister($src2$$reg));
14482 %}
14483
14484 ins_pipe(fp_uop_d);
14485 %}
14486
14487 instruct sqrtD_reg(vRegD dst, vRegD src) %{
14488 match(Set dst (SqrtD src));
14489
14490 ins_cost(INSN_COST * 50);
14491 format %{ "fsqrtd $dst, $src" %}
14492 ins_encode %{
14493 __ fsqrtd(as_FloatRegister($dst$$reg),
14494 as_FloatRegister($src$$reg));
14495 %}
14496
14497 ins_pipe(fp_div_s);
14498 %}
14499
14500 instruct sqrtF_reg(vRegF dst, vRegF src) %{
14501 match(Set dst (SqrtF src));
14502
14503 ins_cost(INSN_COST * 50);
14504 format %{ "fsqrts $dst, $src" %}
14505 ins_encode %{
14506 __ fsqrts(as_FloatRegister($dst$$reg),
14507 as_FloatRegister($src$$reg));
14508 %}
14509
14510 ins_pipe(fp_div_d);
14511 %}
14512
14513 // Math.rint, floor, ceil
14514 instruct roundD_reg(vRegD dst, vRegD src, immI rmode) %{
14515 match(Set dst (RoundDoubleMode src rmode));
14516 format %{ "frint $dst, $src, $rmode" %}
14517 ins_encode %{
14518 switch ($rmode$$constant) {
14519 case RoundDoubleModeNode::rmode_rint:
14520 __ frintnd(as_FloatRegister($dst$$reg),
14521 as_FloatRegister($src$$reg));
14522 break;
14523 case RoundDoubleModeNode::rmode_floor:
14524 __ frintmd(as_FloatRegister($dst$$reg),
14525 as_FloatRegister($src$$reg));
14526 break;
14527 case RoundDoubleModeNode::rmode_ceil:
14528 __ frintpd(as_FloatRegister($dst$$reg),
14529 as_FloatRegister($src$$reg));
14530 break;
14531 }
14532 %}
14533 ins_pipe(fp_uop_d);
14534 %}
14535
14536 instruct copySignD_reg(vRegD dst, vRegD src1, vRegD src2, vRegD zero) %{
14537 match(Set dst (CopySignD src1 (Binary src2 zero)));
14538 effect(TEMP_DEF dst, USE src1, USE src2, USE zero);
14539 format %{ "CopySignD $dst $src1 $src2" %}
14540 ins_encode %{
14541 FloatRegister dst = as_FloatRegister($dst$$reg),
14542 src1 = as_FloatRegister($src1$$reg),
14543 src2 = as_FloatRegister($src2$$reg),
14544 zero = as_FloatRegister($zero$$reg);
14545 __ fnegd(dst, zero);
14546 __ bsl(dst, __ T8B, src2, src1);
14547 %}
14548 ins_pipe(fp_uop_d);
14549 %}
14550
14551 instruct copySignF_reg(vRegF dst, vRegF src1, vRegF src2) %{
14552 match(Set dst (CopySignF src1 src2));
14553 effect(TEMP_DEF dst, USE src1, USE src2);
14554 format %{ "CopySignF $dst $src1 $src2" %}
14555 ins_encode %{
14556 FloatRegister dst = as_FloatRegister($dst$$reg),
14557 src1 = as_FloatRegister($src1$$reg),
14558 src2 = as_FloatRegister($src2$$reg);
14559 __ movi(dst, __ T2S, 0x80, 24);
14560 __ bsl(dst, __ T8B, src2, src1);
14561 %}
14562 ins_pipe(fp_uop_d);
14563 %}
14564
14565 instruct signumD_reg(vRegD dst, vRegD src, vRegD zero, vRegD one) %{
14566 match(Set dst (SignumD src (Binary zero one)));
14567 effect(TEMP_DEF dst, USE src, USE zero, USE one);
14568 format %{ "signumD $dst, $src" %}
14569 ins_encode %{
14570 FloatRegister src = as_FloatRegister($src$$reg),
14571 dst = as_FloatRegister($dst$$reg),
14572 zero = as_FloatRegister($zero$$reg),
14573 one = as_FloatRegister($one$$reg);
14574 __ facgtd(dst, src, zero); // dst=0 for +-0.0 and NaN. 0xFFF..F otherwise
14575 __ ushrd(dst, dst, 1); // dst=0 for +-0.0 and NaN. 0x7FF..F otherwise
14576 // Bit selection instruction gets bit from "one" for each enabled bit in
14577 // "dst", otherwise gets a bit from "src". For "src" that contains +-0.0 or
14578 // NaN the whole "src" will be copied because "dst" is zero. For all other
14579 // "src" values dst is 0x7FF..F, which means only the sign bit is copied
14580 // from "src", and all other bits are copied from 1.0.
14581 __ bsl(dst, __ T8B, one, src);
14582 %}
14583 ins_pipe(fp_uop_d);
14584 %}
14585
14586 instruct signumF_reg(vRegF dst, vRegF src, vRegF zero, vRegF one) %{
14587 match(Set dst (SignumF src (Binary zero one)));
14588 effect(TEMP_DEF dst, USE src, USE zero, USE one);
14589 format %{ "signumF $dst, $src" %}
14590 ins_encode %{
14591 FloatRegister src = as_FloatRegister($src$$reg),
14592 dst = as_FloatRegister($dst$$reg),
14593 zero = as_FloatRegister($zero$$reg),
14594 one = as_FloatRegister($one$$reg);
14595 __ facgts(dst, src, zero); // dst=0 for +-0.0 and NaN. 0xFFF..F otherwise
14596 __ ushr(dst, __ T2S, dst, 1); // dst=0 for +-0.0 and NaN. 0x7FF..F otherwise
14597 // Bit selection instruction gets bit from "one" for each enabled bit in
14598 // "dst", otherwise gets a bit from "src". For "src" that contains +-0.0 or
14599 // NaN the whole "src" will be copied because "dst" is zero. For all other
14600 // "src" values dst is 0x7FF..F, which means only the sign bit is copied
14601 // from "src", and all other bits are copied from 1.0.
14602 __ bsl(dst, __ T8B, one, src);
14603 %}
14604 ins_pipe(fp_uop_d);
14605 %}
14606
14607 instruct onspinwait() %{
14608 match(OnSpinWait);
14609 ins_cost(INSN_COST);
14610
14611 format %{ "onspinwait" %}
14612
14613 ins_encode %{
14614 __ spin_wait();
14615 %}
14616 ins_pipe(pipe_class_empty);
14617 %}
14618
14619 // ============================================================================
14620 // Logical Instructions
14621
14622 // Integer Logical Instructions
14623
14624 // And Instructions
14625
14626
14627 instruct andI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, rFlagsReg cr) %{
14628 match(Set dst (AndI src1 src2));
14629
14630 format %{ "andw $dst, $src1, $src2\t# int" %}
14631
14632 ins_cost(INSN_COST);
14633 ins_encode %{
14634 __ andw(as_Register($dst$$reg),
14635 as_Register($src1$$reg),
14636 as_Register($src2$$reg));
14637 %}
14638
14639 ins_pipe(ialu_reg_reg);
14640 %}
14641
14642 instruct andI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immILog src2, rFlagsReg cr) %{
14643 match(Set dst (AndI src1 src2));
14644
14645 format %{ "andsw $dst, $src1, $src2\t# int" %}
14646
14647 ins_cost(INSN_COST);
14648 ins_encode %{
14649 __ andw(as_Register($dst$$reg),
14650 as_Register($src1$$reg),
14651 (uint64_t)($src2$$constant));
14652 %}
14653
14654 ins_pipe(ialu_reg_imm);
14655 %}
14656
14657 // Or Instructions
14658
14659 instruct orI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
14660 match(Set dst (OrI src1 src2));
14661
14662 format %{ "orrw $dst, $src1, $src2\t# int" %}
14663
14664 ins_cost(INSN_COST);
14665 ins_encode %{
14666 __ orrw(as_Register($dst$$reg),
14667 as_Register($src1$$reg),
14668 as_Register($src2$$reg));
14669 %}
14670
14671 ins_pipe(ialu_reg_reg);
14672 %}
14673
14674 instruct orI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immILog src2) %{
14675 match(Set dst (OrI src1 src2));
14676
14677 format %{ "orrw $dst, $src1, $src2\t# int" %}
14678
14679 ins_cost(INSN_COST);
14680 ins_encode %{
14681 __ orrw(as_Register($dst$$reg),
14682 as_Register($src1$$reg),
14683 (uint64_t)($src2$$constant));
14684 %}
14685
14686 ins_pipe(ialu_reg_imm);
14687 %}
14688
14689 // Xor Instructions
14690
14691 instruct xorI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
14692 match(Set dst (XorI src1 src2));
14693
14694 format %{ "eorw $dst, $src1, $src2\t# int" %}
14695
14696 ins_cost(INSN_COST);
14697 ins_encode %{
14698 __ eorw(as_Register($dst$$reg),
14699 as_Register($src1$$reg),
14700 as_Register($src2$$reg));
14701 %}
14702
14703 ins_pipe(ialu_reg_reg);
14704 %}
14705
14706 instruct xorI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immILog src2) %{
14707 match(Set dst (XorI src1 src2));
14708
14709 format %{ "eorw $dst, $src1, $src2\t# int" %}
14710
14711 ins_cost(INSN_COST);
14712 ins_encode %{
14713 __ eorw(as_Register($dst$$reg),
14714 as_Register($src1$$reg),
14715 (uint64_t)($src2$$constant));
14716 %}
14717
14718 ins_pipe(ialu_reg_imm);
14719 %}
14720
14721 // Long Logical Instructions
14722 // TODO
14723
14724 instruct andL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2, rFlagsReg cr) %{
14725 match(Set dst (AndL src1 src2));
14726
14727 format %{ "and $dst, $src1, $src2\t# int" %}
14728
14729 ins_cost(INSN_COST);
14730 ins_encode %{
14731 __ andr(as_Register($dst$$reg),
14732 as_Register($src1$$reg),
14733 as_Register($src2$$reg));
14734 %}
14735
14736 ins_pipe(ialu_reg_reg);
14737 %}
14738
14739 instruct andL_reg_imm(iRegLNoSp dst, iRegL src1, immLLog src2, rFlagsReg cr) %{
14740 match(Set dst (AndL src1 src2));
14741
14742 format %{ "and $dst, $src1, $src2\t# int" %}
14743
14744 ins_cost(INSN_COST);
14745 ins_encode %{
14746 __ andr(as_Register($dst$$reg),
14747 as_Register($src1$$reg),
14748 (uint64_t)($src2$$constant));
14749 %}
14750
14751 ins_pipe(ialu_reg_imm);
14752 %}
14753
14754 // Or Instructions
14755
14756 instruct orL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
14757 match(Set dst (OrL src1 src2));
14758
14759 format %{ "orr $dst, $src1, $src2\t# int" %}
14760
14761 ins_cost(INSN_COST);
14762 ins_encode %{
14763 __ orr(as_Register($dst$$reg),
14764 as_Register($src1$$reg),
14765 as_Register($src2$$reg));
14766 %}
14767
14768 ins_pipe(ialu_reg_reg);
14769 %}
14770
14771 instruct orL_reg_imm(iRegLNoSp dst, iRegL src1, immLLog src2) %{
14772 match(Set dst (OrL src1 src2));
14773
14774 format %{ "orr $dst, $src1, $src2\t# int" %}
14775
14776 ins_cost(INSN_COST);
14777 ins_encode %{
14778 __ orr(as_Register($dst$$reg),
14779 as_Register($src1$$reg),
14780 (uint64_t)($src2$$constant));
14781 %}
14782
14783 ins_pipe(ialu_reg_imm);
14784 %}
14785
14786 // Xor Instructions
14787
14788 instruct xorL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
14789 match(Set dst (XorL src1 src2));
14790
14791 format %{ "eor $dst, $src1, $src2\t# int" %}
14792
14793 ins_cost(INSN_COST);
14794 ins_encode %{
14795 __ eor(as_Register($dst$$reg),
14796 as_Register($src1$$reg),
14797 as_Register($src2$$reg));
14798 %}
14799
14800 ins_pipe(ialu_reg_reg);
14801 %}
14802
14803 instruct xorL_reg_imm(iRegLNoSp dst, iRegL src1, immLLog src2) %{
14804 match(Set dst (XorL src1 src2));
14805
14806 ins_cost(INSN_COST);
14807 format %{ "eor $dst, $src1, $src2\t# int" %}
14808
14809 ins_encode %{
14810 __ eor(as_Register($dst$$reg),
14811 as_Register($src1$$reg),
14812 (uint64_t)($src2$$constant));
14813 %}
14814
14815 ins_pipe(ialu_reg_imm);
14816 %}
14817
14818 instruct convI2L_reg_reg(iRegLNoSp dst, iRegIorL2I src)
14819 %{
14820 match(Set dst (ConvI2L src));
14821
14822 ins_cost(INSN_COST);
14823 format %{ "sxtw $dst, $src\t# i2l" %}
14824 ins_encode %{
14825 __ sbfm($dst$$Register, $src$$Register, 0, 31);
14826 %}
14827 ins_pipe(ialu_reg_shift);
14828 %}
14829
14830 // this pattern occurs in bigmath arithmetic
14831 instruct convUI2L_reg_reg(iRegLNoSp dst, iRegIorL2I src, immL_32bits mask)
14832 %{
14833 match(Set dst (AndL (ConvI2L src) mask));
14834
14835 ins_cost(INSN_COST);
14836 format %{ "ubfm $dst, $src, 0, 31\t# ui2l" %}
14837 ins_encode %{
14838 __ ubfm($dst$$Register, $src$$Register, 0, 31);
14839 %}
14840
14841 ins_pipe(ialu_reg_shift);
14842 %}
14843
14844 instruct convL2I_reg(iRegINoSp dst, iRegL src) %{
14845 match(Set dst (ConvL2I src));
14846
14847 ins_cost(INSN_COST);
14848 format %{ "movw $dst, $src \t// l2i" %}
14849
14850 ins_encode %{
14851 __ movw(as_Register($dst$$reg), as_Register($src$$reg));
14852 %}
14853
14854 ins_pipe(ialu_reg);
14855 %}
14856
14857 instruct convI2B(iRegINoSp dst, iRegIorL2I src, rFlagsReg cr)
14858 %{
14859 match(Set dst (Conv2B src));
14860 effect(KILL cr);
14861
14862 format %{
14863 "cmpw $src, zr\n\t"
14864 "cset $dst, ne"
14865 %}
14866
14867 ins_encode %{
14868 __ cmpw(as_Register($src$$reg), zr);
14869 __ cset(as_Register($dst$$reg), Assembler::NE);
14870 %}
14871
14872 ins_pipe(ialu_reg);
14873 %}
14874
14875 instruct convP2B(iRegINoSp dst, iRegP src, rFlagsReg cr)
14876 %{
14877 match(Set dst (Conv2B src));
14878 effect(KILL cr);
14879
14880 format %{
14881 "cmp $src, zr\n\t"
14882 "cset $dst, ne"
14883 %}
14884
14885 ins_encode %{
14886 __ cmp(as_Register($src$$reg), zr);
14887 __ cset(as_Register($dst$$reg), Assembler::NE);
14888 %}
14889
14890 ins_pipe(ialu_reg);
14891 %}
14892
14893 instruct convD2F_reg(vRegF dst, vRegD src) %{
14894 match(Set dst (ConvD2F src));
14895
14896 ins_cost(INSN_COST * 5);
14897 format %{ "fcvtd $dst, $src \t// d2f" %}
14898
14899 ins_encode %{
14900 __ fcvtd(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg));
14901 %}
14902
14903 ins_pipe(fp_d2f);
14904 %}
14905
14906 instruct convF2D_reg(vRegD dst, vRegF src) %{
14907 match(Set dst (ConvF2D src));
14908
14909 ins_cost(INSN_COST * 5);
14910 format %{ "fcvts $dst, $src \t// f2d" %}
14911
14912 ins_encode %{
14913 __ fcvts(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg));
14914 %}
14915
14916 ins_pipe(fp_f2d);
14917 %}
14918
14919 instruct convF2I_reg_reg(iRegINoSp dst, vRegF src) %{
14920 match(Set dst (ConvF2I src));
14921
14922 ins_cost(INSN_COST * 5);
14923 format %{ "fcvtzsw $dst, $src \t// f2i" %}
14924
14925 ins_encode %{
14926 __ fcvtzsw(as_Register($dst$$reg), as_FloatRegister($src$$reg));
14927 %}
14928
14929 ins_pipe(fp_f2i);
14930 %}
14931
14932 instruct convF2L_reg_reg(iRegLNoSp dst, vRegF src) %{
14933 match(Set dst (ConvF2L src));
14934
14935 ins_cost(INSN_COST * 5);
14936 format %{ "fcvtzs $dst, $src \t// f2l" %}
14937
14938 ins_encode %{
14939 __ fcvtzs(as_Register($dst$$reg), as_FloatRegister($src$$reg));
14940 %}
14941
14942 ins_pipe(fp_f2l);
14943 %}
14944
14945 instruct convI2F_reg_reg(vRegF dst, iRegIorL2I src) %{
14946 match(Set dst (ConvI2F src));
14947
14948 ins_cost(INSN_COST * 5);
14949 format %{ "scvtfws $dst, $src \t// i2f" %}
14950
14951 ins_encode %{
14952 __ scvtfws(as_FloatRegister($dst$$reg), as_Register($src$$reg));
14953 %}
14954
14955 ins_pipe(fp_i2f);
14956 %}
14957
14958 instruct convL2F_reg_reg(vRegF dst, iRegL src) %{
14959 match(Set dst (ConvL2F src));
14960
14961 ins_cost(INSN_COST * 5);
14962 format %{ "scvtfs $dst, $src \t// l2f" %}
14963
14964 ins_encode %{
14965 __ scvtfs(as_FloatRegister($dst$$reg), as_Register($src$$reg));
14966 %}
14967
14968 ins_pipe(fp_l2f);
14969 %}
14970
14971 instruct convD2I_reg_reg(iRegINoSp dst, vRegD src) %{
14972 match(Set dst (ConvD2I src));
14973
14974 ins_cost(INSN_COST * 5);
14975 format %{ "fcvtzdw $dst, $src \t// d2i" %}
14976
14977 ins_encode %{
14978 __ fcvtzdw(as_Register($dst$$reg), as_FloatRegister($src$$reg));
14979 %}
14980
14981 ins_pipe(fp_d2i);
14982 %}
14983
14984 instruct convD2L_reg_reg(iRegLNoSp dst, vRegD src) %{
14985 match(Set dst (ConvD2L src));
14986
14987 ins_cost(INSN_COST * 5);
14988 format %{ "fcvtzd $dst, $src \t// d2l" %}
14989
14990 ins_encode %{
14991 __ fcvtzd(as_Register($dst$$reg), as_FloatRegister($src$$reg));
14992 %}
14993
14994 ins_pipe(fp_d2l);
14995 %}
14996
14997 instruct convI2D_reg_reg(vRegD dst, iRegIorL2I src) %{
14998 match(Set dst (ConvI2D src));
14999
15000 ins_cost(INSN_COST * 5);
15001 format %{ "scvtfwd $dst, $src \t// i2d" %}
15002
15003 ins_encode %{
15004 __ scvtfwd(as_FloatRegister($dst$$reg), as_Register($src$$reg));
15005 %}
15006
15007 ins_pipe(fp_i2d);
15008 %}
15009
15010 instruct convL2D_reg_reg(vRegD dst, iRegL src) %{
15011 match(Set dst (ConvL2D src));
15012
15013 ins_cost(INSN_COST * 5);
15014 format %{ "scvtfd $dst, $src \t// l2d" %}
15015
15016 ins_encode %{
15017 __ scvtfd(as_FloatRegister($dst$$reg), as_Register($src$$reg));
15018 %}
15019
15020 ins_pipe(fp_l2d);
15021 %}
15022
15023 // stack <-> reg and reg <-> reg shuffles with no conversion
15024
15025 instruct MoveF2I_stack_reg(iRegINoSp dst, stackSlotF src) %{
15026
15027 match(Set dst (MoveF2I src));
15028
15029 effect(DEF dst, USE src);
15030
15031 ins_cost(4 * INSN_COST);
15032
15033 format %{ "ldrw $dst, $src\t# MoveF2I_stack_reg" %}
15034
15035 ins_encode %{
15036 __ ldrw($dst$$Register, Address(sp, $src$$disp));
15037 %}
15038
15039 ins_pipe(iload_reg_reg);
15040
15041 %}
15042
15043 instruct MoveI2F_stack_reg(vRegF dst, stackSlotI src) %{
15044
15045 match(Set dst (MoveI2F src));
15046
15047 effect(DEF dst, USE src);
15048
15049 ins_cost(4 * INSN_COST);
15050
15051 format %{ "ldrs $dst, $src\t# MoveI2F_stack_reg" %}
15052
15053 ins_encode %{
15054 __ ldrs(as_FloatRegister($dst$$reg), Address(sp, $src$$disp));
15055 %}
15056
15057 ins_pipe(pipe_class_memory);
15058
15059 %}
15060
15061 instruct MoveD2L_stack_reg(iRegLNoSp dst, stackSlotD src) %{
15062
15063 match(Set dst (MoveD2L src));
15064
15065 effect(DEF dst, USE src);
15066
15067 ins_cost(4 * INSN_COST);
15068
15069 format %{ "ldr $dst, $src\t# MoveD2L_stack_reg" %}
15070
15071 ins_encode %{
15072 __ ldr($dst$$Register, Address(sp, $src$$disp));
15073 %}
15074
15075 ins_pipe(iload_reg_reg);
15076
15077 %}
15078
15079 instruct MoveL2D_stack_reg(vRegD dst, stackSlotL src) %{
15080
15081 match(Set dst (MoveL2D src));
15082
15083 effect(DEF dst, USE src);
15084
15085 ins_cost(4 * INSN_COST);
15086
15087 format %{ "ldrd $dst, $src\t# MoveL2D_stack_reg" %}
15088
15089 ins_encode %{
15090 __ ldrd(as_FloatRegister($dst$$reg), Address(sp, $src$$disp));
15091 %}
15092
15093 ins_pipe(pipe_class_memory);
15094
15095 %}
15096
15097 instruct MoveF2I_reg_stack(stackSlotI dst, vRegF src) %{
15098
15099 match(Set dst (MoveF2I src));
15100
15101 effect(DEF dst, USE src);
15102
15103 ins_cost(INSN_COST);
15104
15105 format %{ "strs $src, $dst\t# MoveF2I_reg_stack" %}
15106
15107 ins_encode %{
15108 __ strs(as_FloatRegister($src$$reg), Address(sp, $dst$$disp));
15109 %}
15110
15111 ins_pipe(pipe_class_memory);
15112
15113 %}
15114
15115 instruct MoveI2F_reg_stack(stackSlotF dst, iRegI src) %{
15116
15117 match(Set dst (MoveI2F src));
15118
15119 effect(DEF dst, USE src);
15120
15121 ins_cost(INSN_COST);
15122
15123 format %{ "strw $src, $dst\t# MoveI2F_reg_stack" %}
15124
15125 ins_encode %{
15126 __ strw($src$$Register, Address(sp, $dst$$disp));
15127 %}
15128
15129 ins_pipe(istore_reg_reg);
15130
15131 %}
15132
15133 instruct MoveD2L_reg_stack(stackSlotL dst, vRegD src) %{
15134
15135 match(Set dst (MoveD2L src));
15136
15137 effect(DEF dst, USE src);
15138
15139 ins_cost(INSN_COST);
15140
15141 format %{ "strd $dst, $src\t# MoveD2L_reg_stack" %}
15142
15143 ins_encode %{
15144 __ strd(as_FloatRegister($src$$reg), Address(sp, $dst$$disp));
15145 %}
15146
15147 ins_pipe(pipe_class_memory);
15148
15149 %}
15150
15151 instruct MoveL2D_reg_stack(stackSlotD dst, iRegL src) %{
15152
15153 match(Set dst (MoveL2D src));
15154
15155 effect(DEF dst, USE src);
15156
15157 ins_cost(INSN_COST);
15158
15159 format %{ "str $src, $dst\t# MoveL2D_reg_stack" %}
15160
15161 ins_encode %{
15162 __ str($src$$Register, Address(sp, $dst$$disp));
15163 %}
15164
15165 ins_pipe(istore_reg_reg);
15166
15167 %}
15168
15169 instruct MoveF2I_reg_reg(iRegINoSp dst, vRegF src) %{
15170
15171 match(Set dst (MoveF2I src));
15172
15173 effect(DEF dst, USE src);
15174
15175 ins_cost(INSN_COST);
15176
15177 format %{ "fmovs $dst, $src\t# MoveF2I_reg_reg" %}
15178
15179 ins_encode %{
15180 __ fmovs($dst$$Register, as_FloatRegister($src$$reg));
15181 %}
15182
15183 ins_pipe(fp_f2i);
15184
15185 %}
15186
15187 instruct MoveI2F_reg_reg(vRegF dst, iRegI src) %{
15188
15189 match(Set dst (MoveI2F src));
15190
15191 effect(DEF dst, USE src);
15192
15193 ins_cost(INSN_COST);
15194
15195 format %{ "fmovs $dst, $src\t# MoveI2F_reg_reg" %}
15196
15197 ins_encode %{
15198 __ fmovs(as_FloatRegister($dst$$reg), $src$$Register);
15199 %}
15200
15201 ins_pipe(fp_i2f);
15202
15203 %}
15204
15205 instruct MoveD2L_reg_reg(iRegLNoSp dst, vRegD src) %{
15206
15207 match(Set dst (MoveD2L src));
15208
15209 effect(DEF dst, USE src);
15210
15211 ins_cost(INSN_COST);
15212
15213 format %{ "fmovd $dst, $src\t# MoveD2L_reg_reg" %}
15214
15215 ins_encode %{
15216 __ fmovd($dst$$Register, as_FloatRegister($src$$reg));
15217 %}
15218
15219 ins_pipe(fp_d2l);
15220
15221 %}
15222
15223 instruct MoveL2D_reg_reg(vRegD dst, iRegL src) %{
15224
15225 match(Set dst (MoveL2D src));
15226
15227 effect(DEF dst, USE src);
15228
15229 ins_cost(INSN_COST);
15230
15231 format %{ "fmovd $dst, $src\t# MoveL2D_reg_reg" %}
15232
15233 ins_encode %{
15234 __ fmovd(as_FloatRegister($dst$$reg), $src$$Register);
15235 %}
15236
15237 ins_pipe(fp_l2d);
15238
15239 %}
15240
15241 // ============================================================================
15242 // clearing of an array
15243
15244 instruct clearArray_reg_reg(iRegL_R11 cnt, iRegP_R10 base, Universe dummy, rFlagsReg cr)
15245 %{
15246 match(Set dummy (ClearArray cnt base));
15247 effect(USE_KILL cnt, USE_KILL base, KILL cr);
15248
15249 ins_cost(4 * INSN_COST);
15250 format %{ "ClearArray $cnt, $base" %}
15251
15252 ins_encode %{
15253 address tpc = __ zero_words($base$$Register, $cnt$$Register);
15254 if (tpc == NULL) {
15255 ciEnv::current()->record_failure("CodeCache is full");
15256 return;
15257 }
15258 %}
15259
15260 ins_pipe(pipe_class_memory);
15261 %}
15262
15263 instruct clearArray_imm_reg(immL cnt, iRegP_R10 base, iRegL_R11 temp, Universe dummy, rFlagsReg cr)
15264 %{
15265 predicate((uint64_t)n->in(2)->get_long()
15266 < (uint64_t)(BlockZeroingLowLimit >> LogBytesPerWord));
15267 match(Set dummy (ClearArray cnt base));
15268 effect(TEMP temp, USE_KILL base, KILL cr);
15269
15270 ins_cost(4 * INSN_COST);
15271 format %{ "ClearArray $cnt, $base" %}
15272
15273 ins_encode %{
15274 address tpc = __ zero_words($base$$Register, (uint64_t)$cnt$$constant);
15275 if (tpc == NULL) {
15276 ciEnv::current()->record_failure("CodeCache is full");
15277 return;
15278 }
15279 %}
15280
15281 ins_pipe(pipe_class_memory);
15282 %}
15283
15284 // ============================================================================
15285 // Overflow Math Instructions
15286
15287 instruct overflowAddI_reg_reg(rFlagsReg cr, iRegIorL2I op1, iRegIorL2I op2)
15288 %{
15289 match(Set cr (OverflowAddI op1 op2));
15290
15291 format %{ "cmnw $op1, $op2\t# overflow check int" %}
15292 ins_cost(INSN_COST);
15293 ins_encode %{
15294 __ cmnw($op1$$Register, $op2$$Register);
15295 %}
15296
15297 ins_pipe(icmp_reg_reg);
15298 %}
15299
15300 instruct overflowAddI_reg_imm(rFlagsReg cr, iRegIorL2I op1, immIAddSub op2)
15301 %{
15302 match(Set cr (OverflowAddI op1 op2));
15303
15304 format %{ "cmnw $op1, $op2\t# overflow check int" %}
15305 ins_cost(INSN_COST);
15306 ins_encode %{
15307 __ cmnw($op1$$Register, $op2$$constant);
15308 %}
15309
15310 ins_pipe(icmp_reg_imm);
15311 %}
15312
15313 instruct overflowAddL_reg_reg(rFlagsReg cr, iRegL op1, iRegL op2)
15314 %{
15315 match(Set cr (OverflowAddL op1 op2));
15316
15317 format %{ "cmn $op1, $op2\t# overflow check long" %}
15318 ins_cost(INSN_COST);
15319 ins_encode %{
15320 __ cmn($op1$$Register, $op2$$Register);
15321 %}
15322
15323 ins_pipe(icmp_reg_reg);
15324 %}
15325
15326 instruct overflowAddL_reg_imm(rFlagsReg cr, iRegL op1, immLAddSub op2)
15327 %{
15328 match(Set cr (OverflowAddL op1 op2));
15329
15330 format %{ "adds zr, $op1, $op2\t# overflow check long" %}
15331 ins_cost(INSN_COST);
15332 ins_encode %{
15333 __ adds(zr, $op1$$Register, $op2$$constant);
15334 %}
15335
15336 ins_pipe(icmp_reg_imm);
15337 %}
15338
15339 instruct overflowSubI_reg_reg(rFlagsReg cr, iRegIorL2I op1, iRegIorL2I op2)
15340 %{
15341 match(Set cr (OverflowSubI op1 op2));
15342
15343 format %{ "cmpw $op1, $op2\t# overflow check int" %}
15344 ins_cost(INSN_COST);
15345 ins_encode %{
15346 __ cmpw($op1$$Register, $op2$$Register);
15347 %}
15348
15349 ins_pipe(icmp_reg_reg);
15350 %}
15351
15352 instruct overflowSubI_reg_imm(rFlagsReg cr, iRegIorL2I op1, immIAddSub op2)
15353 %{
15354 match(Set cr (OverflowSubI op1 op2));
15355
15356 format %{ "cmpw $op1, $op2\t# overflow check int" %}
15357 ins_cost(INSN_COST);
15358 ins_encode %{
15359 __ cmpw($op1$$Register, $op2$$constant);
15360 %}
15361
15362 ins_pipe(icmp_reg_imm);
15363 %}
15364
15365 instruct overflowSubL_reg_reg(rFlagsReg cr, iRegL op1, iRegL op2)
15366 %{
15367 match(Set cr (OverflowSubL op1 op2));
15368
15369 format %{ "cmp $op1, $op2\t# overflow check long" %}
15370 ins_cost(INSN_COST);
15371 ins_encode %{
15372 __ cmp($op1$$Register, $op2$$Register);
15373 %}
15374
15375 ins_pipe(icmp_reg_reg);
15376 %}
15377
15378 instruct overflowSubL_reg_imm(rFlagsReg cr, iRegL op1, immLAddSub op2)
15379 %{
15380 match(Set cr (OverflowSubL op1 op2));
15381
15382 format %{ "cmp $op1, $op2\t# overflow check long" %}
15383 ins_cost(INSN_COST);
15384 ins_encode %{
15385 __ subs(zr, $op1$$Register, $op2$$constant);
15386 %}
15387
15388 ins_pipe(icmp_reg_imm);
15389 %}
15390
15391 instruct overflowNegI_reg(rFlagsReg cr, immI0 zero, iRegIorL2I op1)
15392 %{
15393 match(Set cr (OverflowSubI zero op1));
15394
15395 format %{ "cmpw zr, $op1\t# overflow check int" %}
15396 ins_cost(INSN_COST);
15397 ins_encode %{
15398 __ cmpw(zr, $op1$$Register);
15399 %}
15400
15401 ins_pipe(icmp_reg_imm);
15402 %}
15403
15404 instruct overflowNegL_reg(rFlagsReg cr, immI0 zero, iRegL op1)
15405 %{
15406 match(Set cr (OverflowSubL zero op1));
15407
15408 format %{ "cmp zr, $op1\t# overflow check long" %}
15409 ins_cost(INSN_COST);
15410 ins_encode %{
15411 __ cmp(zr, $op1$$Register);
15412 %}
15413
15414 ins_pipe(icmp_reg_imm);
15415 %}
15416
15417 instruct overflowMulI_reg(rFlagsReg cr, iRegIorL2I op1, iRegIorL2I op2)
15418 %{
15419 match(Set cr (OverflowMulI op1 op2));
15420
15421 format %{ "smull rscratch1, $op1, $op2\t# overflow check int\n\t"
15422 "cmp rscratch1, rscratch1, sxtw\n\t"
15423 "movw rscratch1, #0x80000000\n\t"
15424 "cselw rscratch1, rscratch1, zr, NE\n\t"
15425 "cmpw rscratch1, #1" %}
15426 ins_cost(5 * INSN_COST);
15427 ins_encode %{
15428 __ smull(rscratch1, $op1$$Register, $op2$$Register);
15429 __ subs(zr, rscratch1, rscratch1, ext::sxtw); // NE => overflow
15430 __ movw(rscratch1, 0x80000000); // Develop 0 (EQ),
15431 __ cselw(rscratch1, rscratch1, zr, Assembler::NE); // or 0x80000000 (NE)
15432 __ cmpw(rscratch1, 1); // 0x80000000 - 1 => VS
15433 %}
15434
15435 ins_pipe(pipe_slow);
15436 %}
15437
15438 instruct overflowMulI_reg_branch(cmpOp cmp, iRegIorL2I op1, iRegIorL2I op2, label labl, rFlagsReg cr)
15439 %{
15440 match(If cmp (OverflowMulI op1 op2));
15441 predicate(n->in(1)->as_Bool()->_test._test == BoolTest::overflow
15442 || n->in(1)->as_Bool()->_test._test == BoolTest::no_overflow);
15443 effect(USE labl, KILL cr);
15444
15445 format %{ "smull rscratch1, $op1, $op2\t# overflow check int\n\t"
15446 "cmp rscratch1, rscratch1, sxtw\n\t"
15447 "b$cmp $labl" %}
15448 ins_cost(3 * INSN_COST); // Branch is rare so treat as INSN_COST
15449 ins_encode %{
15450 Label* L = $labl$$label;
15451 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15452 __ smull(rscratch1, $op1$$Register, $op2$$Register);
15453 __ subs(zr, rscratch1, rscratch1, ext::sxtw); // NE => overflow
15454 __ br(cond == Assembler::VS ? Assembler::NE : Assembler::EQ, *L);
15455 %}
15456
15457 ins_pipe(pipe_serial);
15458 %}
15459
15460 instruct overflowMulL_reg(rFlagsReg cr, iRegL op1, iRegL op2)
15461 %{
15462 match(Set cr (OverflowMulL op1 op2));
15463
15464 format %{ "mul rscratch1, $op1, $op2\t#overflow check long\n\t"
15465 "smulh rscratch2, $op1, $op2\n\t"
15466 "cmp rscratch2, rscratch1, ASR #63\n\t"
15467 "movw rscratch1, #0x80000000\n\t"
15468 "cselw rscratch1, rscratch1, zr, NE\n\t"
15469 "cmpw rscratch1, #1" %}
15470 ins_cost(6 * INSN_COST);
15471 ins_encode %{
15472 __ mul(rscratch1, $op1$$Register, $op2$$Register); // Result bits 0..63
15473 __ smulh(rscratch2, $op1$$Register, $op2$$Register); // Result bits 64..127
15474 __ cmp(rscratch2, rscratch1, Assembler::ASR, 63); // Top is pure sign ext
15475 __ movw(rscratch1, 0x80000000); // Develop 0 (EQ),
15476 __ cselw(rscratch1, rscratch1, zr, Assembler::NE); // or 0x80000000 (NE)
15477 __ cmpw(rscratch1, 1); // 0x80000000 - 1 => VS
15478 %}
15479
15480 ins_pipe(pipe_slow);
15481 %}
15482
15483 instruct overflowMulL_reg_branch(cmpOp cmp, iRegL op1, iRegL op2, label labl, rFlagsReg cr)
15484 %{
15485 match(If cmp (OverflowMulL op1 op2));
15486 predicate(n->in(1)->as_Bool()->_test._test == BoolTest::overflow
15487 || n->in(1)->as_Bool()->_test._test == BoolTest::no_overflow);
15488 effect(USE labl, KILL cr);
15489
15490 format %{ "mul rscratch1, $op1, $op2\t#overflow check long\n\t"
15491 "smulh rscratch2, $op1, $op2\n\t"
15492 "cmp rscratch2, rscratch1, ASR #63\n\t"
15493 "b$cmp $labl" %}
15494 ins_cost(4 * INSN_COST); // Branch is rare so treat as INSN_COST
15495 ins_encode %{
15496 Label* L = $labl$$label;
15497 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15498 __ mul(rscratch1, $op1$$Register, $op2$$Register); // Result bits 0..63
15499 __ smulh(rscratch2, $op1$$Register, $op2$$Register); // Result bits 64..127
15500 __ cmp(rscratch2, rscratch1, Assembler::ASR, 63); // Top is pure sign ext
15501 __ br(cond == Assembler::VS ? Assembler::NE : Assembler::EQ, *L);
15502 %}
15503
15504 ins_pipe(pipe_serial);
15505 %}
15506
15507 // ============================================================================
15508 // Compare Instructions
15509
15510 instruct compI_reg_reg(rFlagsReg cr, iRegI op1, iRegI op2)
15511 %{
15512 match(Set cr (CmpI op1 op2));
15513
15514 effect(DEF cr, USE op1, USE op2);
15515
15516 ins_cost(INSN_COST);
15517 format %{ "cmpw $op1, $op2" %}
15518
15519 ins_encode(aarch64_enc_cmpw(op1, op2));
15520
15521 ins_pipe(icmp_reg_reg);
15522 %}
15523
15524 instruct compI_reg_immI0(rFlagsReg cr, iRegI op1, immI0 zero)
15525 %{
15526 match(Set cr (CmpI op1 zero));
15527
15528 effect(DEF cr, USE op1);
15529
15530 ins_cost(INSN_COST);
15531 format %{ "cmpw $op1, 0" %}
15532
15533 ins_encode(aarch64_enc_cmpw_imm_addsub(op1, zero));
15534
15535 ins_pipe(icmp_reg_imm);
15536 %}
15537
15538 instruct compI_reg_immIAddSub(rFlagsReg cr, iRegI op1, immIAddSub op2)
15539 %{
15540 match(Set cr (CmpI op1 op2));
15541
15542 effect(DEF cr, USE op1);
15543
15544 ins_cost(INSN_COST);
15545 format %{ "cmpw $op1, $op2" %}
15546
15547 ins_encode(aarch64_enc_cmpw_imm_addsub(op1, op2));
15548
15549 ins_pipe(icmp_reg_imm);
15550 %}
15551
15552 instruct compI_reg_immI(rFlagsReg cr, iRegI op1, immI op2)
15553 %{
15554 match(Set cr (CmpI op1 op2));
15555
15556 effect(DEF cr, USE op1);
15557
15558 ins_cost(INSN_COST * 2);
15559 format %{ "cmpw $op1, $op2" %}
15560
15561 ins_encode(aarch64_enc_cmpw_imm(op1, op2));
15562
15563 ins_pipe(icmp_reg_imm);
15564 %}
15565
15566 // Unsigned compare Instructions; really, same as signed compare
15567 // except it should only be used to feed an If or a CMovI which takes a
15568 // cmpOpU.
15569
15570 instruct compU_reg_reg(rFlagsRegU cr, iRegI op1, iRegI op2)
15571 %{
15572 match(Set cr (CmpU op1 op2));
15573
15574 effect(DEF cr, USE op1, USE op2);
15575
15576 ins_cost(INSN_COST);
15577 format %{ "cmpw $op1, $op2\t# unsigned" %}
15578
15579 ins_encode(aarch64_enc_cmpw(op1, op2));
15580
15581 ins_pipe(icmp_reg_reg);
15582 %}
15583
15584 instruct compU_reg_immI0(rFlagsRegU cr, iRegI op1, immI0 zero)
15585 %{
15586 match(Set cr (CmpU op1 zero));
15587
15588 effect(DEF cr, USE op1);
15589
15590 ins_cost(INSN_COST);
15591 format %{ "cmpw $op1, #0\t# unsigned" %}
15592
15593 ins_encode(aarch64_enc_cmpw_imm_addsub(op1, zero));
15594
15595 ins_pipe(icmp_reg_imm);
15596 %}
15597
15598 instruct compU_reg_immIAddSub(rFlagsRegU cr, iRegI op1, immIAddSub op2)
15599 %{
15600 match(Set cr (CmpU op1 op2));
15601
15602 effect(DEF cr, USE op1);
15603
15604 ins_cost(INSN_COST);
15605 format %{ "cmpw $op1, $op2\t# unsigned" %}
15606
15607 ins_encode(aarch64_enc_cmpw_imm_addsub(op1, op2));
15608
15609 ins_pipe(icmp_reg_imm);
15610 %}
15611
15612 instruct compU_reg_immI(rFlagsRegU cr, iRegI op1, immI op2)
15613 %{
15614 match(Set cr (CmpU op1 op2));
15615
15616 effect(DEF cr, USE op1);
15617
15618 ins_cost(INSN_COST * 2);
15619 format %{ "cmpw $op1, $op2\t# unsigned" %}
15620
15621 ins_encode(aarch64_enc_cmpw_imm(op1, op2));
15622
15623 ins_pipe(icmp_reg_imm);
15624 %}
15625
15626 instruct compL_reg_reg(rFlagsReg cr, iRegL op1, iRegL op2)
15627 %{
15628 match(Set cr (CmpL op1 op2));
15629
15630 effect(DEF cr, USE op1, USE op2);
15631
15632 ins_cost(INSN_COST);
15633 format %{ "cmp $op1, $op2" %}
15634
15635 ins_encode(aarch64_enc_cmp(op1, op2));
15636
15637 ins_pipe(icmp_reg_reg);
15638 %}
15639
15640 instruct compL_reg_immL0(rFlagsReg cr, iRegL op1, immL0 zero)
15641 %{
15642 match(Set cr (CmpL op1 zero));
15643
15644 effect(DEF cr, USE op1);
15645
15646 ins_cost(INSN_COST);
15647 format %{ "tst $op1" %}
15648
15649 ins_encode(aarch64_enc_cmp_imm_addsub(op1, zero));
15650
15651 ins_pipe(icmp_reg_imm);
15652 %}
15653
15654 instruct compL_reg_immLAddSub(rFlagsReg cr, iRegL op1, immLAddSub op2)
15655 %{
15656 match(Set cr (CmpL op1 op2));
15657
15658 effect(DEF cr, USE op1);
15659
15660 ins_cost(INSN_COST);
15661 format %{ "cmp $op1, $op2" %}
15662
15663 ins_encode(aarch64_enc_cmp_imm_addsub(op1, op2));
15664
15665 ins_pipe(icmp_reg_imm);
15666 %}
15667
15668 instruct compL_reg_immL(rFlagsReg cr, iRegL op1, immL op2)
15669 %{
15670 match(Set cr (CmpL op1 op2));
15671
15672 effect(DEF cr, USE op1);
15673
15674 ins_cost(INSN_COST * 2);
15675 format %{ "cmp $op1, $op2" %}
15676
15677 ins_encode(aarch64_enc_cmp_imm(op1, op2));
15678
15679 ins_pipe(icmp_reg_imm);
15680 %}
15681
15682 instruct compUL_reg_reg(rFlagsRegU cr, iRegL op1, iRegL op2)
15683 %{
15684 match(Set cr (CmpUL op1 op2));
15685
15686 effect(DEF cr, USE op1, USE op2);
15687
15688 ins_cost(INSN_COST);
15689 format %{ "cmp $op1, $op2" %}
15690
15691 ins_encode(aarch64_enc_cmp(op1, op2));
15692
15693 ins_pipe(icmp_reg_reg);
15694 %}
15695
15696 instruct compUL_reg_immL0(rFlagsRegU cr, iRegL op1, immL0 zero)
15697 %{
15698 match(Set cr (CmpUL op1 zero));
15699
15700 effect(DEF cr, USE op1);
15701
15702 ins_cost(INSN_COST);
15703 format %{ "tst $op1" %}
15704
15705 ins_encode(aarch64_enc_cmp_imm_addsub(op1, zero));
15706
15707 ins_pipe(icmp_reg_imm);
15708 %}
15709
15710 instruct compUL_reg_immLAddSub(rFlagsRegU cr, iRegL op1, immLAddSub op2)
15711 %{
15712 match(Set cr (CmpUL op1 op2));
15713
15714 effect(DEF cr, USE op1);
15715
15716 ins_cost(INSN_COST);
15717 format %{ "cmp $op1, $op2" %}
15718
15719 ins_encode(aarch64_enc_cmp_imm_addsub(op1, op2));
15720
15721 ins_pipe(icmp_reg_imm);
15722 %}
15723
15724 instruct compUL_reg_immL(rFlagsRegU cr, iRegL op1, immL op2)
15725 %{
15726 match(Set cr (CmpUL op1 op2));
15727
15728 effect(DEF cr, USE op1);
15729
15730 ins_cost(INSN_COST * 2);
15731 format %{ "cmp $op1, $op2" %}
15732
15733 ins_encode(aarch64_enc_cmp_imm(op1, op2));
15734
15735 ins_pipe(icmp_reg_imm);
15736 %}
15737
15738 instruct compP_reg_reg(rFlagsRegU cr, iRegP op1, iRegP op2)
15739 %{
15740 match(Set cr (CmpP op1 op2));
15741
15742 effect(DEF cr, USE op1, USE op2);
15743
15744 ins_cost(INSN_COST);
15745 format %{ "cmp $op1, $op2\t // ptr" %}
15746
15747 ins_encode(aarch64_enc_cmpp(op1, op2));
15748
15749 ins_pipe(icmp_reg_reg);
15750 %}
15751
15752 instruct compN_reg_reg(rFlagsRegU cr, iRegN op1, iRegN op2)
15753 %{
15754 match(Set cr (CmpN op1 op2));
15755
15756 effect(DEF cr, USE op1, USE op2);
15757
15758 ins_cost(INSN_COST);
15759 format %{ "cmp $op1, $op2\t // compressed ptr" %}
15760
15761 ins_encode(aarch64_enc_cmpn(op1, op2));
15762
15763 ins_pipe(icmp_reg_reg);
15764 %}
15765
15766 instruct testP_reg(rFlagsRegU cr, iRegP op1, immP0 zero)
15767 %{
15768 match(Set cr (CmpP op1 zero));
15769
15770 effect(DEF cr, USE op1, USE zero);
15771
15772 ins_cost(INSN_COST);
15773 format %{ "cmp $op1, 0\t // ptr" %}
15774
15775 ins_encode(aarch64_enc_testp(op1));
15776
15777 ins_pipe(icmp_reg_imm);
15778 %}
15779
15780 instruct testN_reg(rFlagsRegU cr, iRegN op1, immN0 zero)
15781 %{
15782 match(Set cr (CmpN op1 zero));
15783
15784 effect(DEF cr, USE op1, USE zero);
15785
15786 ins_cost(INSN_COST);
15787 format %{ "cmp $op1, 0\t // compressed ptr" %}
15788
15789 ins_encode(aarch64_enc_testn(op1));
15790
15791 ins_pipe(icmp_reg_imm);
15792 %}
15793
15794 // FP comparisons
15795 //
15796 // n.b. CmpF/CmpD set a normal flags reg which then gets compared
15797 // using normal cmpOp. See declaration of rFlagsReg for details.
15798
15799 instruct compF_reg_reg(rFlagsReg cr, vRegF src1, vRegF src2)
15800 %{
15801 match(Set cr (CmpF src1 src2));
15802
15803 ins_cost(3 * INSN_COST);
15804 format %{ "fcmps $src1, $src2" %}
15805
15806 ins_encode %{
15807 __ fcmps(as_FloatRegister($src1$$reg), as_FloatRegister($src2$$reg));
15808 %}
15809
15810 ins_pipe(pipe_class_compare);
15811 %}
15812
15813 instruct compF_reg_zero(rFlagsReg cr, vRegF src1, immF0 src2)
15814 %{
15815 match(Set cr (CmpF src1 src2));
15816
15817 ins_cost(3 * INSN_COST);
15818 format %{ "fcmps $src1, 0.0" %}
15819
15820 ins_encode %{
15821 __ fcmps(as_FloatRegister($src1$$reg), 0.0);
15822 %}
15823
15824 ins_pipe(pipe_class_compare);
15825 %}
15826 // FROM HERE
15827
15828 instruct compD_reg_reg(rFlagsReg cr, vRegD src1, vRegD src2)
15829 %{
15830 match(Set cr (CmpD src1 src2));
15831
15832 ins_cost(3 * INSN_COST);
15833 format %{ "fcmpd $src1, $src2" %}
15834
15835 ins_encode %{
15836 __ fcmpd(as_FloatRegister($src1$$reg), as_FloatRegister($src2$$reg));
15837 %}
15838
15839 ins_pipe(pipe_class_compare);
15840 %}
15841
15842 instruct compD_reg_zero(rFlagsReg cr, vRegD src1, immD0 src2)
15843 %{
15844 match(Set cr (CmpD src1 src2));
15845
15846 ins_cost(3 * INSN_COST);
15847 format %{ "fcmpd $src1, 0.0" %}
15848
15849 ins_encode %{
15850 __ fcmpd(as_FloatRegister($src1$$reg), 0.0);
15851 %}
15852
15853 ins_pipe(pipe_class_compare);
15854 %}
15855
15856 instruct compF3_reg_reg(iRegINoSp dst, vRegF src1, vRegF src2, rFlagsReg cr)
15857 %{
15858 match(Set dst (CmpF3 src1 src2));
15859 effect(KILL cr);
15860
15861 ins_cost(5 * INSN_COST);
15862 format %{ "fcmps $src1, $src2\n\t"
15863 "csinvw($dst, zr, zr, eq\n\t"
15864 "csnegw($dst, $dst, $dst, lt)"
15865 %}
15866
15867 ins_encode %{
15868 Label done;
15869 FloatRegister s1 = as_FloatRegister($src1$$reg);
15870 FloatRegister s2 = as_FloatRegister($src2$$reg);
15871 Register d = as_Register($dst$$reg);
15872 __ fcmps(s1, s2);
15873 // installs 0 if EQ else -1
15874 __ csinvw(d, zr, zr, Assembler::EQ);
15875 // keeps -1 if less or unordered else installs 1
15876 __ csnegw(d, d, d, Assembler::LT);
15877 __ bind(done);
15878 %}
15879
15880 ins_pipe(pipe_class_default);
15881
15882 %}
15883
15884 instruct compD3_reg_reg(iRegINoSp dst, vRegD src1, vRegD src2, rFlagsReg cr)
15885 %{
15886 match(Set dst (CmpD3 src1 src2));
15887 effect(KILL cr);
15888
15889 ins_cost(5 * INSN_COST);
15890 format %{ "fcmpd $src1, $src2\n\t"
15891 "csinvw($dst, zr, zr, eq\n\t"
15892 "csnegw($dst, $dst, $dst, lt)"
15893 %}
15894
15895 ins_encode %{
15896 Label done;
15897 FloatRegister s1 = as_FloatRegister($src1$$reg);
15898 FloatRegister s2 = as_FloatRegister($src2$$reg);
15899 Register d = as_Register($dst$$reg);
15900 __ fcmpd(s1, s2);
15901 // installs 0 if EQ else -1
15902 __ csinvw(d, zr, zr, Assembler::EQ);
15903 // keeps -1 if less or unordered else installs 1
15904 __ csnegw(d, d, d, Assembler::LT);
15905 __ bind(done);
15906 %}
15907 ins_pipe(pipe_class_default);
15908
15909 %}
15910
15911 instruct compF3_reg_immF0(iRegINoSp dst, vRegF src1, immF0 zero, rFlagsReg cr)
15912 %{
15913 match(Set dst (CmpF3 src1 zero));
15914 effect(KILL cr);
15915
15916 ins_cost(5 * INSN_COST);
15917 format %{ "fcmps $src1, 0.0\n\t"
15918 "csinvw($dst, zr, zr, eq\n\t"
15919 "csnegw($dst, $dst, $dst, lt)"
15920 %}
15921
15922 ins_encode %{
15923 Label done;
15924 FloatRegister s1 = as_FloatRegister($src1$$reg);
15925 Register d = as_Register($dst$$reg);
15926 __ fcmps(s1, 0.0);
15927 // installs 0 if EQ else -1
15928 __ csinvw(d, zr, zr, Assembler::EQ);
15929 // keeps -1 if less or unordered else installs 1
15930 __ csnegw(d, d, d, Assembler::LT);
15931 __ bind(done);
15932 %}
15933
15934 ins_pipe(pipe_class_default);
15935
15936 %}
15937
15938 instruct compD3_reg_immD0(iRegINoSp dst, vRegD src1, immD0 zero, rFlagsReg cr)
15939 %{
15940 match(Set dst (CmpD3 src1 zero));
15941 effect(KILL cr);
15942
15943 ins_cost(5 * INSN_COST);
15944 format %{ "fcmpd $src1, 0.0\n\t"
15945 "csinvw($dst, zr, zr, eq\n\t"
15946 "csnegw($dst, $dst, $dst, lt)"
15947 %}
15948
15949 ins_encode %{
15950 Label done;
15951 FloatRegister s1 = as_FloatRegister($src1$$reg);
15952 Register d = as_Register($dst$$reg);
15953 __ fcmpd(s1, 0.0);
15954 // installs 0 if EQ else -1
15955 __ csinvw(d, zr, zr, Assembler::EQ);
15956 // keeps -1 if less or unordered else installs 1
15957 __ csnegw(d, d, d, Assembler::LT);
15958 __ bind(done);
15959 %}
15960 ins_pipe(pipe_class_default);
15961
15962 %}
15963
15964 instruct cmpLTMask_reg_reg(iRegINoSp dst, iRegIorL2I p, iRegIorL2I q, rFlagsReg cr)
15965 %{
15966 match(Set dst (CmpLTMask p q));
15967 effect(KILL cr);
15968
15969 ins_cost(3 * INSN_COST);
15970
15971 format %{ "cmpw $p, $q\t# cmpLTMask\n\t"
15972 "csetw $dst, lt\n\t"
15973 "subw $dst, zr, $dst"
15974 %}
15975
15976 ins_encode %{
15977 __ cmpw(as_Register($p$$reg), as_Register($q$$reg));
15978 __ csetw(as_Register($dst$$reg), Assembler::LT);
15979 __ subw(as_Register($dst$$reg), zr, as_Register($dst$$reg));
15980 %}
15981
15982 ins_pipe(ialu_reg_reg);
15983 %}
15984
15985 instruct cmpLTMask_reg_zero(iRegINoSp dst, iRegIorL2I src, immI0 zero, rFlagsReg cr)
15986 %{
15987 match(Set dst (CmpLTMask src zero));
15988 effect(KILL cr);
15989
15990 ins_cost(INSN_COST);
15991
15992 format %{ "asrw $dst, $src, #31\t# cmpLTMask0" %}
15993
15994 ins_encode %{
15995 __ asrw(as_Register($dst$$reg), as_Register($src$$reg), 31);
15996 %}
15997
15998 ins_pipe(ialu_reg_shift);
15999 %}
16000
16001 // ============================================================================
16002 // Max and Min
16003
16004 instruct cmovI_reg_reg_lt(iRegINoSp dst, iRegI src1, iRegI src2, rFlagsReg cr)
16005 %{
16006 effect( DEF dst, USE src1, USE src2, USE cr );
16007
16008 ins_cost(INSN_COST * 2);
16009 format %{ "cselw $dst, $src1, $src2 lt\t" %}
16010
16011 ins_encode %{
16012 __ cselw(as_Register($dst$$reg),
16013 as_Register($src1$$reg),
16014 as_Register($src2$$reg),
16015 Assembler::LT);
16016 %}
16017
16018 ins_pipe(icond_reg_reg);
16019 %}
16020
16021 instruct minI_rReg(iRegINoSp dst, iRegI src1, iRegI src2)
16022 %{
16023 match(Set dst (MinI src1 src2));
16024 ins_cost(INSN_COST * 3);
16025
16026 expand %{
16027 rFlagsReg cr;
16028 compI_reg_reg(cr, src1, src2);
16029 cmovI_reg_reg_lt(dst, src1, src2, cr);
16030 %}
16031
16032 %}
16033 // FROM HERE
16034
16035 instruct cmovI_reg_reg_gt(iRegINoSp dst, iRegI src1, iRegI src2, rFlagsReg cr)
16036 %{
16037 effect( DEF dst, USE src1, USE src2, USE cr );
16038
16039 ins_cost(INSN_COST * 2);
16040 format %{ "cselw $dst, $src1, $src2 gt\t" %}
16041
16042 ins_encode %{
16043 __ cselw(as_Register($dst$$reg),
16044 as_Register($src1$$reg),
16045 as_Register($src2$$reg),
16046 Assembler::GT);
16047 %}
16048
16049 ins_pipe(icond_reg_reg);
16050 %}
16051
16052 instruct maxI_rReg(iRegINoSp dst, iRegI src1, iRegI src2)
16053 %{
16054 match(Set dst (MaxI src1 src2));
16055 ins_cost(INSN_COST * 3);
16056 expand %{
16057 rFlagsReg cr;
16058 compI_reg_reg(cr, src1, src2);
16059 cmovI_reg_reg_gt(dst, src1, src2, cr);
16060 %}
16061 %}
16062
16063 // ============================================================================
16064 // Branch Instructions
16065
16066 // Direct Branch.
16067 instruct branch(label lbl)
16068 %{
16069 match(Goto);
16070
16071 effect(USE lbl);
16072
16073 ins_cost(BRANCH_COST);
16074 format %{ "b $lbl" %}
16075
16076 ins_encode(aarch64_enc_b(lbl));
16077
16078 ins_pipe(pipe_branch);
16079 %}
16080
16081 // Conditional Near Branch
16082 instruct branchCon(cmpOp cmp, rFlagsReg cr, label lbl)
16083 %{
16084 // Same match rule as `branchConFar'.
16085 match(If cmp cr);
16086
16087 effect(USE lbl);
16088
16089 ins_cost(BRANCH_COST);
16090 // If set to 1 this indicates that the current instruction is a
16091 // short variant of a long branch. This avoids using this
16092 // instruction in first-pass matching. It will then only be used in
16093 // the `Shorten_branches' pass.
16094 // ins_short_branch(1);
16095 format %{ "b$cmp $lbl" %}
16096
16097 ins_encode(aarch64_enc_br_con(cmp, lbl));
16098
16099 ins_pipe(pipe_branch_cond);
16100 %}
16101
16102 // Conditional Near Branch Unsigned
16103 instruct branchConU(cmpOpU cmp, rFlagsRegU cr, label lbl)
16104 %{
16105 // Same match rule as `branchConFar'.
16106 match(If cmp cr);
16107
16108 effect(USE lbl);
16109
16110 ins_cost(BRANCH_COST);
16111 // If set to 1 this indicates that the current instruction is a
16112 // short variant of a long branch. This avoids using this
16113 // instruction in first-pass matching. It will then only be used in
16114 // the `Shorten_branches' pass.
16115 // ins_short_branch(1);
16116 format %{ "b$cmp $lbl\t# unsigned" %}
16117
16118 ins_encode(aarch64_enc_br_conU(cmp, lbl));
16119
16120 ins_pipe(pipe_branch_cond);
16121 %}
16122
16123 // Make use of CBZ and CBNZ. These instructions, as well as being
16124 // shorter than (cmp; branch), have the additional benefit of not
16125 // killing the flags.
16126
16127 instruct cmpI_imm0_branch(cmpOpEqNe cmp, iRegIorL2I op1, immI0 op2, label labl, rFlagsReg cr) %{
16128 match(If cmp (CmpI op1 op2));
16129 effect(USE labl);
16130
16131 ins_cost(BRANCH_COST);
16132 format %{ "cbw$cmp $op1, $labl" %}
16133 ins_encode %{
16134 Label* L = $labl$$label;
16135 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
16136 if (cond == Assembler::EQ)
16137 __ cbzw($op1$$Register, *L);
16138 else
16139 __ cbnzw($op1$$Register, *L);
16140 %}
16141 ins_pipe(pipe_cmp_branch);
16142 %}
16143
16144 instruct cmpL_imm0_branch(cmpOpEqNe cmp, iRegL op1, immL0 op2, label labl, rFlagsReg cr) %{
16145 match(If cmp (CmpL op1 op2));
16146 effect(USE labl);
16147
16148 ins_cost(BRANCH_COST);
16149 format %{ "cb$cmp $op1, $labl" %}
16150 ins_encode %{
16151 Label* L = $labl$$label;
16152 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
16153 if (cond == Assembler::EQ)
16154 __ cbz($op1$$Register, *L);
16155 else
16156 __ cbnz($op1$$Register, *L);
16157 %}
16158 ins_pipe(pipe_cmp_branch);
16159 %}
16160
16161 instruct cmpP_imm0_branch(cmpOpEqNe cmp, iRegP op1, immP0 op2, label labl, rFlagsReg cr) %{
16162 match(If cmp (CmpP op1 op2));
16163 effect(USE labl);
16164
16165 ins_cost(BRANCH_COST);
16166 format %{ "cb$cmp $op1, $labl" %}
16167 ins_encode %{
16168 Label* L = $labl$$label;
16169 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
16170 if (cond == Assembler::EQ)
16171 __ cbz($op1$$Register, *L);
16172 else
16173 __ cbnz($op1$$Register, *L);
16174 %}
16175 ins_pipe(pipe_cmp_branch);
16176 %}
16177
16178 instruct cmpN_imm0_branch(cmpOpEqNe cmp, iRegN op1, immN0 op2, label labl, rFlagsReg cr) %{
16179 match(If cmp (CmpN op1 op2));
16180 effect(USE labl);
16181
16182 ins_cost(BRANCH_COST);
16183 format %{ "cbw$cmp $op1, $labl" %}
16184 ins_encode %{
16185 Label* L = $labl$$label;
16186 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
16187 if (cond == Assembler::EQ)
16188 __ cbzw($op1$$Register, *L);
16189 else
16190 __ cbnzw($op1$$Register, *L);
16191 %}
16192 ins_pipe(pipe_cmp_branch);
16193 %}
16194
16195 instruct cmpP_narrowOop_imm0_branch(cmpOpEqNe cmp, iRegN oop, immP0 zero, label labl, rFlagsReg cr) %{
16196 match(If cmp (CmpP (DecodeN oop) zero));
16197 effect(USE labl);
16198
16199 ins_cost(BRANCH_COST);
16200 format %{ "cb$cmp $oop, $labl" %}
16201 ins_encode %{
16202 Label* L = $labl$$label;
16203 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
16204 if (cond == Assembler::EQ)
16205 __ cbzw($oop$$Register, *L);
16206 else
16207 __ cbnzw($oop$$Register, *L);
16208 %}
16209 ins_pipe(pipe_cmp_branch);
16210 %}
16211
16212 instruct cmpUI_imm0_branch(cmpOpUEqNeLtGe cmp, iRegIorL2I op1, immI0 op2, label labl, rFlagsRegU cr) %{
16213 match(If cmp (CmpU op1 op2));
16214 effect(USE labl);
16215
16216 ins_cost(BRANCH_COST);
16217 format %{ "cbw$cmp $op1, $labl" %}
16218 ins_encode %{
16219 Label* L = $labl$$label;
16220 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
16221 if (cond == Assembler::EQ || cond == Assembler::LS)
16222 __ cbzw($op1$$Register, *L);
16223 else
16224 __ cbnzw($op1$$Register, *L);
16225 %}
16226 ins_pipe(pipe_cmp_branch);
16227 %}
16228
16229 instruct cmpUL_imm0_branch(cmpOpUEqNeLtGe cmp, iRegL op1, immL0 op2, label labl, rFlagsRegU cr) %{
16230 match(If cmp (CmpUL op1 op2));
16231 effect(USE labl);
16232
16233 ins_cost(BRANCH_COST);
16234 format %{ "cb$cmp $op1, $labl" %}
16235 ins_encode %{
16236 Label* L = $labl$$label;
16237 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
16238 if (cond == Assembler::EQ || cond == Assembler::LS)
16239 __ cbz($op1$$Register, *L);
16240 else
16241 __ cbnz($op1$$Register, *L);
16242 %}
16243 ins_pipe(pipe_cmp_branch);
16244 %}
16245
16246 // Test bit and Branch
16247
16248 // Patterns for short (< 32KiB) variants
16249 instruct cmpL_branch_sign(cmpOpLtGe cmp, iRegL op1, immL0 op2, label labl) %{
16250 match(If cmp (CmpL op1 op2));
16251 effect(USE labl);
16252
16253 ins_cost(BRANCH_COST);
16254 format %{ "cb$cmp $op1, $labl # long" %}
16255 ins_encode %{
16256 Label* L = $labl$$label;
16257 Assembler::Condition cond =
16258 ((Assembler::Condition)$cmp$$cmpcode == Assembler::LT) ? Assembler::NE : Assembler::EQ;
16259 __ tbr(cond, $op1$$Register, 63, *L);
16260 %}
16261 ins_pipe(pipe_cmp_branch);
16262 ins_short_branch(1);
16263 %}
16264
16265 instruct cmpI_branch_sign(cmpOpLtGe cmp, iRegIorL2I op1, immI0 op2, label labl) %{
16266 match(If cmp (CmpI op1 op2));
16267 effect(USE labl);
16268
16269 ins_cost(BRANCH_COST);
16270 format %{ "cb$cmp $op1, $labl # int" %}
16271 ins_encode %{
16272 Label* L = $labl$$label;
16273 Assembler::Condition cond =
16274 ((Assembler::Condition)$cmp$$cmpcode == Assembler::LT) ? Assembler::NE : Assembler::EQ;
16275 __ tbr(cond, $op1$$Register, 31, *L);
16276 %}
16277 ins_pipe(pipe_cmp_branch);
16278 ins_short_branch(1);
16279 %}
16280
16281 instruct cmpL_branch_bit(cmpOpEqNe cmp, iRegL op1, immL op2, immL0 op3, label labl) %{
16282 match(If cmp (CmpL (AndL op1 op2) op3));
16283 predicate(is_power_of_2((julong)n->in(2)->in(1)->in(2)->get_long()));
16284 effect(USE labl);
16285
16286 ins_cost(BRANCH_COST);
16287 format %{ "tb$cmp $op1, $op2, $labl" %}
16288 ins_encode %{
16289 Label* L = $labl$$label;
16290 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
16291 int bit = exact_log2_long($op2$$constant);
16292 __ tbr(cond, $op1$$Register, bit, *L);
16293 %}
16294 ins_pipe(pipe_cmp_branch);
16295 ins_short_branch(1);
16296 %}
16297
16298 instruct cmpI_branch_bit(cmpOpEqNe cmp, iRegIorL2I op1, immI op2, immI0 op3, label labl) %{
16299 match(If cmp (CmpI (AndI op1 op2) op3));
16300 predicate(is_power_of_2((juint)n->in(2)->in(1)->in(2)->get_int()));
16301 effect(USE labl);
16302
16303 ins_cost(BRANCH_COST);
16304 format %{ "tb$cmp $op1, $op2, $labl" %}
16305 ins_encode %{
16306 Label* L = $labl$$label;
16307 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
16308 int bit = exact_log2((juint)$op2$$constant);
16309 __ tbr(cond, $op1$$Register, bit, *L);
16310 %}
16311 ins_pipe(pipe_cmp_branch);
16312 ins_short_branch(1);
16313 %}
16314
16315 // And far variants
16316 instruct far_cmpL_branch_sign(cmpOpLtGe cmp, iRegL op1, immL0 op2, label labl) %{
16317 match(If cmp (CmpL op1 op2));
16318 effect(USE labl);
16319
16320 ins_cost(BRANCH_COST);
16321 format %{ "cb$cmp $op1, $labl # long" %}
16322 ins_encode %{
16323 Label* L = $labl$$label;
16324 Assembler::Condition cond =
16325 ((Assembler::Condition)$cmp$$cmpcode == Assembler::LT) ? Assembler::NE : Assembler::EQ;
16326 __ tbr(cond, $op1$$Register, 63, *L, /*far*/true);
16327 %}
16328 ins_pipe(pipe_cmp_branch);
16329 %}
16330
16331 instruct far_cmpI_branch_sign(cmpOpLtGe cmp, iRegIorL2I op1, immI0 op2, label labl) %{
16332 match(If cmp (CmpI op1 op2));
16333 effect(USE labl);
16334
16335 ins_cost(BRANCH_COST);
16336 format %{ "cb$cmp $op1, $labl # int" %}
16337 ins_encode %{
16338 Label* L = $labl$$label;
16339 Assembler::Condition cond =
16340 ((Assembler::Condition)$cmp$$cmpcode == Assembler::LT) ? Assembler::NE : Assembler::EQ;
16341 __ tbr(cond, $op1$$Register, 31, *L, /*far*/true);
16342 %}
16343 ins_pipe(pipe_cmp_branch);
16344 %}
16345
16346 instruct far_cmpL_branch_bit(cmpOpEqNe cmp, iRegL op1, immL op2, immL0 op3, label labl) %{
16347 match(If cmp (CmpL (AndL op1 op2) op3));
16348 predicate(is_power_of_2((julong)n->in(2)->in(1)->in(2)->get_long()));
16349 effect(USE labl);
16350
16351 ins_cost(BRANCH_COST);
16352 format %{ "tb$cmp $op1, $op2, $labl" %}
16353 ins_encode %{
16354 Label* L = $labl$$label;
16355 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
16356 int bit = exact_log2_long($op2$$constant);
16357 __ tbr(cond, $op1$$Register, bit, *L, /*far*/true);
16358 %}
16359 ins_pipe(pipe_cmp_branch);
16360 %}
16361
16362 instruct far_cmpI_branch_bit(cmpOpEqNe cmp, iRegIorL2I op1, immI op2, immI0 op3, label labl) %{
16363 match(If cmp (CmpI (AndI op1 op2) op3));
16364 predicate(is_power_of_2((juint)n->in(2)->in(1)->in(2)->get_int()));
16365 effect(USE labl);
16366
16367 ins_cost(BRANCH_COST);
16368 format %{ "tb$cmp $op1, $op2, $labl" %}
16369 ins_encode %{
16370 Label* L = $labl$$label;
16371 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
16372 int bit = exact_log2((juint)$op2$$constant);
16373 __ tbr(cond, $op1$$Register, bit, *L, /*far*/true);
16374 %}
16375 ins_pipe(pipe_cmp_branch);
16376 %}
16377
16378 // Test bits
16379
16380 instruct cmpL_and(cmpOp cmp, iRegL op1, immL op2, immL0 op3, rFlagsReg cr) %{
16381 match(Set cr (CmpL (AndL op1 op2) op3));
16382 predicate(Assembler::operand_valid_for_logical_immediate
16383 (/*is_32*/false, n->in(1)->in(2)->get_long()));
16384
16385 ins_cost(INSN_COST);
16386 format %{ "tst $op1, $op2 # long" %}
16387 ins_encode %{
16388 __ tst($op1$$Register, $op2$$constant);
16389 %}
16390 ins_pipe(ialu_reg_reg);
16391 %}
16392
16393 instruct cmpI_and(cmpOp cmp, iRegIorL2I op1, immI op2, immI0 op3, rFlagsReg cr) %{
16394 match(Set cr (CmpI (AndI op1 op2) op3));
16395 predicate(Assembler::operand_valid_for_logical_immediate
16396 (/*is_32*/true, n->in(1)->in(2)->get_int()));
16397
16398 ins_cost(INSN_COST);
16399 format %{ "tst $op1, $op2 # int" %}
16400 ins_encode %{
16401 __ tstw($op1$$Register, $op2$$constant);
16402 %}
16403 ins_pipe(ialu_reg_reg);
16404 %}
16405
16406 instruct cmpL_and_reg(cmpOp cmp, iRegL op1, iRegL op2, immL0 op3, rFlagsReg cr) %{
16407 match(Set cr (CmpL (AndL op1 op2) op3));
16408
16409 ins_cost(INSN_COST);
16410 format %{ "tst $op1, $op2 # long" %}
16411 ins_encode %{
16412 __ tst($op1$$Register, $op2$$Register);
16413 %}
16414 ins_pipe(ialu_reg_reg);
16415 %}
16416
16417 instruct cmpI_and_reg(cmpOp cmp, iRegIorL2I op1, iRegIorL2I op2, immI0 op3, rFlagsReg cr) %{
16418 match(Set cr (CmpI (AndI op1 op2) op3));
16419
16420 ins_cost(INSN_COST);
16421 format %{ "tstw $op1, $op2 # int" %}
16422 ins_encode %{
16423 __ tstw($op1$$Register, $op2$$Register);
16424 %}
16425 ins_pipe(ialu_reg_reg);
16426 %}
16427
16428
16429 // Conditional Far Branch
16430 // Conditional Far Branch Unsigned
16431 // TODO: fixme
16432
16433 // counted loop end branch near
16434 instruct branchLoopEnd(cmpOp cmp, rFlagsReg cr, label lbl)
16435 %{
16436 match(CountedLoopEnd cmp cr);
16437
16438 effect(USE lbl);
16439
16440 ins_cost(BRANCH_COST);
16441 // short variant.
16442 // ins_short_branch(1);
16443 format %{ "b$cmp $lbl \t// counted loop end" %}
16444
16445 ins_encode(aarch64_enc_br_con(cmp, lbl));
16446
16447 ins_pipe(pipe_branch);
16448 %}
16449
16450 // counted loop end branch near Unsigned
16451 instruct branchLoopEndU(cmpOpU cmp, rFlagsRegU cr, label lbl)
16452 %{
16453 match(CountedLoopEnd cmp cr);
16454
16455 effect(USE lbl);
16456
16457 ins_cost(BRANCH_COST);
16458 // short variant.
16459 // ins_short_branch(1);
16460 format %{ "b$cmp $lbl \t// counted loop end unsigned" %}
16461
16462 ins_encode(aarch64_enc_br_conU(cmp, lbl));
16463
16464 ins_pipe(pipe_branch);
16465 %}
16466
16467 // counted loop end branch far
16468 // counted loop end branch far unsigned
16469 // TODO: fixme
16470
16471 // ============================================================================
16472 // inlined locking and unlocking
16473
16474 instruct cmpFastLock(rFlagsReg cr, iRegP object, iRegP box, iRegPNoSp tmp, iRegPNoSp tmp2)
16475 %{
16476 match(Set cr (FastLock object box));
16477 effect(TEMP tmp, TEMP tmp2);
16478
16479 // TODO
16480 // identify correct cost
16481 ins_cost(5 * INSN_COST);
16482 format %{ "fastlock $object,$box\t! kills $tmp,$tmp2" %}
16483
16484 ins_encode(aarch64_enc_fast_lock(object, box, tmp, tmp2));
16485
16486 ins_pipe(pipe_serial);
16487 %}
16488
16489 instruct cmpFastUnlock(rFlagsReg cr, iRegP object, iRegP box, iRegPNoSp tmp, iRegPNoSp tmp2)
16490 %{
16491 match(Set cr (FastUnlock object box));
16492 effect(TEMP tmp, TEMP tmp2);
16493
16494 ins_cost(5 * INSN_COST);
16495 format %{ "fastunlock $object,$box\t! kills $tmp, $tmp2" %}
16496
16497 ins_encode(aarch64_enc_fast_unlock(object, box, tmp, tmp2));
16498
16499 ins_pipe(pipe_serial);
16500 %}
16501
16502
16503 // ============================================================================
16504 // Safepoint Instructions
16505
16506 // TODO
16507 // provide a near and far version of this code
16508
16509 instruct safePoint(rFlagsReg cr, iRegP poll)
16510 %{
16511 match(SafePoint poll);
16512 effect(KILL cr);
16513
16514 format %{
16515 "ldrw zr, [$poll]\t# Safepoint: poll for GC"
16516 %}
16517 ins_encode %{
16518 __ read_polling_page(as_Register($poll$$reg), relocInfo::poll_type);
16519 %}
16520 ins_pipe(pipe_serial); // ins_pipe(iload_reg_mem);
16521 %}
16522
16523
16524 // ============================================================================
16525 // Procedure Call/Return Instructions
16526
16527 // Call Java Static Instruction
16528
16529 instruct CallStaticJavaDirect(method meth)
16530 %{
16531 match(CallStaticJava);
16532
16533 effect(USE meth);
16534
16535 ins_cost(CALL_COST);
16536
16537 format %{ "call,static $meth \t// ==> " %}
16538
16539 ins_encode(aarch64_enc_java_static_call(meth),
16540 aarch64_enc_call_epilog);
16541
16542 ins_pipe(pipe_class_call);
16543 %}
16544
16545 // TO HERE
16546
16547 // Call Java Dynamic Instruction
16548 instruct CallDynamicJavaDirect(method meth)
16549 %{
16550 match(CallDynamicJava);
16551
16552 effect(USE meth);
16553
16554 ins_cost(CALL_COST);
16555
16556 format %{ "CALL,dynamic $meth \t// ==> " %}
16557
16558 ins_encode(aarch64_enc_java_dynamic_call(meth),
16559 aarch64_enc_call_epilog);
16560
16561 ins_pipe(pipe_class_call);
16562 %}
16563
16564 // Call Runtime Instruction
16565
16566 instruct CallRuntimeDirect(method meth)
16567 %{
16568 match(CallRuntime);
16569
16570 effect(USE meth);
16571
16572 ins_cost(CALL_COST);
16573
16574 format %{ "CALL, runtime $meth" %}
16575
16576 ins_encode( aarch64_enc_java_to_runtime(meth) );
16577
16578 ins_pipe(pipe_class_call);
16579 %}
16580
16581 // Call Runtime Instruction
16582
16583 instruct CallLeafDirect(method meth)
16584 %{
16585 match(CallLeaf);
16586
16587 effect(USE meth);
16588
16589 ins_cost(CALL_COST);
16590
16591 format %{ "CALL, runtime leaf $meth" %}
16592
16593 ins_encode( aarch64_enc_java_to_runtime(meth) );
16594
16595 ins_pipe(pipe_class_call);
16596 %}
16597
16598 // Call Runtime Instruction
16599
16600 instruct CallLeafNoFPDirect(method meth)
16601 %{
16602 match(CallLeafNoFP);
16603
16604 effect(USE meth);
16605
16606 ins_cost(CALL_COST);
16607
16608 format %{ "CALL, runtime leaf nofp $meth" %}
16609
16610 ins_encode( aarch64_enc_java_to_runtime(meth) );
16611
16612 ins_pipe(pipe_class_call);
16613 %}
16614
16615 instruct CallNativeDirect(method meth)
16616 %{
16617 match(CallNative);
16618
16619 effect(USE meth);
16620
16621 ins_cost(CALL_COST);
16622
16623 format %{ "CALL, native $meth" %}
16624
16625 ins_encode( aarch64_enc_java_to_runtime(meth) );
16626
16627 ins_pipe(pipe_class_call);
16628 %}
16629
16630 // Tail Call; Jump from runtime stub to Java code.
16631 // Also known as an 'interprocedural jump'.
16632 // Target of jump will eventually return to caller.
16633 // TailJump below removes the return address.
16634 instruct TailCalljmpInd(iRegPNoSp jump_target, inline_cache_RegP method_ptr)
16635 %{
16636 match(TailCall jump_target method_ptr);
16637
16638 ins_cost(CALL_COST);
16639
16640 format %{ "br $jump_target\t# $method_ptr holds method" %}
16641
16642 ins_encode(aarch64_enc_tail_call(jump_target));
16643
16644 ins_pipe(pipe_class_call);
16645 %}
16646
16647 instruct TailjmpInd(iRegPNoSp jump_target, iRegP_R0 ex_oop)
16648 %{
16649 match(TailJump jump_target ex_oop);
16650
16651 ins_cost(CALL_COST);
16652
16653 format %{ "br $jump_target\t# $ex_oop holds exception oop" %}
16654
16655 ins_encode(aarch64_enc_tail_jmp(jump_target));
16656
16657 ins_pipe(pipe_class_call);
16658 %}
16659
16660 // Create exception oop: created by stack-crawling runtime code.
16661 // Created exception is now available to this handler, and is setup
16662 // just prior to jumping to this handler. No code emitted.
16663 // TODO check
16664 // should ex_oop be in r0? intel uses rax, ppc cannot use r0 so uses rarg1
16665 instruct CreateException(iRegP_R0 ex_oop)
16666 %{
16667 match(Set ex_oop (CreateEx));
16668
16669 format %{ " -- \t// exception oop; no code emitted" %}
16670
16671 size(0);
16672
16673 ins_encode( /*empty*/ );
16674
16675 ins_pipe(pipe_class_empty);
16676 %}
16677
16678 // Rethrow exception: The exception oop will come in the first
16679 // argument position. Then JUMP (not call) to the rethrow stub code.
16680 instruct RethrowException() %{
16681 match(Rethrow);
16682 ins_cost(CALL_COST);
16683
16684 format %{ "b rethrow_stub" %}
16685
16686 ins_encode( aarch64_enc_rethrow() );
16687
16688 ins_pipe(pipe_class_call);
16689 %}
16690
16691
16692 // Return Instruction
16693 // epilog node loads ret address into lr as part of frame pop
16694 instruct Ret()
16695 %{
16696 match(Return);
16697
16698 format %{ "ret\t// return register" %}
16699
16700 ins_encode( aarch64_enc_ret() );
16701
16702 ins_pipe(pipe_branch);
16703 %}
16704
16705 // Die now.
16706 instruct ShouldNotReachHere() %{
16707 match(Halt);
16708
16709 ins_cost(CALL_COST);
16710 format %{ "ShouldNotReachHere" %}
16711
16712 ins_encode %{
16713 if (is_reachable()) {
16714 __ stop(_halt_reason);
16715 }
16716 %}
16717
16718 ins_pipe(pipe_class_default);
16719 %}
16720
16721 // ============================================================================
16722 // Partial Subtype Check
16723 //
16724 // superklass array for an instance of the superklass. Set a hidden
16725 // internal cache on a hit (cache is checked with exposed code in
16726 // gen_subtype_check()). Return NZ for a miss or zero for a hit. The
16727 // encoding ALSO sets flags.
16728
16729 instruct partialSubtypeCheck(iRegP_R4 sub, iRegP_R0 super, iRegP_R2 temp, iRegP_R5 result, rFlagsReg cr)
16730 %{
16731 match(Set result (PartialSubtypeCheck sub super));
16732 effect(KILL cr, KILL temp);
16733
16734 ins_cost(1100); // slightly larger than the next version
16735 format %{ "partialSubtypeCheck $result, $sub, $super" %}
16736
16737 ins_encode(aarch64_enc_partial_subtype_check(sub, super, temp, result));
16738
16739 opcode(0x1); // Force zero of result reg on hit
16740
16741 ins_pipe(pipe_class_memory);
16742 %}
16743
16744 instruct partialSubtypeCheckVsZero(iRegP_R4 sub, iRegP_R0 super, iRegP_R2 temp, iRegP_R5 result, immP0 zero, rFlagsReg cr)
16745 %{
16746 match(Set cr (CmpP (PartialSubtypeCheck sub super) zero));
16747 effect(KILL temp, KILL result);
16748
16749 ins_cost(1100); // slightly larger than the next version
16750 format %{ "partialSubtypeCheck $result, $sub, $super == 0" %}
16751
16752 ins_encode(aarch64_enc_partial_subtype_check(sub, super, temp, result));
16753
16754 opcode(0x0); // Don't zero result reg on hit
16755
16756 ins_pipe(pipe_class_memory);
16757 %}
16758
16759 instruct string_compareU(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
16760 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2, rFlagsReg cr)
16761 %{
16762 predicate(((StrCompNode*)n)->encoding() == StrIntrinsicNode::UU);
16763 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
16764 effect(KILL tmp1, KILL tmp2, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
16765
16766 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # KILL $tmp1" %}
16767 ins_encode %{
16768 // Count is in 8-bit bytes; non-Compact chars are 16 bits.
16769 __ string_compare($str1$$Register, $str2$$Register,
16770 $cnt1$$Register, $cnt2$$Register, $result$$Register,
16771 $tmp1$$Register, $tmp2$$Register,
16772 fnoreg, fnoreg, fnoreg, StrIntrinsicNode::UU);
16773 %}
16774 ins_pipe(pipe_class_memory);
16775 %}
16776
16777 instruct string_compareL(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
16778 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2, rFlagsReg cr)
16779 %{
16780 predicate(((StrCompNode*)n)->encoding() == StrIntrinsicNode::LL);
16781 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
16782 effect(KILL tmp1, KILL tmp2, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
16783
16784 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # KILL $tmp1" %}
16785 ins_encode %{
16786 __ string_compare($str1$$Register, $str2$$Register,
16787 $cnt1$$Register, $cnt2$$Register, $result$$Register,
16788 $tmp1$$Register, $tmp2$$Register,
16789 fnoreg, fnoreg, fnoreg, StrIntrinsicNode::LL);
16790 %}
16791 ins_pipe(pipe_class_memory);
16792 %}
16793
16794 instruct string_compareUL(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
16795 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
16796 vRegD_V0 vtmp1, vRegD_V1 vtmp2, vRegD_V2 vtmp3, rFlagsReg cr)
16797 %{
16798 predicate(((StrCompNode*)n)->encoding() == StrIntrinsicNode::UL);
16799 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
16800 effect(KILL tmp1, KILL tmp2, KILL vtmp1, KILL vtmp2, KILL vtmp3,
16801 USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
16802
16803 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # KILL $tmp1, $tmp2, $vtmp1, $vtmp2, $vtmp3" %}
16804 ins_encode %{
16805 __ string_compare($str1$$Register, $str2$$Register,
16806 $cnt1$$Register, $cnt2$$Register, $result$$Register,
16807 $tmp1$$Register, $tmp2$$Register,
16808 $vtmp1$$FloatRegister, $vtmp2$$FloatRegister,
16809 $vtmp3$$FloatRegister, StrIntrinsicNode::UL);
16810 %}
16811 ins_pipe(pipe_class_memory);
16812 %}
16813
16814 instruct string_compareLU(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
16815 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
16816 vRegD_V0 vtmp1, vRegD_V1 vtmp2, vRegD_V2 vtmp3, rFlagsReg cr)
16817 %{
16818 predicate(((StrCompNode*)n)->encoding() == StrIntrinsicNode::LU);
16819 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
16820 effect(KILL tmp1, KILL tmp2, KILL vtmp1, KILL vtmp2, KILL vtmp3,
16821 USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
16822
16823 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # KILL $tmp1, $tmp2, $vtmp1, $vtmp2, $vtmp3" %}
16824 ins_encode %{
16825 __ string_compare($str1$$Register, $str2$$Register,
16826 $cnt1$$Register, $cnt2$$Register, $result$$Register,
16827 $tmp1$$Register, $tmp2$$Register,
16828 $vtmp1$$FloatRegister, $vtmp2$$FloatRegister,
16829 $vtmp3$$FloatRegister,StrIntrinsicNode::LU);
16830 %}
16831 ins_pipe(pipe_class_memory);
16832 %}
16833
16834 instruct string_indexofUU(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2, iRegI_R2 cnt2,
16835 iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2, iRegINoSp tmp3,
16836 iRegINoSp tmp4, iRegINoSp tmp5, iRegINoSp tmp6, rFlagsReg cr)
16837 %{
16838 predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UU);
16839 match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 cnt2)));
16840 effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2,
16841 TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, TEMP tmp5, TEMP tmp6, KILL cr);
16842 format %{ "String IndexOf $str1,$cnt1,$str2,$cnt2 -> $result (UU)" %}
16843
16844 ins_encode %{
16845 __ string_indexof($str1$$Register, $str2$$Register,
16846 $cnt1$$Register, $cnt2$$Register,
16847 $tmp1$$Register, $tmp2$$Register,
16848 $tmp3$$Register, $tmp4$$Register,
16849 $tmp5$$Register, $tmp6$$Register,
16850 -1, $result$$Register, StrIntrinsicNode::UU);
16851 %}
16852 ins_pipe(pipe_class_memory);
16853 %}
16854
16855 instruct string_indexofLL(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2, iRegI_R2 cnt2,
16856 iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2, iRegINoSp tmp3,
16857 iRegINoSp tmp4, iRegINoSp tmp5, iRegINoSp tmp6, rFlagsReg cr)
16858 %{
16859 predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::LL);
16860 match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 cnt2)));
16861 effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2,
16862 TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, TEMP tmp5, TEMP tmp6, KILL cr);
16863 format %{ "String IndexOf $str1,$cnt1,$str2,$cnt2 -> $result (LL)" %}
16864
16865 ins_encode %{
16866 __ string_indexof($str1$$Register, $str2$$Register,
16867 $cnt1$$Register, $cnt2$$Register,
16868 $tmp1$$Register, $tmp2$$Register,
16869 $tmp3$$Register, $tmp4$$Register,
16870 $tmp5$$Register, $tmp6$$Register,
16871 -1, $result$$Register, StrIntrinsicNode::LL);
16872 %}
16873 ins_pipe(pipe_class_memory);
16874 %}
16875
16876 instruct string_indexofUL(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2, iRegI_R2 cnt2,
16877 iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2, iRegINoSp tmp3,
16878 iRegINoSp tmp4, iRegINoSp tmp5, iRegINoSp tmp6, rFlagsReg cr)
16879 %{
16880 predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UL);
16881 match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 cnt2)));
16882 effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2,
16883 TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, TEMP tmp5, TEMP tmp6, KILL cr);
16884 format %{ "String IndexOf $str1,$cnt1,$str2,$cnt2 -> $result (UL)" %}
16885
16886 ins_encode %{
16887 __ string_indexof($str1$$Register, $str2$$Register,
16888 $cnt1$$Register, $cnt2$$Register,
16889 $tmp1$$Register, $tmp2$$Register,
16890 $tmp3$$Register, $tmp4$$Register,
16891 $tmp5$$Register, $tmp6$$Register,
16892 -1, $result$$Register, StrIntrinsicNode::UL);
16893 %}
16894 ins_pipe(pipe_class_memory);
16895 %}
16896
16897 instruct string_indexof_conUU(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2,
16898 immI_le_4 int_cnt2, iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2,
16899 iRegINoSp tmp3, iRegINoSp tmp4, rFlagsReg cr)
16900 %{
16901 predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UU);
16902 match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 int_cnt2)));
16903 effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1,
16904 TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, KILL cr);
16905 format %{ "String IndexOf $str1,$cnt1,$str2,$int_cnt2 -> $result (UU)" %}
16906
16907 ins_encode %{
16908 int icnt2 = (int)$int_cnt2$$constant;
16909 __ string_indexof($str1$$Register, $str2$$Register,
16910 $cnt1$$Register, zr,
16911 $tmp1$$Register, $tmp2$$Register,
16912 $tmp3$$Register, $tmp4$$Register, zr, zr,
16913 icnt2, $result$$Register, StrIntrinsicNode::UU);
16914 %}
16915 ins_pipe(pipe_class_memory);
16916 %}
16917
16918 instruct string_indexof_conLL(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2,
16919 immI_le_4 int_cnt2, iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2,
16920 iRegINoSp tmp3, iRegINoSp tmp4, rFlagsReg cr)
16921 %{
16922 predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::LL);
16923 match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 int_cnt2)));
16924 effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1,
16925 TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, KILL cr);
16926 format %{ "String IndexOf $str1,$cnt1,$str2,$int_cnt2 -> $result (LL)" %}
16927
16928 ins_encode %{
16929 int icnt2 = (int)$int_cnt2$$constant;
16930 __ string_indexof($str1$$Register, $str2$$Register,
16931 $cnt1$$Register, zr,
16932 $tmp1$$Register, $tmp2$$Register,
16933 $tmp3$$Register, $tmp4$$Register, zr, zr,
16934 icnt2, $result$$Register, StrIntrinsicNode::LL);
16935 %}
16936 ins_pipe(pipe_class_memory);
16937 %}
16938
16939 instruct string_indexof_conUL(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2,
16940 immI_1 int_cnt2, iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2,
16941 iRegINoSp tmp3, iRegINoSp tmp4, rFlagsReg cr)
16942 %{
16943 predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UL);
16944 match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 int_cnt2)));
16945 effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1,
16946 TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, KILL cr);
16947 format %{ "String IndexOf $str1,$cnt1,$str2,$int_cnt2 -> $result (UL)" %}
16948
16949 ins_encode %{
16950 int icnt2 = (int)$int_cnt2$$constant;
16951 __ string_indexof($str1$$Register, $str2$$Register,
16952 $cnt1$$Register, zr,
16953 $tmp1$$Register, $tmp2$$Register,
16954 $tmp3$$Register, $tmp4$$Register, zr, zr,
16955 icnt2, $result$$Register, StrIntrinsicNode::UL);
16956 %}
16957 ins_pipe(pipe_class_memory);
16958 %}
16959
16960 instruct string_indexof_char(iRegP_R1 str1, iRegI_R2 cnt1, iRegI_R3 ch,
16961 iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2,
16962 iRegINoSp tmp3, rFlagsReg cr)
16963 %{
16964 match(Set result (StrIndexOfChar (Binary str1 cnt1) ch));
16965 predicate(((StrIndexOfCharNode*)n)->encoding() == StrIntrinsicNode::U);
16966 effect(USE_KILL str1, USE_KILL cnt1, USE_KILL ch,
16967 TEMP tmp1, TEMP tmp2, TEMP tmp3, KILL cr);
16968
16969 format %{ "StringUTF16 IndexOf char[] $str1,$cnt1,$ch -> $result" %}
16970
16971 ins_encode %{
16972 __ string_indexof_char($str1$$Register, $cnt1$$Register, $ch$$Register,
16973 $result$$Register, $tmp1$$Register, $tmp2$$Register,
16974 $tmp3$$Register);
16975 %}
16976 ins_pipe(pipe_class_memory);
16977 %}
16978
16979 instruct stringL_indexof_char(iRegP_R1 str1, iRegI_R2 cnt1, iRegI_R3 ch,
16980 iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2,
16981 iRegINoSp tmp3, rFlagsReg cr)
16982 %{
16983 match(Set result (StrIndexOfChar (Binary str1 cnt1) ch));
16984 predicate(((StrIndexOfCharNode*)n)->encoding() == StrIntrinsicNode::L);
16985 effect(USE_KILL str1, USE_KILL cnt1, USE_KILL ch,
16986 TEMP tmp1, TEMP tmp2, TEMP tmp3, KILL cr);
16987
16988 format %{ "StringLatin1 IndexOf char[] $str1,$cnt1,$ch -> $result" %}
16989
16990 ins_encode %{
16991 __ stringL_indexof_char($str1$$Register, $cnt1$$Register, $ch$$Register,
16992 $result$$Register, $tmp1$$Register, $tmp2$$Register,
16993 $tmp3$$Register);
16994 %}
16995 ins_pipe(pipe_class_memory);
16996 %}
16997
16998 instruct string_equalsL(iRegP_R1 str1, iRegP_R3 str2, iRegI_R4 cnt,
16999 iRegI_R0 result, rFlagsReg cr)
17000 %{
17001 predicate(((StrEqualsNode*)n)->encoding() == StrIntrinsicNode::LL);
17002 match(Set result (StrEquals (Binary str1 str2) cnt));
17003 effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt, KILL cr);
17004
17005 format %{ "String Equals $str1,$str2,$cnt -> $result" %}
17006 ins_encode %{
17007 // Count is in 8-bit bytes; non-Compact chars are 16 bits.
17008 __ string_equals($str1$$Register, $str2$$Register,
17009 $result$$Register, $cnt$$Register, 1);
17010 %}
17011 ins_pipe(pipe_class_memory);
17012 %}
17013
17014 instruct string_equalsU(iRegP_R1 str1, iRegP_R3 str2, iRegI_R4 cnt,
17015 iRegI_R0 result, rFlagsReg cr)
17016 %{
17017 predicate(((StrEqualsNode*)n)->encoding() == StrIntrinsicNode::UU);
17018 match(Set result (StrEquals (Binary str1 str2) cnt));
17019 effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt, KILL cr);
17020
17021 format %{ "String Equals $str1,$str2,$cnt -> $result" %}
17022 ins_encode %{
17023 // Count is in 8-bit bytes; non-Compact chars are 16 bits.
17024 __ string_equals($str1$$Register, $str2$$Register,
17025 $result$$Register, $cnt$$Register, 2);
17026 %}
17027 ins_pipe(pipe_class_memory);
17028 %}
17029
17030 instruct array_equalsB(iRegP_R1 ary1, iRegP_R2 ary2, iRegI_R0 result,
17031 iRegP_R3 tmp1, iRegP_R4 tmp2, iRegP_R5 tmp3,
17032 iRegP_R10 tmp, rFlagsReg cr)
17033 %{
17034 predicate(((AryEqNode*)n)->encoding() == StrIntrinsicNode::LL);
17035 match(Set result (AryEq ary1 ary2));
17036 effect(KILL tmp, USE_KILL ary1, USE_KILL ary2, TEMP tmp1, TEMP tmp2, TEMP tmp3, KILL cr);
17037
17038 format %{ "Array Equals $ary1,ary2 -> $result // KILL $tmp" %}
17039 ins_encode %{
17040 address tpc = __ arrays_equals($ary1$$Register, $ary2$$Register,
17041 $tmp1$$Register, $tmp2$$Register, $tmp3$$Register,
17042 $result$$Register, $tmp$$Register, 1);
17043 if (tpc == NULL) {
17044 ciEnv::current()->record_failure("CodeCache is full");
17045 return;
17046 }
17047 %}
17048 ins_pipe(pipe_class_memory);
17049 %}
17050
17051 instruct array_equalsC(iRegP_R1 ary1, iRegP_R2 ary2, iRegI_R0 result,
17052 iRegP_R3 tmp1, iRegP_R4 tmp2, iRegP_R5 tmp3,
17053 iRegP_R10 tmp, rFlagsReg cr)
17054 %{
17055 predicate(((AryEqNode*)n)->encoding() == StrIntrinsicNode::UU);
17056 match(Set result (AryEq ary1 ary2));
17057 effect(KILL tmp, USE_KILL ary1, USE_KILL ary2, TEMP tmp1, TEMP tmp2, TEMP tmp3, KILL cr);
17058
17059 format %{ "Array Equals $ary1,ary2 -> $result // KILL $tmp" %}
17060 ins_encode %{
17061 address tpc = __ arrays_equals($ary1$$Register, $ary2$$Register,
17062 $tmp1$$Register, $tmp2$$Register, $tmp3$$Register,
17063 $result$$Register, $tmp$$Register, 2);
17064 if (tpc == NULL) {
17065 ciEnv::current()->record_failure("CodeCache is full");
17066 return;
17067 }
17068 %}
17069 ins_pipe(pipe_class_memory);
17070 %}
17071
17072 instruct has_negatives(iRegP_R1 ary1, iRegI_R2 len, iRegI_R0 result, rFlagsReg cr)
17073 %{
17074 match(Set result (HasNegatives ary1 len));
17075 effect(USE_KILL ary1, USE_KILL len, KILL cr);
17076 format %{ "has negatives byte[] $ary1,$len -> $result" %}
17077 ins_encode %{
17078 address tpc = __ has_negatives($ary1$$Register, $len$$Register, $result$$Register);
17079 if (tpc == NULL) {
17080 ciEnv::current()->record_failure("CodeCache is full");
17081 return;
17082 }
17083 %}
17084 ins_pipe( pipe_slow );
17085 %}
17086
17087 // fast char[] to byte[] compression
17088 instruct string_compress(iRegP_R2 src, iRegP_R1 dst, iRegI_R3 len,
17089 vRegD_V0 tmp1, vRegD_V1 tmp2,
17090 vRegD_V2 tmp3, vRegD_V3 tmp4,
17091 iRegI_R0 result, rFlagsReg cr)
17092 %{
17093 match(Set result (StrCompressedCopy src (Binary dst len)));
17094 effect(TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, USE_KILL src, USE_KILL dst, USE_KILL len, KILL cr);
17095
17096 format %{ "String Compress $src,$dst -> $result // KILL R1, R2, R3, R4" %}
17097 ins_encode %{
17098 __ char_array_compress($src$$Register, $dst$$Register, $len$$Register,
17099 $tmp1$$FloatRegister, $tmp2$$FloatRegister,
17100 $tmp3$$FloatRegister, $tmp4$$FloatRegister,
17101 $result$$Register);
17102 %}
17103 ins_pipe( pipe_slow );
17104 %}
17105
17106 // fast byte[] to char[] inflation
17107 instruct string_inflate(Universe dummy, iRegP_R0 src, iRegP_R1 dst, iRegI_R2 len,
17108 vRegD_V0 tmp1, vRegD_V1 tmp2, vRegD_V2 tmp3, iRegP_R3 tmp4, rFlagsReg cr)
17109 %{
17110 match(Set dummy (StrInflatedCopy src (Binary dst len)));
17111 effect(TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, USE_KILL src, USE_KILL dst, USE_KILL len, KILL cr);
17112
17113 format %{ "String Inflate $src,$dst // KILL $tmp1, $tmp2" %}
17114 ins_encode %{
17115 address tpc = __ byte_array_inflate($src$$Register, $dst$$Register, $len$$Register,
17116 $tmp1$$FloatRegister, $tmp2$$FloatRegister,
17117 $tmp3$$FloatRegister, $tmp4$$Register);
17118 if (tpc == NULL) {
17119 ciEnv::current()->record_failure("CodeCache is full");
17120 return;
17121 }
17122 %}
17123 ins_pipe(pipe_class_memory);
17124 %}
17125
17126 // encode char[] to byte[] in ISO_8859_1
17127 instruct encode_iso_array(iRegP_R2 src, iRegP_R1 dst, iRegI_R3 len,
17128 vRegD_V0 Vtmp1, vRegD_V1 Vtmp2,
17129 vRegD_V2 Vtmp3, vRegD_V3 Vtmp4,
17130 iRegI_R0 result, rFlagsReg cr)
17131 %{
17132 predicate(!((EncodeISOArrayNode*)n)->is_ascii());
17133 match(Set result (EncodeISOArray src (Binary dst len)));
17134 effect(USE_KILL src, USE_KILL dst, USE_KILL len,
17135 KILL Vtmp1, KILL Vtmp2, KILL Vtmp3, KILL Vtmp4, KILL cr);
17136
17137 format %{ "Encode array $src,$dst,$len -> $result" %}
17138 ins_encode %{
17139 __ encode_iso_array($src$$Register, $dst$$Register, $len$$Register,
17140 $result$$Register, $Vtmp1$$FloatRegister, $Vtmp2$$FloatRegister,
17141 $Vtmp3$$FloatRegister, $Vtmp4$$FloatRegister);
17142 %}
17143 ins_pipe( pipe_class_memory );
17144 %}
17145
17146 // ============================================================================
17147 // This name is KNOWN by the ADLC and cannot be changed.
17148 // The ADLC forces a 'TypeRawPtr::BOTTOM' output type
17149 // for this guy.
17150 instruct tlsLoadP(thread_RegP dst)
17151 %{
17152 match(Set dst (ThreadLocal));
17153
17154 ins_cost(0);
17155
17156 format %{ " -- \t// $dst=Thread::current(), empty" %}
17157
17158 size(0);
17159
17160 ins_encode( /*empty*/ );
17161
17162 ins_pipe(pipe_class_empty);
17163 %}
17164
17165 //----------PEEPHOLE RULES-----------------------------------------------------
17166 // These must follow all instruction definitions as they use the names
17167 // defined in the instructions definitions.
17168 //
17169 // peepmatch ( root_instr_name [preceding_instruction]* );
17170 //
17171 // peepconstraint %{
17172 // (instruction_number.operand_name relational_op instruction_number.operand_name
17173 // [, ...] );
17174 // // instruction numbers are zero-based using left to right order in peepmatch
17175 //
17176 // peepreplace ( instr_name ( [instruction_number.operand_name]* ) );
17177 // // provide an instruction_number.operand_name for each operand that appears
17178 // // in the replacement instruction's match rule
17179 //
17180 // ---------VM FLAGS---------------------------------------------------------
17181 //
17182 // All peephole optimizations can be turned off using -XX:-OptoPeephole
17183 //
17184 // Each peephole rule is given an identifying number starting with zero and
17185 // increasing by one in the order seen by the parser. An individual peephole
17186 // can be enabled, and all others disabled, by using -XX:OptoPeepholeAt=#
17187 // on the command-line.
17188 //
17189 // ---------CURRENT LIMITATIONS----------------------------------------------
17190 //
17191 // Only match adjacent instructions in same basic block
17192 // Only equality constraints
17193 // Only constraints between operands, not (0.dest_reg == RAX_enc)
17194 // Only one replacement instruction
17195 //
17196 // ---------EXAMPLE----------------------------------------------------------
17197 //
17198 // // pertinent parts of existing instructions in architecture description
17199 // instruct movI(iRegINoSp dst, iRegI src)
17200 // %{
17201 // match(Set dst (CopyI src));
17202 // %}
17203 //
17204 // instruct incI_iReg(iRegINoSp dst, immI1 src, rFlagsReg cr)
17205 // %{
17206 // match(Set dst (AddI dst src));
17207 // effect(KILL cr);
17208 // %}
17209 //
17210 // // Change (inc mov) to lea
17211 // peephole %{
17212 // // increment preceeded by register-register move
17213 // peepmatch ( incI_iReg movI );
17214 // // require that the destination register of the increment
17215 // // match the destination register of the move
17216 // peepconstraint ( 0.dst == 1.dst );
17217 // // construct a replacement instruction that sets
17218 // // the destination to ( move's source register + one )
17219 // peepreplace ( leaI_iReg_immI( 0.dst 1.src 0.src ) );
17220 // %}
17221 //
17222
17223 // Implementation no longer uses movX instructions since
17224 // machine-independent system no longer uses CopyX nodes.
17225 //
17226 // peephole
17227 // %{
17228 // peepmatch (incI_iReg movI);
17229 // peepconstraint (0.dst == 1.dst);
17230 // peepreplace (leaI_iReg_immI(0.dst 1.src 0.src));
17231 // %}
17232
17233 // peephole
17234 // %{
17235 // peepmatch (decI_iReg movI);
17236 // peepconstraint (0.dst == 1.dst);
17237 // peepreplace (leaI_iReg_immI(0.dst 1.src 0.src));
17238 // %}
17239
17240 // peephole
17241 // %{
17242 // peepmatch (addI_iReg_imm movI);
17243 // peepconstraint (0.dst == 1.dst);
17244 // peepreplace (leaI_iReg_immI(0.dst 1.src 0.src));
17245 // %}
17246
17247 // peephole
17248 // %{
17249 // peepmatch (incL_iReg movL);
17250 // peepconstraint (0.dst == 1.dst);
17251 // peepreplace (leaL_iReg_immL(0.dst 1.src 0.src));
17252 // %}
17253
17254 // peephole
17255 // %{
17256 // peepmatch (decL_iReg movL);
17257 // peepconstraint (0.dst == 1.dst);
17258 // peepreplace (leaL_iReg_immL(0.dst 1.src 0.src));
17259 // %}
17260
17261 // peephole
17262 // %{
17263 // peepmatch (addL_iReg_imm movL);
17264 // peepconstraint (0.dst == 1.dst);
17265 // peepreplace (leaL_iReg_immL(0.dst 1.src 0.src));
17266 // %}
17267
17268 // peephole
17269 // %{
17270 // peepmatch (addP_iReg_imm movP);
17271 // peepconstraint (0.dst == 1.dst);
17272 // peepreplace (leaP_iReg_imm(0.dst 1.src 0.src));
17273 // %}
17274
17275 // // Change load of spilled value to only a spill
17276 // instruct storeI(memory mem, iRegI src)
17277 // %{
17278 // match(Set mem (StoreI mem src));
17279 // %}
17280 //
17281 // instruct loadI(iRegINoSp dst, memory mem)
17282 // %{
17283 // match(Set dst (LoadI mem));
17284 // %}
17285 //
17286
17287 //----------SMARTSPILL RULES---------------------------------------------------
17288 // These must follow all instruction definitions as they use the names
17289 // defined in the instructions definitions.
17290
17291 // Local Variables:
17292 // mode: c++
17293 // End: