1 //
2 // Copyright (c) 2003, 2022, Oracle and/or its affiliates. All rights reserved.
3 // Copyright (c) 2014, 2021, Red Hat, Inc. All rights reserved.
4 // DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
5 //
6 // This code is free software; you can redistribute it and/or modify it
7 // under the terms of the GNU General Public License version 2 only, as
8 // published by the Free Software Foundation.
9 //
10 // This code is distributed in the hope that it will be useful, but WITHOUT
11 // ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
12 // FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
13 // version 2 for more details (a copy is included in the LICENSE file that
14 // accompanied this code).
15 //
16 // You should have received a copy of the GNU General Public License version
17 // 2 along with this work; if not, write to the Free Software Foundation,
18 // Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
19 //
20 // Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
21 // or visit www.oracle.com if you need additional information or have any
22 // questions.
23 //
24 //
25
26 // AArch64 Architecture Description File
27
28 //----------REGISTER DEFINITION BLOCK------------------------------------------
29 // This information is used by the matcher and the register allocator to
30 // describe individual registers and classes of registers within the target
31 // archtecture.
32
33 register %{
34 //----------Architecture Description Register Definitions----------------------
35 // General Registers
36 // "reg_def" name ( register save type, C convention save type,
37 // ideal register type, encoding );
38 // Register Save Types:
39 //
40 // NS = No-Save: The register allocator assumes that these registers
41 // can be used without saving upon entry to the method, &
42 // that they do not need to be saved at call sites.
43 //
44 // SOC = Save-On-Call: The register allocator assumes that these registers
45 // can be used without saving upon entry to the method,
46 // but that they must be saved at call sites.
47 //
48 // SOE = Save-On-Entry: The register allocator assumes that these registers
49 // must be saved before using them upon entry to the
50 // method, but they do not need to be saved at call
51 // sites.
52 //
53 // AS = Always-Save: The register allocator assumes that these registers
54 // must be saved before using them upon entry to the
55 // method, & that they must be saved at call sites.
56 //
57 // Ideal Register Type is used to determine how to save & restore a
58 // register. Op_RegI will get spilled with LoadI/StoreI, Op_RegP will get
59 // spilled with LoadP/StoreP. If the register supports both, use Op_RegI.
60 //
61 // The encoding number is the actual bit-pattern placed into the opcodes.
62
63 // We must define the 64 bit int registers in two 32 bit halves, the
64 // real lower register and a virtual upper half register. upper halves
65 // are used by the register allocator but are not actually supplied as
66 // operands to memory ops.
67 //
68 // follow the C1 compiler in making registers
69 //
70 // r0-r7,r10-r26 volatile (caller save)
71 // r27-r32 system (no save, no allocate)
72 // r8-r9 non-allocatable (so we can use them as scratch regs)
73 //
74 // as regards Java usage. we don't use any callee save registers
75 // because this makes it difficult to de-optimise a frame (see comment
76 // in x86 implementation of Deoptimization::unwind_callee_save_values)
77 //
78
79 // General Registers
80
81 reg_def R0 ( SOC, SOC, Op_RegI, 0, r0->as_VMReg() );
82 reg_def R0_H ( SOC, SOC, Op_RegI, 0, r0->as_VMReg()->next() );
83 reg_def R1 ( SOC, SOC, Op_RegI, 1, r1->as_VMReg() );
84 reg_def R1_H ( SOC, SOC, Op_RegI, 1, r1->as_VMReg()->next() );
85 reg_def R2 ( SOC, SOC, Op_RegI, 2, r2->as_VMReg() );
86 reg_def R2_H ( SOC, SOC, Op_RegI, 2, r2->as_VMReg()->next() );
87 reg_def R3 ( SOC, SOC, Op_RegI, 3, r3->as_VMReg() );
88 reg_def R3_H ( SOC, SOC, Op_RegI, 3, r3->as_VMReg()->next() );
89 reg_def R4 ( SOC, SOC, Op_RegI, 4, r4->as_VMReg() );
90 reg_def R4_H ( SOC, SOC, Op_RegI, 4, r4->as_VMReg()->next() );
91 reg_def R5 ( SOC, SOC, Op_RegI, 5, r5->as_VMReg() );
92 reg_def R5_H ( SOC, SOC, Op_RegI, 5, r5->as_VMReg()->next() );
93 reg_def R6 ( SOC, SOC, Op_RegI, 6, r6->as_VMReg() );
94 reg_def R6_H ( SOC, SOC, Op_RegI, 6, r6->as_VMReg()->next() );
95 reg_def R7 ( SOC, SOC, Op_RegI, 7, r7->as_VMReg() );
96 reg_def R7_H ( SOC, SOC, Op_RegI, 7, r7->as_VMReg()->next() );
97 reg_def R8 ( NS, SOC, Op_RegI, 8, r8->as_VMReg() ); // rscratch1, non-allocatable
98 reg_def R8_H ( NS, SOC, Op_RegI, 8, r8->as_VMReg()->next() );
99 reg_def R9 ( NS, SOC, Op_RegI, 9, r9->as_VMReg() ); // rscratch2, non-allocatable
100 reg_def R9_H ( NS, SOC, Op_RegI, 9, r9->as_VMReg()->next() );
101 reg_def R10 ( SOC, SOC, Op_RegI, 10, r10->as_VMReg() );
102 reg_def R10_H ( SOC, SOC, Op_RegI, 10, r10->as_VMReg()->next());
103 reg_def R11 ( SOC, SOC, Op_RegI, 11, r11->as_VMReg() );
104 reg_def R11_H ( SOC, SOC, Op_RegI, 11, r11->as_VMReg()->next());
105 reg_def R12 ( SOC, SOC, Op_RegI, 12, r12->as_VMReg() );
106 reg_def R12_H ( SOC, SOC, Op_RegI, 12, r12->as_VMReg()->next());
107 reg_def R13 ( SOC, SOC, Op_RegI, 13, r13->as_VMReg() );
108 reg_def R13_H ( SOC, SOC, Op_RegI, 13, r13->as_VMReg()->next());
109 reg_def R14 ( SOC, SOC, Op_RegI, 14, r14->as_VMReg() );
110 reg_def R14_H ( SOC, SOC, Op_RegI, 14, r14->as_VMReg()->next());
111 reg_def R15 ( SOC, SOC, Op_RegI, 15, r15->as_VMReg() );
112 reg_def R15_H ( SOC, SOC, Op_RegI, 15, r15->as_VMReg()->next());
113 reg_def R16 ( SOC, SOC, Op_RegI, 16, r16->as_VMReg() );
114 reg_def R16_H ( SOC, SOC, Op_RegI, 16, r16->as_VMReg()->next());
115 reg_def R17 ( SOC, SOC, Op_RegI, 17, r17->as_VMReg() );
116 reg_def R17_H ( SOC, SOC, Op_RegI, 17, r17->as_VMReg()->next());
117 reg_def R18 ( SOC, SOC, Op_RegI, 18, r18_tls->as_VMReg() );
118 reg_def R18_H ( SOC, SOC, Op_RegI, 18, r18_tls->as_VMReg()->next());
119 reg_def R19 ( SOC, SOE, Op_RegI, 19, r19->as_VMReg() );
120 reg_def R19_H ( SOC, SOE, Op_RegI, 19, r19->as_VMReg()->next());
121 reg_def R20 ( SOC, SOE, Op_RegI, 20, r20->as_VMReg() ); // caller esp
122 reg_def R20_H ( SOC, SOE, Op_RegI, 20, r20->as_VMReg()->next());
123 reg_def R21 ( SOC, SOE, Op_RegI, 21, r21->as_VMReg() );
124 reg_def R21_H ( SOC, SOE, Op_RegI, 21, r21->as_VMReg()->next());
125 reg_def R22 ( SOC, SOE, Op_RegI, 22, r22->as_VMReg() );
126 reg_def R22_H ( SOC, SOE, Op_RegI, 22, r22->as_VMReg()->next());
127 reg_def R23 ( SOC, SOE, Op_RegI, 23, r23->as_VMReg() );
128 reg_def R23_H ( SOC, SOE, Op_RegI, 23, r23->as_VMReg()->next());
129 reg_def R24 ( SOC, SOE, Op_RegI, 24, r24->as_VMReg() );
130 reg_def R24_H ( SOC, SOE, Op_RegI, 24, r24->as_VMReg()->next());
131 reg_def R25 ( SOC, SOE, Op_RegI, 25, r25->as_VMReg() );
132 reg_def R25_H ( SOC, SOE, Op_RegI, 25, r25->as_VMReg()->next());
133 reg_def R26 ( SOC, SOE, Op_RegI, 26, r26->as_VMReg() );
134 reg_def R26_H ( SOC, SOE, Op_RegI, 26, r26->as_VMReg()->next());
135 reg_def R27 ( SOC, SOE, Op_RegI, 27, r27->as_VMReg() ); // heapbase
136 reg_def R27_H ( SOC, SOE, Op_RegI, 27, r27->as_VMReg()->next());
137 reg_def R28 ( NS, SOE, Op_RegI, 28, r28->as_VMReg() ); // thread
138 reg_def R28_H ( NS, SOE, Op_RegI, 28, r28->as_VMReg()->next());
139 reg_def R29 ( NS, NS, Op_RegI, 29, r29->as_VMReg() ); // fp
140 reg_def R29_H ( NS, NS, Op_RegI, 29, r29->as_VMReg()->next());
141 reg_def R30 ( NS, NS, Op_RegI, 30, r30->as_VMReg() ); // lr
142 reg_def R30_H ( NS, NS, Op_RegI, 30, r30->as_VMReg()->next());
143 reg_def R31 ( NS, NS, Op_RegI, 31, r31_sp->as_VMReg() ); // sp
144 reg_def R31_H ( NS, NS, Op_RegI, 31, r31_sp->as_VMReg()->next());
145
146 // ----------------------------
147 // Float/Double/Vector Registers
148 // ----------------------------
149
150 // Double Registers
151
152 // The rules of ADL require that double registers be defined in pairs.
153 // Each pair must be two 32-bit values, but not necessarily a pair of
154 // single float registers. In each pair, ADLC-assigned register numbers
155 // must be adjacent, with the lower number even. Finally, when the
156 // CPU stores such a register pair to memory, the word associated with
157 // the lower ADLC-assigned number must be stored to the lower address.
158
159 // AArch64 has 32 floating-point registers. Each can store a vector of
160 // single or double precision floating-point values up to 8 * 32
161 // floats, 4 * 64 bit floats or 2 * 128 bit floats. We currently only
162 // use the first float or double element of the vector.
163
164 // for Java use float registers v0-v15 are always save on call whereas
165 // the platform ABI treats v8-v15 as callee save). float registers
166 // v16-v31 are SOC as per the platform spec
167
168 // For SVE vector registers, we simply extend vector register size to 8
169 // 'logical' slots. This is nominally 256 bits but it actually covers
170 // all possible 'physical' SVE vector register lengths from 128 ~ 2048
171 // bits. The 'physical' SVE vector register length is detected during
172 // startup, so the register allocator is able to identify the correct
173 // number of bytes needed for an SVE spill/unspill.
174 // Note that a vector register with 4 slots denotes a 128-bit NEON
175 // register allowing it to be distinguished from the corresponding SVE
176 // vector register when the SVE vector length is 128 bits.
177
178 reg_def V0 ( SOC, SOC, Op_RegF, 0, v0->as_VMReg() );
179 reg_def V0_H ( SOC, SOC, Op_RegF, 0, v0->as_VMReg()->next() );
180 reg_def V0_J ( SOC, SOC, Op_RegF, 0, v0->as_VMReg()->next(2) );
181 reg_def V0_K ( SOC, SOC, Op_RegF, 0, v0->as_VMReg()->next(3) );
182 reg_def V0_L ( SOC, SOC, Op_RegF, 0, v0->as_VMReg()->next(4) );
183 reg_def V0_M ( SOC, SOC, Op_RegF, 0, v0->as_VMReg()->next(5) );
184 reg_def V0_N ( SOC, SOC, Op_RegF, 0, v0->as_VMReg()->next(6) );
185 reg_def V0_O ( SOC, SOC, Op_RegF, 0, v0->as_VMReg()->next(7) );
186
187 reg_def V1 ( SOC, SOC, Op_RegF, 1, v1->as_VMReg() );
188 reg_def V1_H ( SOC, SOC, Op_RegF, 1, v1->as_VMReg()->next() );
189 reg_def V1_J ( SOC, SOC, Op_RegF, 1, v1->as_VMReg()->next(2) );
190 reg_def V1_K ( SOC, SOC, Op_RegF, 1, v1->as_VMReg()->next(3) );
191 reg_def V1_L ( SOC, SOC, Op_RegF, 1, v1->as_VMReg()->next(4) );
192 reg_def V1_M ( SOC, SOC, Op_RegF, 1, v1->as_VMReg()->next(5) );
193 reg_def V1_N ( SOC, SOC, Op_RegF, 1, v1->as_VMReg()->next(6) );
194 reg_def V1_O ( SOC, SOC, Op_RegF, 1, v1->as_VMReg()->next(7) );
195
196 reg_def V2 ( SOC, SOC, Op_RegF, 2, v2->as_VMReg() );
197 reg_def V2_H ( SOC, SOC, Op_RegF, 2, v2->as_VMReg()->next() );
198 reg_def V2_J ( SOC, SOC, Op_RegF, 2, v2->as_VMReg()->next(2) );
199 reg_def V2_K ( SOC, SOC, Op_RegF, 2, v2->as_VMReg()->next(3) );
200 reg_def V2_L ( SOC, SOC, Op_RegF, 2, v2->as_VMReg()->next(4) );
201 reg_def V2_M ( SOC, SOC, Op_RegF, 2, v2->as_VMReg()->next(5) );
202 reg_def V2_N ( SOC, SOC, Op_RegF, 2, v2->as_VMReg()->next(6) );
203 reg_def V2_O ( SOC, SOC, Op_RegF, 2, v2->as_VMReg()->next(7) );
204
205 reg_def V3 ( SOC, SOC, Op_RegF, 3, v3->as_VMReg() );
206 reg_def V3_H ( SOC, SOC, Op_RegF, 3, v3->as_VMReg()->next() );
207 reg_def V3_J ( SOC, SOC, Op_RegF, 3, v3->as_VMReg()->next(2) );
208 reg_def V3_K ( SOC, SOC, Op_RegF, 3, v3->as_VMReg()->next(3) );
209 reg_def V3_L ( SOC, SOC, Op_RegF, 3, v3->as_VMReg()->next(4) );
210 reg_def V3_M ( SOC, SOC, Op_RegF, 3, v3->as_VMReg()->next(5) );
211 reg_def V3_N ( SOC, SOC, Op_RegF, 3, v3->as_VMReg()->next(6) );
212 reg_def V3_O ( SOC, SOC, Op_RegF, 3, v3->as_VMReg()->next(7) );
213
214 reg_def V4 ( SOC, SOC, Op_RegF, 4, v4->as_VMReg() );
215 reg_def V4_H ( SOC, SOC, Op_RegF, 4, v4->as_VMReg()->next() );
216 reg_def V4_J ( SOC, SOC, Op_RegF, 4, v4->as_VMReg()->next(2) );
217 reg_def V4_K ( SOC, SOC, Op_RegF, 4, v4->as_VMReg()->next(3) );
218 reg_def V4_L ( SOC, SOC, Op_RegF, 4, v4->as_VMReg()->next(4) );
219 reg_def V4_M ( SOC, SOC, Op_RegF, 4, v4->as_VMReg()->next(5) );
220 reg_def V4_N ( SOC, SOC, Op_RegF, 4, v4->as_VMReg()->next(6) );
221 reg_def V4_O ( SOC, SOC, Op_RegF, 4, v4->as_VMReg()->next(7) );
222
223 reg_def V5 ( SOC, SOC, Op_RegF, 5, v5->as_VMReg() );
224 reg_def V5_H ( SOC, SOC, Op_RegF, 5, v5->as_VMReg()->next() );
225 reg_def V5_J ( SOC, SOC, Op_RegF, 5, v5->as_VMReg()->next(2) );
226 reg_def V5_K ( SOC, SOC, Op_RegF, 5, v5->as_VMReg()->next(3) );
227 reg_def V5_L ( SOC, SOC, Op_RegF, 5, v5->as_VMReg()->next(4) );
228 reg_def V5_M ( SOC, SOC, Op_RegF, 5, v5->as_VMReg()->next(5) );
229 reg_def V5_N ( SOC, SOC, Op_RegF, 5, v5->as_VMReg()->next(6) );
230 reg_def V5_O ( SOC, SOC, Op_RegF, 5, v5->as_VMReg()->next(7) );
231
232 reg_def V6 ( SOC, SOC, Op_RegF, 6, v6->as_VMReg() );
233 reg_def V6_H ( SOC, SOC, Op_RegF, 6, v6->as_VMReg()->next() );
234 reg_def V6_J ( SOC, SOC, Op_RegF, 6, v6->as_VMReg()->next(2) );
235 reg_def V6_K ( SOC, SOC, Op_RegF, 6, v6->as_VMReg()->next(3) );
236 reg_def V6_L ( SOC, SOC, Op_RegF, 6, v6->as_VMReg()->next(4) );
237 reg_def V6_M ( SOC, SOC, Op_RegF, 6, v6->as_VMReg()->next(5) );
238 reg_def V6_N ( SOC, SOC, Op_RegF, 6, v6->as_VMReg()->next(6) );
239 reg_def V6_O ( SOC, SOC, Op_RegF, 6, v6->as_VMReg()->next(7) );
240
241 reg_def V7 ( SOC, SOC, Op_RegF, 7, v7->as_VMReg() );
242 reg_def V7_H ( SOC, SOC, Op_RegF, 7, v7->as_VMReg()->next() );
243 reg_def V7_J ( SOC, SOC, Op_RegF, 7, v7->as_VMReg()->next(2) );
244 reg_def V7_K ( SOC, SOC, Op_RegF, 7, v7->as_VMReg()->next(3) );
245 reg_def V7_L ( SOC, SOC, Op_RegF, 7, v7->as_VMReg()->next(4) );
246 reg_def V7_M ( SOC, SOC, Op_RegF, 7, v7->as_VMReg()->next(5) );
247 reg_def V7_N ( SOC, SOC, Op_RegF, 7, v7->as_VMReg()->next(6) );
248 reg_def V7_O ( SOC, SOC, Op_RegF, 7, v7->as_VMReg()->next(7) );
249
250 reg_def V8 ( SOC, SOE, Op_RegF, 8, v8->as_VMReg() );
251 reg_def V8_H ( SOC, SOE, Op_RegF, 8, v8->as_VMReg()->next() );
252 reg_def V8_J ( SOC, SOC, Op_RegF, 8, v8->as_VMReg()->next(2) );
253 reg_def V8_K ( SOC, SOC, Op_RegF, 8, v8->as_VMReg()->next(3) );
254 reg_def V8_L ( SOC, SOC, Op_RegF, 8, v8->as_VMReg()->next(4) );
255 reg_def V8_M ( SOC, SOC, Op_RegF, 8, v8->as_VMReg()->next(5) );
256 reg_def V8_N ( SOC, SOC, Op_RegF, 8, v8->as_VMReg()->next(6) );
257 reg_def V8_O ( SOC, SOC, Op_RegF, 8, v8->as_VMReg()->next(7) );
258
259 reg_def V9 ( SOC, SOE, Op_RegF, 9, v9->as_VMReg() );
260 reg_def V9_H ( SOC, SOE, Op_RegF, 9, v9->as_VMReg()->next() );
261 reg_def V9_J ( SOC, SOC, Op_RegF, 9, v9->as_VMReg()->next(2) );
262 reg_def V9_K ( SOC, SOC, Op_RegF, 9, v9->as_VMReg()->next(3) );
263 reg_def V9_L ( SOC, SOC, Op_RegF, 9, v9->as_VMReg()->next(4) );
264 reg_def V9_M ( SOC, SOC, Op_RegF, 9, v9->as_VMReg()->next(5) );
265 reg_def V9_N ( SOC, SOC, Op_RegF, 9, v9->as_VMReg()->next(6) );
266 reg_def V9_O ( SOC, SOC, Op_RegF, 9, v9->as_VMReg()->next(7) );
267
268 reg_def V10 ( SOC, SOE, Op_RegF, 10, v10->as_VMReg() );
269 reg_def V10_H ( SOC, SOE, Op_RegF, 10, v10->as_VMReg()->next() );
270 reg_def V10_J ( SOC, SOC, Op_RegF, 10, v10->as_VMReg()->next(2) );
271 reg_def V10_K ( SOC, SOC, Op_RegF, 10, v10->as_VMReg()->next(3) );
272 reg_def V10_L ( SOC, SOC, Op_RegF, 10, v10->as_VMReg()->next(4) );
273 reg_def V10_M ( SOC, SOC, Op_RegF, 10, v10->as_VMReg()->next(5) );
274 reg_def V10_N ( SOC, SOC, Op_RegF, 10, v10->as_VMReg()->next(6) );
275 reg_def V10_O ( SOC, SOC, Op_RegF, 10, v10->as_VMReg()->next(7) );
276
277 reg_def V11 ( SOC, SOE, Op_RegF, 11, v11->as_VMReg() );
278 reg_def V11_H ( SOC, SOE, Op_RegF, 11, v11->as_VMReg()->next() );
279 reg_def V11_J ( SOC, SOC, Op_RegF, 11, v11->as_VMReg()->next(2) );
280 reg_def V11_K ( SOC, SOC, Op_RegF, 11, v11->as_VMReg()->next(3) );
281 reg_def V11_L ( SOC, SOC, Op_RegF, 11, v11->as_VMReg()->next(4) );
282 reg_def V11_M ( SOC, SOC, Op_RegF, 11, v11->as_VMReg()->next(5) );
283 reg_def V11_N ( SOC, SOC, Op_RegF, 11, v11->as_VMReg()->next(6) );
284 reg_def V11_O ( SOC, SOC, Op_RegF, 11, v11->as_VMReg()->next(7) );
285
286 reg_def V12 ( SOC, SOE, Op_RegF, 12, v12->as_VMReg() );
287 reg_def V12_H ( SOC, SOE, Op_RegF, 12, v12->as_VMReg()->next() );
288 reg_def V12_J ( SOC, SOC, Op_RegF, 12, v12->as_VMReg()->next(2) );
289 reg_def V12_K ( SOC, SOC, Op_RegF, 12, v12->as_VMReg()->next(3) );
290 reg_def V12_L ( SOC, SOC, Op_RegF, 12, v12->as_VMReg()->next(4) );
291 reg_def V12_M ( SOC, SOC, Op_RegF, 12, v12->as_VMReg()->next(5) );
292 reg_def V12_N ( SOC, SOC, Op_RegF, 12, v12->as_VMReg()->next(6) );
293 reg_def V12_O ( SOC, SOC, Op_RegF, 12, v12->as_VMReg()->next(7) );
294
295 reg_def V13 ( SOC, SOE, Op_RegF, 13, v13->as_VMReg() );
296 reg_def V13_H ( SOC, SOE, Op_RegF, 13, v13->as_VMReg()->next() );
297 reg_def V13_J ( SOC, SOC, Op_RegF, 13, v13->as_VMReg()->next(2) );
298 reg_def V13_K ( SOC, SOC, Op_RegF, 13, v13->as_VMReg()->next(3) );
299 reg_def V13_L ( SOC, SOC, Op_RegF, 13, v13->as_VMReg()->next(4) );
300 reg_def V13_M ( SOC, SOC, Op_RegF, 13, v13->as_VMReg()->next(5) );
301 reg_def V13_N ( SOC, SOC, Op_RegF, 13, v13->as_VMReg()->next(6) );
302 reg_def V13_O ( SOC, SOC, Op_RegF, 13, v13->as_VMReg()->next(7) );
303
304 reg_def V14 ( SOC, SOE, Op_RegF, 14, v14->as_VMReg() );
305 reg_def V14_H ( SOC, SOE, Op_RegF, 14, v14->as_VMReg()->next() );
306 reg_def V14_J ( SOC, SOC, Op_RegF, 14, v14->as_VMReg()->next(2) );
307 reg_def V14_K ( SOC, SOC, Op_RegF, 14, v14->as_VMReg()->next(3) );
308 reg_def V14_L ( SOC, SOC, Op_RegF, 14, v14->as_VMReg()->next(4) );
309 reg_def V14_M ( SOC, SOC, Op_RegF, 14, v14->as_VMReg()->next(5) );
310 reg_def V14_N ( SOC, SOC, Op_RegF, 14, v14->as_VMReg()->next(6) );
311 reg_def V14_O ( SOC, SOC, Op_RegF, 14, v14->as_VMReg()->next(7) );
312
313 reg_def V15 ( SOC, SOE, Op_RegF, 15, v15->as_VMReg() );
314 reg_def V15_H ( SOC, SOE, Op_RegF, 15, v15->as_VMReg()->next() );
315 reg_def V15_J ( SOC, SOC, Op_RegF, 15, v15->as_VMReg()->next(2) );
316 reg_def V15_K ( SOC, SOC, Op_RegF, 15, v15->as_VMReg()->next(3) );
317 reg_def V15_L ( SOC, SOC, Op_RegF, 15, v15->as_VMReg()->next(4) );
318 reg_def V15_M ( SOC, SOC, Op_RegF, 15, v15->as_VMReg()->next(5) );
319 reg_def V15_N ( SOC, SOC, Op_RegF, 15, v15->as_VMReg()->next(6) );
320 reg_def V15_O ( SOC, SOC, Op_RegF, 15, v15->as_VMReg()->next(7) );
321
322 reg_def V16 ( SOC, SOC, Op_RegF, 16, v16->as_VMReg() );
323 reg_def V16_H ( SOC, SOC, Op_RegF, 16, v16->as_VMReg()->next() );
324 reg_def V16_J ( SOC, SOC, Op_RegF, 16, v16->as_VMReg()->next(2) );
325 reg_def V16_K ( SOC, SOC, Op_RegF, 16, v16->as_VMReg()->next(3) );
326 reg_def V16_L ( SOC, SOC, Op_RegF, 16, v16->as_VMReg()->next(4) );
327 reg_def V16_M ( SOC, SOC, Op_RegF, 16, v16->as_VMReg()->next(5) );
328 reg_def V16_N ( SOC, SOC, Op_RegF, 16, v16->as_VMReg()->next(6) );
329 reg_def V16_O ( SOC, SOC, Op_RegF, 16, v16->as_VMReg()->next(7) );
330
331 reg_def V17 ( SOC, SOC, Op_RegF, 17, v17->as_VMReg() );
332 reg_def V17_H ( SOC, SOC, Op_RegF, 17, v17->as_VMReg()->next() );
333 reg_def V17_J ( SOC, SOC, Op_RegF, 17, v17->as_VMReg()->next(2) );
334 reg_def V17_K ( SOC, SOC, Op_RegF, 17, v17->as_VMReg()->next(3) );
335 reg_def V17_L ( SOC, SOC, Op_RegF, 17, v17->as_VMReg()->next(4) );
336 reg_def V17_M ( SOC, SOC, Op_RegF, 17, v17->as_VMReg()->next(5) );
337 reg_def V17_N ( SOC, SOC, Op_RegF, 17, v17->as_VMReg()->next(6) );
338 reg_def V17_O ( SOC, SOC, Op_RegF, 17, v17->as_VMReg()->next(7) );
339
340 reg_def V18 ( SOC, SOC, Op_RegF, 18, v18->as_VMReg() );
341 reg_def V18_H ( SOC, SOC, Op_RegF, 18, v18->as_VMReg()->next() );
342 reg_def V18_J ( SOC, SOC, Op_RegF, 18, v18->as_VMReg()->next(2) );
343 reg_def V18_K ( SOC, SOC, Op_RegF, 18, v18->as_VMReg()->next(3) );
344 reg_def V18_L ( SOC, SOC, Op_RegF, 18, v18->as_VMReg()->next(4) );
345 reg_def V18_M ( SOC, SOC, Op_RegF, 18, v18->as_VMReg()->next(5) );
346 reg_def V18_N ( SOC, SOC, Op_RegF, 18, v18->as_VMReg()->next(6) );
347 reg_def V18_O ( SOC, SOC, Op_RegF, 18, v18->as_VMReg()->next(7) );
348
349 reg_def V19 ( SOC, SOC, Op_RegF, 19, v19->as_VMReg() );
350 reg_def V19_H ( SOC, SOC, Op_RegF, 19, v19->as_VMReg()->next() );
351 reg_def V19_J ( SOC, SOC, Op_RegF, 19, v19->as_VMReg()->next(2) );
352 reg_def V19_K ( SOC, SOC, Op_RegF, 19, v19->as_VMReg()->next(3) );
353 reg_def V19_L ( SOC, SOC, Op_RegF, 19, v19->as_VMReg()->next(4) );
354 reg_def V19_M ( SOC, SOC, Op_RegF, 19, v19->as_VMReg()->next(5) );
355 reg_def V19_N ( SOC, SOC, Op_RegF, 19, v19->as_VMReg()->next(6) );
356 reg_def V19_O ( SOC, SOC, Op_RegF, 19, v19->as_VMReg()->next(7) );
357
358 reg_def V20 ( SOC, SOC, Op_RegF, 20, v20->as_VMReg() );
359 reg_def V20_H ( SOC, SOC, Op_RegF, 20, v20->as_VMReg()->next() );
360 reg_def V20_J ( SOC, SOC, Op_RegF, 20, v20->as_VMReg()->next(2) );
361 reg_def V20_K ( SOC, SOC, Op_RegF, 20, v20->as_VMReg()->next(3) );
362 reg_def V20_L ( SOC, SOC, Op_RegF, 20, v20->as_VMReg()->next(4) );
363 reg_def V20_M ( SOC, SOC, Op_RegF, 20, v20->as_VMReg()->next(5) );
364 reg_def V20_N ( SOC, SOC, Op_RegF, 20, v20->as_VMReg()->next(6) );
365 reg_def V20_O ( SOC, SOC, Op_RegF, 20, v20->as_VMReg()->next(7) );
366
367 reg_def V21 ( SOC, SOC, Op_RegF, 21, v21->as_VMReg() );
368 reg_def V21_H ( SOC, SOC, Op_RegF, 21, v21->as_VMReg()->next() );
369 reg_def V21_J ( SOC, SOC, Op_RegF, 21, v21->as_VMReg()->next(2) );
370 reg_def V21_K ( SOC, SOC, Op_RegF, 21, v21->as_VMReg()->next(3) );
371 reg_def V21_L ( SOC, SOC, Op_RegF, 21, v21->as_VMReg()->next(4) );
372 reg_def V21_M ( SOC, SOC, Op_RegF, 21, v21->as_VMReg()->next(5) );
373 reg_def V21_N ( SOC, SOC, Op_RegF, 21, v21->as_VMReg()->next(6) );
374 reg_def V21_O ( SOC, SOC, Op_RegF, 21, v21->as_VMReg()->next(7) );
375
376 reg_def V22 ( SOC, SOC, Op_RegF, 22, v22->as_VMReg() );
377 reg_def V22_H ( SOC, SOC, Op_RegF, 22, v22->as_VMReg()->next() );
378 reg_def V22_J ( SOC, SOC, Op_RegF, 22, v22->as_VMReg()->next(2) );
379 reg_def V22_K ( SOC, SOC, Op_RegF, 22, v22->as_VMReg()->next(3) );
380 reg_def V22_L ( SOC, SOC, Op_RegF, 22, v22->as_VMReg()->next(4) );
381 reg_def V22_M ( SOC, SOC, Op_RegF, 22, v22->as_VMReg()->next(5) );
382 reg_def V22_N ( SOC, SOC, Op_RegF, 22, v22->as_VMReg()->next(6) );
383 reg_def V22_O ( SOC, SOC, Op_RegF, 22, v22->as_VMReg()->next(7) );
384
385 reg_def V23 ( SOC, SOC, Op_RegF, 23, v23->as_VMReg() );
386 reg_def V23_H ( SOC, SOC, Op_RegF, 23, v23->as_VMReg()->next() );
387 reg_def V23_J ( SOC, SOC, Op_RegF, 23, v23->as_VMReg()->next(2) );
388 reg_def V23_K ( SOC, SOC, Op_RegF, 23, v23->as_VMReg()->next(3) );
389 reg_def V23_L ( SOC, SOC, Op_RegF, 23, v23->as_VMReg()->next(4) );
390 reg_def V23_M ( SOC, SOC, Op_RegF, 23, v23->as_VMReg()->next(5) );
391 reg_def V23_N ( SOC, SOC, Op_RegF, 23, v23->as_VMReg()->next(6) );
392 reg_def V23_O ( SOC, SOC, Op_RegF, 23, v23->as_VMReg()->next(7) );
393
394 reg_def V24 ( SOC, SOC, Op_RegF, 24, v24->as_VMReg() );
395 reg_def V24_H ( SOC, SOC, Op_RegF, 24, v24->as_VMReg()->next() );
396 reg_def V24_J ( SOC, SOC, Op_RegF, 24, v24->as_VMReg()->next(2) );
397 reg_def V24_K ( SOC, SOC, Op_RegF, 24, v24->as_VMReg()->next(3) );
398 reg_def V24_L ( SOC, SOC, Op_RegF, 24, v24->as_VMReg()->next(4) );
399 reg_def V24_M ( SOC, SOC, Op_RegF, 24, v24->as_VMReg()->next(5) );
400 reg_def V24_N ( SOC, SOC, Op_RegF, 24, v24->as_VMReg()->next(6) );
401 reg_def V24_O ( SOC, SOC, Op_RegF, 24, v24->as_VMReg()->next(7) );
402
403 reg_def V25 ( SOC, SOC, Op_RegF, 25, v25->as_VMReg() );
404 reg_def V25_H ( SOC, SOC, Op_RegF, 25, v25->as_VMReg()->next() );
405 reg_def V25_J ( SOC, SOC, Op_RegF, 25, v25->as_VMReg()->next(2) );
406 reg_def V25_K ( SOC, SOC, Op_RegF, 25, v25->as_VMReg()->next(3) );
407 reg_def V25_L ( SOC, SOC, Op_RegF, 25, v25->as_VMReg()->next(4) );
408 reg_def V25_M ( SOC, SOC, Op_RegF, 25, v25->as_VMReg()->next(5) );
409 reg_def V25_N ( SOC, SOC, Op_RegF, 25, v25->as_VMReg()->next(6) );
410 reg_def V25_O ( SOC, SOC, Op_RegF, 25, v25->as_VMReg()->next(7) );
411
412 reg_def V26 ( SOC, SOC, Op_RegF, 26, v26->as_VMReg() );
413 reg_def V26_H ( SOC, SOC, Op_RegF, 26, v26->as_VMReg()->next() );
414 reg_def V26_J ( SOC, SOC, Op_RegF, 26, v26->as_VMReg()->next(2) );
415 reg_def V26_K ( SOC, SOC, Op_RegF, 26, v26->as_VMReg()->next(3) );
416 reg_def V26_L ( SOC, SOC, Op_RegF, 26, v26->as_VMReg()->next(4) );
417 reg_def V26_M ( SOC, SOC, Op_RegF, 26, v26->as_VMReg()->next(5) );
418 reg_def V26_N ( SOC, SOC, Op_RegF, 26, v26->as_VMReg()->next(6) );
419 reg_def V26_O ( SOC, SOC, Op_RegF, 26, v26->as_VMReg()->next(7) );
420
421 reg_def V27 ( SOC, SOC, Op_RegF, 27, v27->as_VMReg() );
422 reg_def V27_H ( SOC, SOC, Op_RegF, 27, v27->as_VMReg()->next() );
423 reg_def V27_J ( SOC, SOC, Op_RegF, 27, v27->as_VMReg()->next(2) );
424 reg_def V27_K ( SOC, SOC, Op_RegF, 27, v27->as_VMReg()->next(3) );
425 reg_def V27_L ( SOC, SOC, Op_RegF, 27, v27->as_VMReg()->next(4) );
426 reg_def V27_M ( SOC, SOC, Op_RegF, 27, v27->as_VMReg()->next(5) );
427 reg_def V27_N ( SOC, SOC, Op_RegF, 27, v27->as_VMReg()->next(6) );
428 reg_def V27_O ( SOC, SOC, Op_RegF, 27, v27->as_VMReg()->next(7) );
429
430 reg_def V28 ( SOC, SOC, Op_RegF, 28, v28->as_VMReg() );
431 reg_def V28_H ( SOC, SOC, Op_RegF, 28, v28->as_VMReg()->next() );
432 reg_def V28_J ( SOC, SOC, Op_RegF, 28, v28->as_VMReg()->next(2) );
433 reg_def V28_K ( SOC, SOC, Op_RegF, 28, v28->as_VMReg()->next(3) );
434 reg_def V28_L ( SOC, SOC, Op_RegF, 28, v28->as_VMReg()->next(4) );
435 reg_def V28_M ( SOC, SOC, Op_RegF, 28, v28->as_VMReg()->next(5) );
436 reg_def V28_N ( SOC, SOC, Op_RegF, 28, v28->as_VMReg()->next(6) );
437 reg_def V28_O ( SOC, SOC, Op_RegF, 28, v28->as_VMReg()->next(7) );
438
439 reg_def V29 ( SOC, SOC, Op_RegF, 29, v29->as_VMReg() );
440 reg_def V29_H ( SOC, SOC, Op_RegF, 29, v29->as_VMReg()->next() );
441 reg_def V29_J ( SOC, SOC, Op_RegF, 29, v29->as_VMReg()->next(2) );
442 reg_def V29_K ( SOC, SOC, Op_RegF, 29, v29->as_VMReg()->next(3) );
443 reg_def V29_L ( SOC, SOC, Op_RegF, 29, v29->as_VMReg()->next(4) );
444 reg_def V29_M ( SOC, SOC, Op_RegF, 29, v29->as_VMReg()->next(5) );
445 reg_def V29_N ( SOC, SOC, Op_RegF, 29, v29->as_VMReg()->next(6) );
446 reg_def V29_O ( SOC, SOC, Op_RegF, 29, v29->as_VMReg()->next(7) );
447
448 reg_def V30 ( SOC, SOC, Op_RegF, 30, v30->as_VMReg() );
449 reg_def V30_H ( SOC, SOC, Op_RegF, 30, v30->as_VMReg()->next() );
450 reg_def V30_J ( SOC, SOC, Op_RegF, 30, v30->as_VMReg()->next(2) );
451 reg_def V30_K ( SOC, SOC, Op_RegF, 30, v30->as_VMReg()->next(3) );
452 reg_def V30_L ( SOC, SOC, Op_RegF, 30, v30->as_VMReg()->next(4) );
453 reg_def V30_M ( SOC, SOC, Op_RegF, 30, v30->as_VMReg()->next(5) );
454 reg_def V30_N ( SOC, SOC, Op_RegF, 30, v30->as_VMReg()->next(6) );
455 reg_def V30_O ( SOC, SOC, Op_RegF, 30, v30->as_VMReg()->next(7) );
456
457 reg_def V31 ( SOC, SOC, Op_RegF, 31, v31->as_VMReg() );
458 reg_def V31_H ( SOC, SOC, Op_RegF, 31, v31->as_VMReg()->next() );
459 reg_def V31_J ( SOC, SOC, Op_RegF, 31, v31->as_VMReg()->next(2) );
460 reg_def V31_K ( SOC, SOC, Op_RegF, 31, v31->as_VMReg()->next(3) );
461 reg_def V31_L ( SOC, SOC, Op_RegF, 31, v31->as_VMReg()->next(4) );
462 reg_def V31_M ( SOC, SOC, Op_RegF, 31, v31->as_VMReg()->next(5) );
463 reg_def V31_N ( SOC, SOC, Op_RegF, 31, v31->as_VMReg()->next(6) );
464 reg_def V31_O ( SOC, SOC, Op_RegF, 31, v31->as_VMReg()->next(7) );
465
466
467 // ----------------------------
468 // SVE Predicate Registers
469 // ----------------------------
470 reg_def P0 (SOC, SOC, Op_RegVectMask, 0, p0->as_VMReg());
471 reg_def P1 (SOC, SOC, Op_RegVectMask, 1, p1->as_VMReg());
472 reg_def P2 (SOC, SOC, Op_RegVectMask, 2, p2->as_VMReg());
473 reg_def P3 (SOC, SOC, Op_RegVectMask, 3, p3->as_VMReg());
474 reg_def P4 (SOC, SOC, Op_RegVectMask, 4, p4->as_VMReg());
475 reg_def P5 (SOC, SOC, Op_RegVectMask, 5, p5->as_VMReg());
476 reg_def P6 (SOC, SOC, Op_RegVectMask, 6, p6->as_VMReg());
477 reg_def P7 (SOC, SOC, Op_RegVectMask, 7, p7->as_VMReg());
478 reg_def P8 (SOC, SOC, Op_RegVectMask, 8, p8->as_VMReg());
479 reg_def P9 (SOC, SOC, Op_RegVectMask, 9, p9->as_VMReg());
480 reg_def P10 (SOC, SOC, Op_RegVectMask, 10, p10->as_VMReg());
481 reg_def P11 (SOC, SOC, Op_RegVectMask, 11, p11->as_VMReg());
482 reg_def P12 (SOC, SOC, Op_RegVectMask, 12, p12->as_VMReg());
483 reg_def P13 (SOC, SOC, Op_RegVectMask, 13, p13->as_VMReg());
484 reg_def P14 (SOC, SOC, Op_RegVectMask, 14, p14->as_VMReg());
485 reg_def P15 (SOC, SOC, Op_RegVectMask, 15, p15->as_VMReg());
486
487 // ----------------------------
488 // Special Registers
489 // ----------------------------
490
491 // the AArch64 CSPR status flag register is not directly acessible as
492 // instruction operand. the FPSR status flag register is a system
493 // register which can be written/read using MSR/MRS but again does not
494 // appear as an operand (a code identifying the FSPR occurs as an
495 // immediate value in the instruction).
496
497 reg_def RFLAGS(SOC, SOC, 0, 32, VMRegImpl::Bad());
498
499 // Specify priority of register selection within phases of register
500 // allocation. Highest priority is first. A useful heuristic is to
501 // give registers a low priority when they are required by machine
502 // instructions, like EAX and EDX on I486, and choose no-save registers
503 // before save-on-call, & save-on-call before save-on-entry. Registers
504 // which participate in fixed calling sequences should come last.
505 // Registers which are used as pairs must fall on an even boundary.
506
507 alloc_class chunk0(
508 // volatiles
509 R10, R10_H,
510 R11, R11_H,
511 R12, R12_H,
512 R13, R13_H,
513 R14, R14_H,
514 R15, R15_H,
515 R16, R16_H,
516 R17, R17_H,
517 R18, R18_H,
518
519 // arg registers
520 R0, R0_H,
521 R1, R1_H,
522 R2, R2_H,
523 R3, R3_H,
524 R4, R4_H,
525 R5, R5_H,
526 R6, R6_H,
527 R7, R7_H,
528
529 // non-volatiles
530 R19, R19_H,
531 R20, R20_H,
532 R21, R21_H,
533 R22, R22_H,
534 R23, R23_H,
535 R24, R24_H,
536 R25, R25_H,
537 R26, R26_H,
538
539 // non-allocatable registers
540
541 R27, R27_H, // heapbase
542 R28, R28_H, // thread
543 R29, R29_H, // fp
544 R30, R30_H, // lr
545 R31, R31_H, // sp
546 R8, R8_H, // rscratch1
547 R9, R9_H, // rscratch2
548 );
549
550 alloc_class chunk1(
551
552 // no save
553 V16, V16_H, V16_J, V16_K, V16_L, V16_M, V16_N, V16_O,
554 V17, V17_H, V17_J, V17_K, V17_L, V17_M, V17_N, V17_O,
555 V18, V18_H, V18_J, V18_K, V18_L, V18_M, V18_N, V18_O,
556 V19, V19_H, V19_J, V19_K, V19_L, V19_M, V19_N, V19_O,
557 V20, V20_H, V20_J, V20_K, V20_L, V20_M, V20_N, V20_O,
558 V21, V21_H, V21_J, V21_K, V21_L, V21_M, V21_N, V21_O,
559 V22, V22_H, V22_J, V22_K, V22_L, V22_M, V22_N, V22_O,
560 V23, V23_H, V23_J, V23_K, V23_L, V23_M, V23_N, V23_O,
561 V24, V24_H, V24_J, V24_K, V24_L, V24_M, V24_N, V24_O,
562 V25, V25_H, V25_J, V25_K, V25_L, V25_M, V25_N, V25_O,
563 V26, V26_H, V26_J, V26_K, V26_L, V26_M, V26_N, V26_O,
564 V27, V27_H, V27_J, V27_K, V27_L, V27_M, V27_N, V27_O,
565 V28, V28_H, V28_J, V28_K, V28_L, V28_M, V28_N, V28_O,
566 V29, V29_H, V29_J, V29_K, V29_L, V29_M, V29_N, V29_O,
567 V30, V30_H, V30_J, V30_K, V30_L, V30_M, V30_N, V30_O,
568 V31, V31_H, V31_J, V31_K, V31_L, V31_M, V31_N, V31_O,
569
570 // arg registers
571 V0, V0_H, V0_J, V0_K, V0_L, V0_M, V0_N, V0_O,
572 V1, V1_H, V1_J, V1_K, V1_L, V1_M, V1_N, V1_O,
573 V2, V2_H, V2_J, V2_K, V2_L, V2_M, V2_N, V2_O,
574 V3, V3_H, V3_J, V3_K, V3_L, V3_M, V3_N, V3_O,
575 V4, V4_H, V4_J, V4_K, V4_L, V4_M, V4_N, V4_O,
576 V5, V5_H, V5_J, V5_K, V5_L, V5_M, V5_N, V5_O,
577 V6, V6_H, V6_J, V6_K, V6_L, V6_M, V6_N, V6_O,
578 V7, V7_H, V7_J, V7_K, V7_L, V7_M, V7_N, V7_O,
579
580 // non-volatiles
581 V8, V8_H, V8_J, V8_K, V8_L, V8_M, V8_N, V8_O,
582 V9, V9_H, V9_J, V9_K, V9_L, V9_M, V9_N, V9_O,
583 V10, V10_H, V10_J, V10_K, V10_L, V10_M, V10_N, V10_O,
584 V11, V11_H, V11_J, V11_K, V11_L, V11_M, V11_N, V11_O,
585 V12, V12_H, V12_J, V12_K, V12_L, V12_M, V12_N, V12_O,
586 V13, V13_H, V13_J, V13_K, V13_L, V13_M, V13_N, V13_O,
587 V14, V14_H, V14_J, V14_K, V14_L, V14_M, V14_N, V14_O,
588 V15, V15_H, V15_J, V15_K, V15_L, V15_M, V15_N, V15_O,
589 );
590
591 alloc_class chunk2 (
592 P0,
593 P1,
594 P2,
595 P3,
596 P4,
597 P5,
598 P6,
599 P7,
600
601 P8,
602 P9,
603 P10,
604 P11,
605 P12,
606 P13,
607 P14,
608 P15,
609 );
610
611 alloc_class chunk3(RFLAGS);
612
613 //----------Architecture Description Register Classes--------------------------
614 // Several register classes are automatically defined based upon information in
615 // this architecture description.
616 // 1) reg_class inline_cache_reg ( /* as def'd in frame section */ )
617 // 2) reg_class stack_slots( /* one chunk of stack-based "registers" */ )
618 //
619
620 // Class for all 32 bit general purpose registers
621 reg_class all_reg32(
622 R0,
623 R1,
624 R2,
625 R3,
626 R4,
627 R5,
628 R6,
629 R7,
630 R10,
631 R11,
632 R12,
633 R13,
634 R14,
635 R15,
636 R16,
637 R17,
638 R18,
639 R19,
640 R20,
641 R21,
642 R22,
643 R23,
644 R24,
645 R25,
646 R26,
647 R27,
648 R28,
649 R29,
650 R30,
651 R31
652 );
653
654
655 // Class for all 32 bit integer registers (excluding SP which
656 // will never be used as an integer register)
657 reg_class any_reg32 %{
658 return _ANY_REG32_mask;
659 %}
660
661 // Singleton class for R0 int register
662 reg_class int_r0_reg(R0);
663
664 // Singleton class for R2 int register
665 reg_class int_r2_reg(R2);
666
667 // Singleton class for R3 int register
668 reg_class int_r3_reg(R3);
669
670 // Singleton class for R4 int register
671 reg_class int_r4_reg(R4);
672
673 // Singleton class for R31 int register
674 reg_class int_r31_reg(R31);
675
676 // Class for all 64 bit general purpose registers
677 reg_class all_reg(
678 R0, R0_H,
679 R1, R1_H,
680 R2, R2_H,
681 R3, R3_H,
682 R4, R4_H,
683 R5, R5_H,
684 R6, R6_H,
685 R7, R7_H,
686 R10, R10_H,
687 R11, R11_H,
688 R12, R12_H,
689 R13, R13_H,
690 R14, R14_H,
691 R15, R15_H,
692 R16, R16_H,
693 R17, R17_H,
694 R18, R18_H,
695 R19, R19_H,
696 R20, R20_H,
697 R21, R21_H,
698 R22, R22_H,
699 R23, R23_H,
700 R24, R24_H,
701 R25, R25_H,
702 R26, R26_H,
703 R27, R27_H,
704 R28, R28_H,
705 R29, R29_H,
706 R30, R30_H,
707 R31, R31_H
708 );
709
710 // Class for all long integer registers (including SP)
711 reg_class any_reg %{
712 return _ANY_REG_mask;
713 %}
714
715 // Class for non-allocatable 32 bit registers
716 reg_class non_allocatable_reg32(
717 #ifdef R18_RESERVED
718 // See comment in register_aarch64.hpp
719 R18, // tls on Windows
720 #endif
721 R28, // thread
722 R30, // lr
723 R31 // sp
724 );
725
726 // Class for non-allocatable 64 bit registers
727 reg_class non_allocatable_reg(
728 #ifdef R18_RESERVED
729 // See comment in register_aarch64.hpp
730 R18, R18_H, // tls on Windows, platform register on macOS
731 #endif
732 R28, R28_H, // thread
733 R30, R30_H, // lr
734 R31, R31_H // sp
735 );
736
737 // Class for all non-special integer registers
738 reg_class no_special_reg32 %{
739 return _NO_SPECIAL_REG32_mask;
740 %}
741
742 // Class for all non-special long integer registers
743 reg_class no_special_reg %{
744 return _NO_SPECIAL_REG_mask;
745 %}
746
747 // Class for 64 bit register r0
748 reg_class r0_reg(
749 R0, R0_H
750 );
751
752 // Class for 64 bit register r1
753 reg_class r1_reg(
754 R1, R1_H
755 );
756
757 // Class for 64 bit register r2
758 reg_class r2_reg(
759 R2, R2_H
760 );
761
762 // Class for 64 bit register r3
763 reg_class r3_reg(
764 R3, R3_H
765 );
766
767 // Class for 64 bit register r4
768 reg_class r4_reg(
769 R4, R4_H
770 );
771
772 // Class for 64 bit register r5
773 reg_class r5_reg(
774 R5, R5_H
775 );
776
777 // Class for 64 bit register r10
778 reg_class r10_reg(
779 R10, R10_H
780 );
781
782 // Class for 64 bit register r11
783 reg_class r11_reg(
784 R11, R11_H
785 );
786
787 // Class for method register
788 reg_class method_reg(
789 R12, R12_H
790 );
791
792 // Class for heapbase register
793 reg_class heapbase_reg(
794 R27, R27_H
795 );
796
797 // Class for thread register
798 reg_class thread_reg(
799 R28, R28_H
800 );
801
802 // Class for frame pointer register
803 reg_class fp_reg(
804 R29, R29_H
805 );
806
807 // Class for link register
808 reg_class lr_reg(
809 R30, R30_H
810 );
811
812 // Class for long sp register
813 reg_class sp_reg(
814 R31, R31_H
815 );
816
817 // Class for all pointer registers
818 reg_class ptr_reg %{
819 return _PTR_REG_mask;
820 %}
821
822 // Class for all non_special pointer registers
823 reg_class no_special_ptr_reg %{
824 return _NO_SPECIAL_PTR_REG_mask;
825 %}
826
827 // Class for all float registers
828 reg_class float_reg(
829 V0,
830 V1,
831 V2,
832 V3,
833 V4,
834 V5,
835 V6,
836 V7,
837 V8,
838 V9,
839 V10,
840 V11,
841 V12,
842 V13,
843 V14,
844 V15,
845 V16,
846 V17,
847 V18,
848 V19,
849 V20,
850 V21,
851 V22,
852 V23,
853 V24,
854 V25,
855 V26,
856 V27,
857 V28,
858 V29,
859 V30,
860 V31
861 );
862
863 // Double precision float registers have virtual `high halves' that
864 // are needed by the allocator.
865 // Class for all double registers
866 reg_class double_reg(
867 V0, V0_H,
868 V1, V1_H,
869 V2, V2_H,
870 V3, V3_H,
871 V4, V4_H,
872 V5, V5_H,
873 V6, V6_H,
874 V7, V7_H,
875 V8, V8_H,
876 V9, V9_H,
877 V10, V10_H,
878 V11, V11_H,
879 V12, V12_H,
880 V13, V13_H,
881 V14, V14_H,
882 V15, V15_H,
883 V16, V16_H,
884 V17, V17_H,
885 V18, V18_H,
886 V19, V19_H,
887 V20, V20_H,
888 V21, V21_H,
889 V22, V22_H,
890 V23, V23_H,
891 V24, V24_H,
892 V25, V25_H,
893 V26, V26_H,
894 V27, V27_H,
895 V28, V28_H,
896 V29, V29_H,
897 V30, V30_H,
898 V31, V31_H
899 );
900
901 // Class for all SVE vector registers.
902 reg_class vectora_reg (
903 V0, V0_H, V0_J, V0_K, V0_L, V0_M, V0_N, V0_O,
904 V1, V1_H, V1_J, V1_K, V1_L, V1_M, V1_N, V1_O,
905 V2, V2_H, V2_J, V2_K, V2_L, V2_M, V2_N, V2_O,
906 V3, V3_H, V3_J, V3_K, V3_L, V3_M, V3_N, V3_O,
907 V4, V4_H, V4_J, V4_K, V4_L, V4_M, V4_N, V4_O,
908 V5, V5_H, V5_J, V5_K, V5_L, V5_M, V5_N, V5_O,
909 V6, V6_H, V6_J, V6_K, V6_L, V6_M, V6_N, V6_O,
910 V7, V7_H, V7_J, V7_K, V7_L, V7_M, V7_N, V7_O,
911 V8, V8_H, V8_J, V8_K, V8_L, V8_M, V8_N, V8_O,
912 V9, V9_H, V9_J, V9_K, V9_L, V9_M, V9_N, V9_O,
913 V10, V10_H, V10_J, V10_K, V10_L, V10_M, V10_N, V10_O,
914 V11, V11_H, V11_J, V11_K, V11_L, V11_M, V11_N, V11_O,
915 V12, V12_H, V12_J, V12_K, V12_L, V12_M, V12_N, V12_O,
916 V13, V13_H, V13_J, V13_K, V13_L, V13_M, V13_N, V13_O,
917 V14, V14_H, V14_J, V14_K, V14_L, V14_M, V14_N, V14_O,
918 V15, V15_H, V15_J, V15_K, V15_L, V15_M, V15_N, V15_O,
919 V16, V16_H, V16_J, V16_K, V16_L, V16_M, V16_N, V16_O,
920 V17, V17_H, V17_J, V17_K, V17_L, V17_M, V17_N, V17_O,
921 V18, V18_H, V18_J, V18_K, V18_L, V18_M, V18_N, V18_O,
922 V19, V19_H, V19_J, V19_K, V19_L, V19_M, V19_N, V19_O,
923 V20, V20_H, V20_J, V20_K, V20_L, V20_M, V20_N, V20_O,
924 V21, V21_H, V21_J, V21_K, V21_L, V21_M, V21_N, V21_O,
925 V22, V22_H, V22_J, V22_K, V22_L, V22_M, V22_N, V22_O,
926 V23, V23_H, V23_J, V23_K, V23_L, V23_M, V23_N, V23_O,
927 V24, V24_H, V24_J, V24_K, V24_L, V24_M, V24_N, V24_O,
928 V25, V25_H, V25_J, V25_K, V25_L, V25_M, V25_N, V25_O,
929 V26, V26_H, V26_J, V26_K, V26_L, V26_M, V26_N, V26_O,
930 V27, V27_H, V27_J, V27_K, V27_L, V27_M, V27_N, V27_O,
931 V28, V28_H, V28_J, V28_K, V28_L, V28_M, V28_N, V28_O,
932 V29, V29_H, V29_J, V29_K, V29_L, V29_M, V29_N, V29_O,
933 V30, V30_H, V30_J, V30_K, V30_L, V30_M, V30_N, V30_O,
934 V31, V31_H, V31_J, V31_K, V31_L, V31_M, V31_N, V31_O,
935 );
936
937 // Class for all 64bit vector registers
938 reg_class vectord_reg(
939 V0, V0_H,
940 V1, V1_H,
941 V2, V2_H,
942 V3, V3_H,
943 V4, V4_H,
944 V5, V5_H,
945 V6, V6_H,
946 V7, V7_H,
947 V8, V8_H,
948 V9, V9_H,
949 V10, V10_H,
950 V11, V11_H,
951 V12, V12_H,
952 V13, V13_H,
953 V14, V14_H,
954 V15, V15_H,
955 V16, V16_H,
956 V17, V17_H,
957 V18, V18_H,
958 V19, V19_H,
959 V20, V20_H,
960 V21, V21_H,
961 V22, V22_H,
962 V23, V23_H,
963 V24, V24_H,
964 V25, V25_H,
965 V26, V26_H,
966 V27, V27_H,
967 V28, V28_H,
968 V29, V29_H,
969 V30, V30_H,
970 V31, V31_H
971 );
972
973 // Class for all 128bit vector registers
974 reg_class vectorx_reg(
975 V0, V0_H, V0_J, V0_K,
976 V1, V1_H, V1_J, V1_K,
977 V2, V2_H, V2_J, V2_K,
978 V3, V3_H, V3_J, V3_K,
979 V4, V4_H, V4_J, V4_K,
980 V5, V5_H, V5_J, V5_K,
981 V6, V6_H, V6_J, V6_K,
982 V7, V7_H, V7_J, V7_K,
983 V8, V8_H, V8_J, V8_K,
984 V9, V9_H, V9_J, V9_K,
985 V10, V10_H, V10_J, V10_K,
986 V11, V11_H, V11_J, V11_K,
987 V12, V12_H, V12_J, V12_K,
988 V13, V13_H, V13_J, V13_K,
989 V14, V14_H, V14_J, V14_K,
990 V15, V15_H, V15_J, V15_K,
991 V16, V16_H, V16_J, V16_K,
992 V17, V17_H, V17_J, V17_K,
993 V18, V18_H, V18_J, V18_K,
994 V19, V19_H, V19_J, V19_K,
995 V20, V20_H, V20_J, V20_K,
996 V21, V21_H, V21_J, V21_K,
997 V22, V22_H, V22_J, V22_K,
998 V23, V23_H, V23_J, V23_K,
999 V24, V24_H, V24_J, V24_K,
1000 V25, V25_H, V25_J, V25_K,
1001 V26, V26_H, V26_J, V26_K,
1002 V27, V27_H, V27_J, V27_K,
1003 V28, V28_H, V28_J, V28_K,
1004 V29, V29_H, V29_J, V29_K,
1005 V30, V30_H, V30_J, V30_K,
1006 V31, V31_H, V31_J, V31_K
1007 );
1008
1009 // Class for 128 bit register v0
1010 reg_class v0_reg(
1011 V0, V0_H
1012 );
1013
1014 // Class for 128 bit register v1
1015 reg_class v1_reg(
1016 V1, V1_H
1017 );
1018
1019 // Class for 128 bit register v2
1020 reg_class v2_reg(
1021 V2, V2_H
1022 );
1023
1024 // Class for 128 bit register v3
1025 reg_class v3_reg(
1026 V3, V3_H
1027 );
1028
1029 // Class for 128 bit register v4
1030 reg_class v4_reg(
1031 V4, V4_H
1032 );
1033
1034 // Class for 128 bit register v5
1035 reg_class v5_reg(
1036 V5, V5_H
1037 );
1038
1039 // Class for 128 bit register v6
1040 reg_class v6_reg(
1041 V6, V6_H
1042 );
1043
1044 // Class for 128 bit register v7
1045 reg_class v7_reg(
1046 V7, V7_H
1047 );
1048
1049 // Class for 128 bit register v8
1050 reg_class v8_reg(
1051 V8, V8_H
1052 );
1053
1054 // Class for 128 bit register v9
1055 reg_class v9_reg(
1056 V9, V9_H
1057 );
1058
1059 // Class for 128 bit register v10
1060 reg_class v10_reg(
1061 V10, V10_H
1062 );
1063
1064 // Class for 128 bit register v11
1065 reg_class v11_reg(
1066 V11, V11_H
1067 );
1068
1069 // Class for 128 bit register v12
1070 reg_class v12_reg(
1071 V12, V12_H
1072 );
1073
1074 // Class for 128 bit register v13
1075 reg_class v13_reg(
1076 V13, V13_H
1077 );
1078
1079 // Class for 128 bit register v14
1080 reg_class v14_reg(
1081 V14, V14_H
1082 );
1083
1084 // Class for 128 bit register v15
1085 reg_class v15_reg(
1086 V15, V15_H
1087 );
1088
1089 // Class for 128 bit register v16
1090 reg_class v16_reg(
1091 V16, V16_H
1092 );
1093
1094 // Class for 128 bit register v17
1095 reg_class v17_reg(
1096 V17, V17_H
1097 );
1098
1099 // Class for 128 bit register v18
1100 reg_class v18_reg(
1101 V18, V18_H
1102 );
1103
1104 // Class for 128 bit register v19
1105 reg_class v19_reg(
1106 V19, V19_H
1107 );
1108
1109 // Class for 128 bit register v20
1110 reg_class v20_reg(
1111 V20, V20_H
1112 );
1113
1114 // Class for 128 bit register v21
1115 reg_class v21_reg(
1116 V21, V21_H
1117 );
1118
1119 // Class for 128 bit register v22
1120 reg_class v22_reg(
1121 V22, V22_H
1122 );
1123
1124 // Class for 128 bit register v23
1125 reg_class v23_reg(
1126 V23, V23_H
1127 );
1128
1129 // Class for 128 bit register v24
1130 reg_class v24_reg(
1131 V24, V24_H
1132 );
1133
1134 // Class for 128 bit register v25
1135 reg_class v25_reg(
1136 V25, V25_H
1137 );
1138
1139 // Class for 128 bit register v26
1140 reg_class v26_reg(
1141 V26, V26_H
1142 );
1143
1144 // Class for 128 bit register v27
1145 reg_class v27_reg(
1146 V27, V27_H
1147 );
1148
1149 // Class for 128 bit register v28
1150 reg_class v28_reg(
1151 V28, V28_H
1152 );
1153
1154 // Class for 128 bit register v29
1155 reg_class v29_reg(
1156 V29, V29_H
1157 );
1158
1159 // Class for 128 bit register v30
1160 reg_class v30_reg(
1161 V30, V30_H
1162 );
1163
1164 // Class for 128 bit register v31
1165 reg_class v31_reg(
1166 V31, V31_H
1167 );
1168
1169 // Class for all SVE predicate registers.
1170 reg_class pr_reg (
1171 P0,
1172 P1,
1173 P2,
1174 P3,
1175 P4,
1176 P5,
1177 P6,
1178 // P7, non-allocatable, preserved with all elements preset to TRUE.
1179 P8,
1180 P9,
1181 P10,
1182 P11,
1183 P12,
1184 P13,
1185 P14,
1186 P15
1187 );
1188
1189 // Class for SVE governing predicate registers, which are used
1190 // to determine the active elements of a predicated instruction.
1191 reg_class gov_pr (
1192 P0,
1193 P1,
1194 P2,
1195 P3,
1196 P4,
1197 P5,
1198 P6,
1199 // P7, non-allocatable, preserved with all elements preset to TRUE.
1200 );
1201
1202 // Singleton class for condition codes
1203 reg_class int_flags(RFLAGS);
1204
1205 %}
1206
1207 //----------DEFINITION BLOCK---------------------------------------------------
1208 // Define name --> value mappings to inform the ADLC of an integer valued name
1209 // Current support includes integer values in the range [0, 0x7FFFFFFF]
1210 // Format:
1211 // int_def <name> ( <int_value>, <expression>);
1212 // Generated Code in ad_<arch>.hpp
1213 // #define <name> (<expression>)
1214 // // value == <int_value>
1215 // Generated code in ad_<arch>.cpp adlc_verification()
1216 // assert( <name> == <int_value>, "Expect (<expression>) to equal <int_value>");
1217 //
1218
1219 // we follow the ppc-aix port in using a simple cost model which ranks
1220 // register operations as cheap, memory ops as more expensive and
1221 // branches as most expensive. the first two have a low as well as a
1222 // normal cost. huge cost appears to be a way of saying don't do
1223 // something
1224
1225 definitions %{
1226 // The default cost (of a register move instruction).
1227 int_def INSN_COST ( 100, 100);
1228 int_def BRANCH_COST ( 200, 2 * INSN_COST);
1229 int_def CALL_COST ( 200, 2 * INSN_COST);
1230 int_def VOLATILE_REF_COST ( 1000, 10 * INSN_COST);
1231 %}
1232
1233
1234 //----------SOURCE BLOCK-------------------------------------------------------
1235 // This is a block of C++ code which provides values, functions, and
1236 // definitions necessary in the rest of the architecture description
1237
1238 source_hpp %{
1239
1240 #include "asm/macroAssembler.hpp"
1241 #include "gc/shared/barrierSetAssembler.hpp"
1242 #include "gc/shared/cardTable.hpp"
1243 #include "gc/shared/cardTableBarrierSet.hpp"
1244 #include "gc/shared/collectedHeap.hpp"
1245 #include "opto/addnode.hpp"
1246 #include "opto/convertnode.hpp"
1247 #include "runtime/objectMonitor.hpp"
1248
1249 extern RegMask _ANY_REG32_mask;
1250 extern RegMask _ANY_REG_mask;
1251 extern RegMask _PTR_REG_mask;
1252 extern RegMask _NO_SPECIAL_REG32_mask;
1253 extern RegMask _NO_SPECIAL_REG_mask;
1254 extern RegMask _NO_SPECIAL_PTR_REG_mask;
1255
1256 class CallStubImpl {
1257
1258 //--------------------------------------------------------------
1259 //---< Used for optimization in Compile::shorten_branches >---
1260 //--------------------------------------------------------------
1261
1262 public:
1263 // Size of call trampoline stub.
1264 static uint size_call_trampoline() {
1265 return 0; // no call trampolines on this platform
1266 }
1267
1268 // number of relocations needed by a call trampoline stub
1269 static uint reloc_call_trampoline() {
1270 return 0; // no call trampolines on this platform
1271 }
1272 };
1273
1274 class HandlerImpl {
1275
1276 public:
1277
1278 static int emit_exception_handler(CodeBuffer &cbuf);
1279 static int emit_deopt_handler(CodeBuffer& cbuf);
1280
1281 static uint size_exception_handler() {
1282 return MacroAssembler::far_branch_size();
1283 }
1284
1285 static uint size_deopt_handler() {
1286 // count one adr and one far branch instruction
1287 return 4 * NativeInstruction::instruction_size;
1288 }
1289 };
1290
1291 class Node::PD {
1292 public:
1293 enum NodeFlags {
1294 _last_flag = Node::_last_flag
1295 };
1296 };
1297
1298 bool is_CAS(int opcode, bool maybe_volatile);
1299
1300 // predicates controlling emit of ldr<x>/ldar<x> and associated dmb
1301
1302 bool unnecessary_acquire(const Node *barrier);
1303 bool needs_acquiring_load(const Node *load);
1304
1305 // predicates controlling emit of str<x>/stlr<x> and associated dmbs
1306
1307 bool unnecessary_release(const Node *barrier);
1308 bool unnecessary_volatile(const Node *barrier);
1309 bool needs_releasing_store(const Node *store);
1310
1311 // predicate controlling translation of CompareAndSwapX
1312 bool needs_acquiring_load_exclusive(const Node *load);
1313
1314 // Assert that the given node is not a variable shift.
1315 bool assert_not_var_shift(const Node* n);
1316
1317 // predicate controlling addressing modes
1318 bool size_fits_all_mem_uses(AddPNode* addp, int shift);
1319 %}
1320
1321 source %{
1322
1323 // Derived RegMask with conditionally allocatable registers
1324
1325 void PhaseOutput::pd_perform_mach_node_analysis() {
1326 }
1327
1328 int MachNode::pd_alignment_required() const {
1329 return 1;
1330 }
1331
1332 int MachNode::compute_padding(int current_offset) const {
1333 return 0;
1334 }
1335
1336 RegMask _ANY_REG32_mask;
1337 RegMask _ANY_REG_mask;
1338 RegMask _PTR_REG_mask;
1339 RegMask _NO_SPECIAL_REG32_mask;
1340 RegMask _NO_SPECIAL_REG_mask;
1341 RegMask _NO_SPECIAL_PTR_REG_mask;
1342
1343 void reg_mask_init() {
1344 // We derive below RegMask(s) from the ones which are auto-generated from
1345 // adlc register classes to make AArch64 rheapbase (r27) and rfp (r29)
1346 // registers conditionally reserved.
1347
1348 _ANY_REG32_mask = _ALL_REG32_mask;
1349 _ANY_REG32_mask.Remove(OptoReg::as_OptoReg(r31_sp->as_VMReg()));
1350
1351 _ANY_REG_mask = _ALL_REG_mask;
1352
1353 _PTR_REG_mask = _ALL_REG_mask;
1354
1355 _NO_SPECIAL_REG32_mask = _ALL_REG32_mask;
1356 _NO_SPECIAL_REG32_mask.SUBTRACT(_NON_ALLOCATABLE_REG32_mask);
1357
1358 _NO_SPECIAL_REG_mask = _ALL_REG_mask;
1359 _NO_SPECIAL_REG_mask.SUBTRACT(_NON_ALLOCATABLE_REG_mask);
1360
1361 _NO_SPECIAL_PTR_REG_mask = _ALL_REG_mask;
1362 _NO_SPECIAL_PTR_REG_mask.SUBTRACT(_NON_ALLOCATABLE_REG_mask);
1363
1364 // r27 is not allocatable when compressed oops is on and heapbase is not
1365 // zero, compressed klass pointers doesn't use r27 after JDK-8234794
1366 if (UseCompressedOops && (CompressedOops::ptrs_base() != NULL)) {
1367 _NO_SPECIAL_REG32_mask.Remove(OptoReg::as_OptoReg(r27->as_VMReg()));
1368 _NO_SPECIAL_REG_mask.SUBTRACT(_HEAPBASE_REG_mask);
1369 _NO_SPECIAL_PTR_REG_mask.SUBTRACT(_HEAPBASE_REG_mask);
1370 }
1371
1372 // r29 is not allocatable when PreserveFramePointer is on
1373 if (PreserveFramePointer) {
1374 _NO_SPECIAL_REG32_mask.Remove(OptoReg::as_OptoReg(r29->as_VMReg()));
1375 _NO_SPECIAL_REG_mask.SUBTRACT(_FP_REG_mask);
1376 _NO_SPECIAL_PTR_REG_mask.SUBTRACT(_FP_REG_mask);
1377 }
1378 }
1379
1380 // Optimizaton of volatile gets and puts
1381 // -------------------------------------
1382 //
1383 // AArch64 has ldar<x> and stlr<x> instructions which we can safely
1384 // use to implement volatile reads and writes. For a volatile read
1385 // we simply need
1386 //
1387 // ldar<x>
1388 //
1389 // and for a volatile write we need
1390 //
1391 // stlr<x>
1392 //
1393 // Alternatively, we can implement them by pairing a normal
1394 // load/store with a memory barrier. For a volatile read we need
1395 //
1396 // ldr<x>
1397 // dmb ishld
1398 //
1399 // for a volatile write
1400 //
1401 // dmb ish
1402 // str<x>
1403 // dmb ish
1404 //
1405 // We can also use ldaxr and stlxr to implement compare and swap CAS
1406 // sequences. These are normally translated to an instruction
1407 // sequence like the following
1408 //
1409 // dmb ish
1410 // retry:
1411 // ldxr<x> rval raddr
1412 // cmp rval rold
1413 // b.ne done
1414 // stlxr<x> rval, rnew, rold
1415 // cbnz rval retry
1416 // done:
1417 // cset r0, eq
1418 // dmb ishld
1419 //
1420 // Note that the exclusive store is already using an stlxr
1421 // instruction. That is required to ensure visibility to other
1422 // threads of the exclusive write (assuming it succeeds) before that
1423 // of any subsequent writes.
1424 //
1425 // The following instruction sequence is an improvement on the above
1426 //
1427 // retry:
1428 // ldaxr<x> rval raddr
1429 // cmp rval rold
1430 // b.ne done
1431 // stlxr<x> rval, rnew, rold
1432 // cbnz rval retry
1433 // done:
1434 // cset r0, eq
1435 //
1436 // We don't need the leading dmb ish since the stlxr guarantees
1437 // visibility of prior writes in the case that the swap is
1438 // successful. Crucially we don't have to worry about the case where
1439 // the swap is not successful since no valid program should be
1440 // relying on visibility of prior changes by the attempting thread
1441 // in the case where the CAS fails.
1442 //
1443 // Similarly, we don't need the trailing dmb ishld if we substitute
1444 // an ldaxr instruction since that will provide all the guarantees we
1445 // require regarding observation of changes made by other threads
1446 // before any change to the CAS address observed by the load.
1447 //
1448 // In order to generate the desired instruction sequence we need to
1449 // be able to identify specific 'signature' ideal graph node
1450 // sequences which i) occur as a translation of a volatile reads or
1451 // writes or CAS operations and ii) do not occur through any other
1452 // translation or graph transformation. We can then provide
1453 // alternative aldc matching rules which translate these node
1454 // sequences to the desired machine code sequences. Selection of the
1455 // alternative rules can be implemented by predicates which identify
1456 // the relevant node sequences.
1457 //
1458 // The ideal graph generator translates a volatile read to the node
1459 // sequence
1460 //
1461 // LoadX[mo_acquire]
1462 // MemBarAcquire
1463 //
1464 // As a special case when using the compressed oops optimization we
1465 // may also see this variant
1466 //
1467 // LoadN[mo_acquire]
1468 // DecodeN
1469 // MemBarAcquire
1470 //
1471 // A volatile write is translated to the node sequence
1472 //
1473 // MemBarRelease
1474 // StoreX[mo_release] {CardMark}-optional
1475 // MemBarVolatile
1476 //
1477 // n.b. the above node patterns are generated with a strict
1478 // 'signature' configuration of input and output dependencies (see
1479 // the predicates below for exact details). The card mark may be as
1480 // simple as a few extra nodes or, in a few GC configurations, may
1481 // include more complex control flow between the leading and
1482 // trailing memory barriers. However, whatever the card mark
1483 // configuration these signatures are unique to translated volatile
1484 // reads/stores -- they will not appear as a result of any other
1485 // bytecode translation or inlining nor as a consequence of
1486 // optimizing transforms.
1487 //
1488 // We also want to catch inlined unsafe volatile gets and puts and
1489 // be able to implement them using either ldar<x>/stlr<x> or some
1490 // combination of ldr<x>/stlr<x> and dmb instructions.
1491 //
1492 // Inlined unsafe volatiles puts manifest as a minor variant of the
1493 // normal volatile put node sequence containing an extra cpuorder
1494 // membar
1495 //
1496 // MemBarRelease
1497 // MemBarCPUOrder
1498 // StoreX[mo_release] {CardMark}-optional
1499 // MemBarCPUOrder
1500 // MemBarVolatile
1501 //
1502 // n.b. as an aside, a cpuorder membar is not itself subject to
1503 // matching and translation by adlc rules. However, the rule
1504 // predicates need to detect its presence in order to correctly
1505 // select the desired adlc rules.
1506 //
1507 // Inlined unsafe volatile gets manifest as a slightly different
1508 // node sequence to a normal volatile get because of the
1509 // introduction of some CPUOrder memory barriers to bracket the
1510 // Load. However, but the same basic skeleton of a LoadX feeding a
1511 // MemBarAcquire, possibly thorugh an optional DecodeN, is still
1512 // present
1513 //
1514 // MemBarCPUOrder
1515 // || \\
1516 // MemBarCPUOrder LoadX[mo_acquire]
1517 // || |
1518 // || {DecodeN} optional
1519 // || /
1520 // MemBarAcquire
1521 //
1522 // In this case the acquire membar does not directly depend on the
1523 // load. However, we can be sure that the load is generated from an
1524 // inlined unsafe volatile get if we see it dependent on this unique
1525 // sequence of membar nodes. Similarly, given an acquire membar we
1526 // can know that it was added because of an inlined unsafe volatile
1527 // get if it is fed and feeds a cpuorder membar and if its feed
1528 // membar also feeds an acquiring load.
1529 //
1530 // Finally an inlined (Unsafe) CAS operation is translated to the
1531 // following ideal graph
1532 //
1533 // MemBarRelease
1534 // MemBarCPUOrder
1535 // CompareAndSwapX {CardMark}-optional
1536 // MemBarCPUOrder
1537 // MemBarAcquire
1538 //
1539 // So, where we can identify these volatile read and write
1540 // signatures we can choose to plant either of the above two code
1541 // sequences. For a volatile read we can simply plant a normal
1542 // ldr<x> and translate the MemBarAcquire to a dmb. However, we can
1543 // also choose to inhibit translation of the MemBarAcquire and
1544 // inhibit planting of the ldr<x>, instead planting an ldar<x>.
1545 //
1546 // When we recognise a volatile store signature we can choose to
1547 // plant at a dmb ish as a translation for the MemBarRelease, a
1548 // normal str<x> and then a dmb ish for the MemBarVolatile.
1549 // Alternatively, we can inhibit translation of the MemBarRelease
1550 // and MemBarVolatile and instead plant a simple stlr<x>
1551 // instruction.
1552 //
1553 // when we recognise a CAS signature we can choose to plant a dmb
1554 // ish as a translation for the MemBarRelease, the conventional
1555 // macro-instruction sequence for the CompareAndSwap node (which
1556 // uses ldxr<x>) and then a dmb ishld for the MemBarAcquire.
1557 // Alternatively, we can elide generation of the dmb instructions
1558 // and plant the alternative CompareAndSwap macro-instruction
1559 // sequence (which uses ldaxr<x>).
1560 //
1561 // Of course, the above only applies when we see these signature
1562 // configurations. We still want to plant dmb instructions in any
1563 // other cases where we may see a MemBarAcquire, MemBarRelease or
1564 // MemBarVolatile. For example, at the end of a constructor which
1565 // writes final/volatile fields we will see a MemBarRelease
1566 // instruction and this needs a 'dmb ish' lest we risk the
1567 // constructed object being visible without making the
1568 // final/volatile field writes visible.
1569 //
1570 // n.b. the translation rules below which rely on detection of the
1571 // volatile signatures and insert ldar<x> or stlr<x> are failsafe.
1572 // If we see anything other than the signature configurations we
1573 // always just translate the loads and stores to ldr<x> and str<x>
1574 // and translate acquire, release and volatile membars to the
1575 // relevant dmb instructions.
1576 //
1577
1578 // is_CAS(int opcode, bool maybe_volatile)
1579 //
1580 // return true if opcode is one of the possible CompareAndSwapX
1581 // values otherwise false.
1582
1583 bool is_CAS(int opcode, bool maybe_volatile)
1584 {
1585 switch(opcode) {
1586 // We handle these
1587 case Op_CompareAndSwapI:
1588 case Op_CompareAndSwapL:
1589 case Op_CompareAndSwapP:
1590 case Op_CompareAndSwapN:
1591 case Op_ShenandoahCompareAndSwapP:
1592 case Op_ShenandoahCompareAndSwapN:
1593 case Op_CompareAndSwapB:
1594 case Op_CompareAndSwapS:
1595 case Op_GetAndSetI:
1596 case Op_GetAndSetL:
1597 case Op_GetAndSetP:
1598 case Op_GetAndSetN:
1599 case Op_GetAndAddI:
1600 case Op_GetAndAddL:
1601 return true;
1602 case Op_CompareAndExchangeI:
1603 case Op_CompareAndExchangeN:
1604 case Op_CompareAndExchangeB:
1605 case Op_CompareAndExchangeS:
1606 case Op_CompareAndExchangeL:
1607 case Op_CompareAndExchangeP:
1608 case Op_WeakCompareAndSwapB:
1609 case Op_WeakCompareAndSwapS:
1610 case Op_WeakCompareAndSwapI:
1611 case Op_WeakCompareAndSwapL:
1612 case Op_WeakCompareAndSwapP:
1613 case Op_WeakCompareAndSwapN:
1614 case Op_ShenandoahWeakCompareAndSwapP:
1615 case Op_ShenandoahWeakCompareAndSwapN:
1616 case Op_ShenandoahCompareAndExchangeP:
1617 case Op_ShenandoahCompareAndExchangeN:
1618 return maybe_volatile;
1619 default:
1620 return false;
1621 }
1622 }
1623
1624 // helper to determine the maximum number of Phi nodes we may need to
1625 // traverse when searching from a card mark membar for the merge mem
1626 // feeding a trailing membar or vice versa
1627
1628 // predicates controlling emit of ldr<x>/ldar<x>
1629
1630 bool unnecessary_acquire(const Node *barrier)
1631 {
1632 assert(barrier->is_MemBar(), "expecting a membar");
1633
1634 MemBarNode* mb = barrier->as_MemBar();
1635
1636 if (mb->trailing_load()) {
1637 return true;
1638 }
1639
1640 if (mb->trailing_load_store()) {
1641 Node* load_store = mb->in(MemBarNode::Precedent);
1642 assert(load_store->is_LoadStore(), "unexpected graph shape");
1643 return is_CAS(load_store->Opcode(), true);
1644 }
1645
1646 return false;
1647 }
1648
1649 bool needs_acquiring_load(const Node *n)
1650 {
1651 assert(n->is_Load(), "expecting a load");
1652 LoadNode *ld = n->as_Load();
1653 return ld->is_acquire();
1654 }
1655
1656 bool unnecessary_release(const Node *n)
1657 {
1658 assert((n->is_MemBar() &&
1659 n->Opcode() == Op_MemBarRelease),
1660 "expecting a release membar");
1661
1662 MemBarNode *barrier = n->as_MemBar();
1663 if (!barrier->leading()) {
1664 return false;
1665 } else {
1666 Node* trailing = barrier->trailing_membar();
1667 MemBarNode* trailing_mb = trailing->as_MemBar();
1668 assert(trailing_mb->trailing(), "Not a trailing membar?");
1669 assert(trailing_mb->leading_membar() == n, "inconsistent leading/trailing membars");
1670
1671 Node* mem = trailing_mb->in(MemBarNode::Precedent);
1672 if (mem->is_Store()) {
1673 assert(mem->as_Store()->is_release(), "");
1674 assert(trailing_mb->Opcode() == Op_MemBarVolatile, "");
1675 return true;
1676 } else {
1677 assert(mem->is_LoadStore(), "");
1678 assert(trailing_mb->Opcode() == Op_MemBarAcquire, "");
1679 return is_CAS(mem->Opcode(), true);
1680 }
1681 }
1682 return false;
1683 }
1684
1685 bool unnecessary_volatile(const Node *n)
1686 {
1687 // assert n->is_MemBar();
1688 MemBarNode *mbvol = n->as_MemBar();
1689
1690 bool release = mbvol->trailing_store();
1691 assert(!release || (mbvol->in(MemBarNode::Precedent)->is_Store() && mbvol->in(MemBarNode::Precedent)->as_Store()->is_release()), "");
1692 #ifdef ASSERT
1693 if (release) {
1694 Node* leading = mbvol->leading_membar();
1695 assert(leading->Opcode() == Op_MemBarRelease, "");
1696 assert(leading->as_MemBar()->leading_store(), "");
1697 assert(leading->as_MemBar()->trailing_membar() == mbvol, "");
1698 }
1699 #endif
1700
1701 return release;
1702 }
1703
1704 // predicates controlling emit of str<x>/stlr<x>
1705
1706 bool needs_releasing_store(const Node *n)
1707 {
1708 // assert n->is_Store();
1709 StoreNode *st = n->as_Store();
1710 return st->trailing_membar() != NULL;
1711 }
1712
1713 // predicate controlling translation of CAS
1714 //
1715 // returns true if CAS needs to use an acquiring load otherwise false
1716
1717 bool needs_acquiring_load_exclusive(const Node *n)
1718 {
1719 assert(is_CAS(n->Opcode(), true), "expecting a compare and swap");
1720 LoadStoreNode* ldst = n->as_LoadStore();
1721 if (is_CAS(n->Opcode(), false)) {
1722 assert(ldst->trailing_membar() != NULL, "expected trailing membar");
1723 } else {
1724 return ldst->trailing_membar() != NULL;
1725 }
1726
1727 // so we can just return true here
1728 return true;
1729 }
1730
1731 // Assert that the given node is not a variable shift.
1732 bool assert_not_var_shift(const Node* n) {
1733 assert(!n->as_ShiftV()->is_var_shift(), "illegal variable shift");
1734 return true;
1735 }
1736
1737 #define __ _masm.
1738
1739 // advance declarations for helper functions to convert register
1740 // indices to register objects
1741
1742 // the ad file has to provide implementations of certain methods
1743 // expected by the generic code
1744 //
1745 // REQUIRED FUNCTIONALITY
1746
1747 //=============================================================================
1748
1749 // !!!!! Special hack to get all types of calls to specify the byte offset
1750 // from the start of the call to the point where the return address
1751 // will point.
1752
1753 int MachCallStaticJavaNode::ret_addr_offset()
1754 {
1755 // call should be a simple bl
1756 int off = 4;
1757 return off;
1758 }
1759
1760 int MachCallDynamicJavaNode::ret_addr_offset()
1761 {
1762 return 16; // movz, movk, movk, bl
1763 }
1764
1765 int MachCallRuntimeNode::ret_addr_offset() {
1766 // for generated stubs the call will be
1767 // bl(addr)
1768 // or with far branches
1769 // bl(trampoline_stub)
1770 // for real runtime callouts it will be six instructions
1771 // see aarch64_enc_java_to_runtime
1772 // adr(rscratch2, retaddr)
1773 // lea(rscratch1, RuntimeAddress(addr)
1774 // stp(zr, rscratch2, Address(__ pre(sp, -2 * wordSize)))
1775 // blr(rscratch1)
1776 CodeBlob *cb = CodeCache::find_blob(_entry_point);
1777 if (cb) {
1778 return 1 * NativeInstruction::instruction_size;
1779 } else if (_entry_point == NULL) {
1780 // See CallLeafNoFPIndirect
1781 return 1 * NativeInstruction::instruction_size;
1782 } else {
1783 return 6 * NativeInstruction::instruction_size;
1784 }
1785 }
1786
1787 int MachCallNativeNode::ret_addr_offset() {
1788 // This is implemented using aarch64_enc_java_to_runtime as above.
1789 CodeBlob *cb = CodeCache::find_blob(_entry_point);
1790 if (cb) {
1791 return 1 * NativeInstruction::instruction_size;
1792 } else {
1793 return 6 * NativeInstruction::instruction_size;
1794 }
1795 }
1796
1797 //=============================================================================
1798
1799 #ifndef PRODUCT
1800 void MachBreakpointNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
1801 st->print("BREAKPOINT");
1802 }
1803 #endif
1804
1805 void MachBreakpointNode::emit(CodeBuffer &cbuf, PhaseRegAlloc *ra_) const {
1806 C2_MacroAssembler _masm(&cbuf);
1807 __ brk(0);
1808 }
1809
1810 uint MachBreakpointNode::size(PhaseRegAlloc *ra_) const {
1811 return MachNode::size(ra_);
1812 }
1813
1814 //=============================================================================
1815
1816 #ifndef PRODUCT
1817 void MachNopNode::format(PhaseRegAlloc*, outputStream* st) const {
1818 st->print("nop \t# %d bytes pad for loops and calls", _count);
1819 }
1820 #endif
1821
1822 void MachNopNode::emit(CodeBuffer &cbuf, PhaseRegAlloc*) const {
1823 C2_MacroAssembler _masm(&cbuf);
1824 for (int i = 0; i < _count; i++) {
1825 __ nop();
1826 }
1827 }
1828
1829 uint MachNopNode::size(PhaseRegAlloc*) const {
1830 return _count * NativeInstruction::instruction_size;
1831 }
1832
1833 //=============================================================================
1834 const RegMask& MachConstantBaseNode::_out_RegMask = RegMask::Empty;
1835
1836 int ConstantTable::calculate_table_base_offset() const {
1837 return 0; // absolute addressing, no offset
1838 }
1839
1840 bool MachConstantBaseNode::requires_postalloc_expand() const { return false; }
1841 void MachConstantBaseNode::postalloc_expand(GrowableArray <Node *> *nodes, PhaseRegAlloc *ra_) {
1842 ShouldNotReachHere();
1843 }
1844
1845 void MachConstantBaseNode::emit(CodeBuffer& cbuf, PhaseRegAlloc* ra_) const {
1846 // Empty encoding
1847 }
1848
1849 uint MachConstantBaseNode::size(PhaseRegAlloc* ra_) const {
1850 return 0;
1851 }
1852
1853 #ifndef PRODUCT
1854 void MachConstantBaseNode::format(PhaseRegAlloc* ra_, outputStream* st) const {
1855 st->print("-- \t// MachConstantBaseNode (empty encoding)");
1856 }
1857 #endif
1858
1859 #ifndef PRODUCT
1860 void MachPrologNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
1861 Compile* C = ra_->C;
1862
1863 int framesize = C->output()->frame_slots() << LogBytesPerInt;
1864
1865 if (C->output()->need_stack_bang(framesize))
1866 st->print("# stack bang size=%d\n\t", framesize);
1867
1868 if (VM_Version::use_rop_protection()) {
1869 st->print("ldr zr, [lr]\n\t");
1870 st->print("pacia lr, rfp\n\t");
1871 }
1872 if (framesize < ((1 << 9) + 2 * wordSize)) {
1873 st->print("sub sp, sp, #%d\n\t", framesize);
1874 st->print("stp rfp, lr, [sp, #%d]", framesize - 2 * wordSize);
1875 if (PreserveFramePointer) st->print("\n\tadd rfp, sp, #%d", framesize - 2 * wordSize);
1876 } else {
1877 st->print("stp lr, rfp, [sp, #%d]!\n\t", -(2 * wordSize));
1878 if (PreserveFramePointer) st->print("mov rfp, sp\n\t");
1879 st->print("mov rscratch1, #%d\n\t", framesize - 2 * wordSize);
1880 st->print("sub sp, sp, rscratch1");
1881 }
1882 if (C->stub_function() == NULL && BarrierSet::barrier_set()->barrier_set_nmethod() != NULL) {
1883 st->print("\n\t");
1884 st->print("ldr rscratch1, [guard]\n\t");
1885 st->print("dmb ishld\n\t");
1886 st->print("ldr rscratch2, [rthread, #thread_disarmed_offset]\n\t");
1887 st->print("cmp rscratch1, rscratch2\n\t");
1888 st->print("b.eq skip");
1889 st->print("\n\t");
1890 st->print("blr #nmethod_entry_barrier_stub\n\t");
1891 st->print("b skip\n\t");
1892 st->print("guard: int\n\t");
1893 st->print("\n\t");
1894 st->print("skip:\n\t");
1895 }
1896 }
1897 #endif
1898
1899 void MachPrologNode::emit(CodeBuffer &cbuf, PhaseRegAlloc *ra_) const {
1900 Compile* C = ra_->C;
1901 C2_MacroAssembler _masm(&cbuf);
1902
1903 // insert a nop at the start of the prolog so we can patch in a
1904 // branch if we need to invalidate the method later
1905 __ nop();
1906
1907 if (C->clinit_barrier_on_entry()) {
1908 assert(!C->method()->holder()->is_not_initialized(), "initialization should have been started");
1909
1910 Label L_skip_barrier;
1911
1912 __ mov_metadata(rscratch2, C->method()->holder()->constant_encoding());
1913 __ clinit_barrier(rscratch2, rscratch1, &L_skip_barrier);
1914 __ far_jump(RuntimeAddress(SharedRuntime::get_handle_wrong_method_stub()));
1915 __ bind(L_skip_barrier);
1916 }
1917
1918 if (C->max_vector_size() > 0) {
1919 __ reinitialize_ptrue();
1920 }
1921
1922 __ verified_entry(C, 0);
1923 __ bind(*_verified_entry);
1924
1925 if (C->stub_function() == NULL) {
1926 BarrierSetAssembler* bs = BarrierSet::barrier_set()->barrier_set_assembler();
1927 bs->nmethod_entry_barrier(&_masm);
1928 }
1929
1930 if (VerifyStackAtCalls) {
1931 Unimplemented();
1932 }
1933
1934 C->output()->set_frame_complete(cbuf.insts_size());
1935
1936 if (C->has_mach_constant_base_node()) {
1937 // NOTE: We set the table base offset here because users might be
1938 // emitted before MachConstantBaseNode.
1939 ConstantTable& constant_table = C->output()->constant_table();
1940 constant_table.set_table_base_offset(constant_table.calculate_table_base_offset());
1941 }
1942 }
1943
1944 int MachPrologNode::reloc() const
1945 {
1946 return 0;
1947 }
1948
1949 //=============================================================================
1950
1951 #ifndef PRODUCT
1952 void MachEpilogNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
1953 Compile* C = ra_->C;
1954 int framesize = C->output()->frame_slots() << LogBytesPerInt;
1955
1956 st->print("# pop frame %d\n\t",framesize);
1957
1958 if (framesize == 0) {
1959 st->print("ldp lr, rfp, [sp],#%d\n\t", (2 * wordSize));
1960 } else if (framesize < ((1 << 9) + 2 * wordSize)) {
1961 st->print("ldp lr, rfp, [sp,#%d]\n\t", framesize - 2 * wordSize);
1962 st->print("add sp, sp, #%d\n\t", framesize);
1963 } else {
1964 st->print("mov rscratch1, #%d\n\t", framesize - 2 * wordSize);
1965 st->print("add sp, sp, rscratch1\n\t");
1966 st->print("ldp lr, rfp, [sp],#%d\n\t", (2 * wordSize));
1967 }
1968 if (VM_Version::use_rop_protection()) {
1969 st->print("autia lr, rfp\n\t");
1970 st->print("ldr zr, [lr]\n\t");
1971 }
1972
1973 if (do_polling() && C->is_method_compilation()) {
1974 st->print("# test polling word\n\t");
1975 st->print("ldr rscratch1, [rthread],#%d\n\t", in_bytes(JavaThread::polling_word_offset()));
1976 st->print("cmp sp, rscratch1\n\t");
1977 st->print("bhi #slow_path");
1978 }
1979 }
1980 #endif
1981
1982 void MachEpilogNode::emit(CodeBuffer &cbuf, PhaseRegAlloc *ra_) const {
1983 Compile* C = ra_->C;
1984 C2_MacroAssembler _masm(&cbuf);
1985 int framesize = C->output()->frame_slots() << LogBytesPerInt;
1986
1987 __ remove_frame(framesize, C->needs_stack_repair());
1988
1989 if (StackReservedPages > 0 && C->has_reserved_stack_access()) {
1990 __ reserved_stack_check();
1991 }
1992
1993 if (do_polling() && C->is_method_compilation()) {
1994 Label dummy_label;
1995 Label* code_stub = &dummy_label;
1996 if (!C->output()->in_scratch_emit_size()) {
1997 code_stub = &C->output()->safepoint_poll_table()->add_safepoint(__ offset());
1998 }
1999 __ relocate(relocInfo::poll_return_type);
2000 __ safepoint_poll(*code_stub, true /* at_return */, false /* acquire */, true /* in_nmethod */);
2001 }
2002 }
2003
2004 int MachEpilogNode::reloc() const {
2005 // Return number of relocatable values contained in this instruction.
2006 return 1; // 1 for polling page.
2007 }
2008
2009 const Pipeline * MachEpilogNode::pipeline() const {
2010 return MachNode::pipeline_class();
2011 }
2012
2013 //=============================================================================
2014
2015 // Figure out which register class each belongs in: rc_int, rc_float or
2016 // rc_stack.
2017 enum RC { rc_bad, rc_int, rc_float, rc_predicate, rc_stack };
2018
2019 static enum RC rc_class(OptoReg::Name reg) {
2020
2021 if (reg == OptoReg::Bad) {
2022 return rc_bad;
2023 }
2024
2025 // we have 32 int registers * 2 halves
2026 int slots_of_int_registers = RegisterImpl::max_slots_per_register * RegisterImpl::number_of_registers;
2027
2028 if (reg < slots_of_int_registers) {
2029 return rc_int;
2030 }
2031
2032 // we have 32 float register * 8 halves
2033 int slots_of_float_registers = FloatRegisterImpl::max_slots_per_register * FloatRegisterImpl::number_of_registers;
2034 if (reg < slots_of_int_registers + slots_of_float_registers) {
2035 return rc_float;
2036 }
2037
2038 int slots_of_predicate_registers = PRegisterImpl::max_slots_per_register * PRegisterImpl::number_of_registers;
2039 if (reg < slots_of_int_registers + slots_of_float_registers + slots_of_predicate_registers) {
2040 return rc_predicate;
2041 }
2042
2043 // Between predicate regs & stack is the flags.
2044 assert(OptoReg::is_stack(reg), "blow up if spilling flags");
2045
2046 return rc_stack;
2047 }
2048
2049 uint MachSpillCopyNode::implementation(CodeBuffer *cbuf, PhaseRegAlloc *ra_, bool do_size, outputStream *st) const {
2050 Compile* C = ra_->C;
2051
2052 // Get registers to move.
2053 OptoReg::Name src_hi = ra_->get_reg_second(in(1));
2054 OptoReg::Name src_lo = ra_->get_reg_first(in(1));
2055 OptoReg::Name dst_hi = ra_->get_reg_second(this);
2056 OptoReg::Name dst_lo = ra_->get_reg_first(this);
2057
2058 enum RC src_hi_rc = rc_class(src_hi);
2059 enum RC src_lo_rc = rc_class(src_lo);
2060 enum RC dst_hi_rc = rc_class(dst_hi);
2061 enum RC dst_lo_rc = rc_class(dst_lo);
2062
2063 assert(src_lo != OptoReg::Bad && dst_lo != OptoReg::Bad, "must move at least 1 register");
2064
2065 if (src_hi != OptoReg::Bad && !bottom_type()->isa_vectmask()) {
2066 assert((src_lo&1)==0 && src_lo+1==src_hi &&
2067 (dst_lo&1)==0 && dst_lo+1==dst_hi,
2068 "expected aligned-adjacent pairs");
2069 }
2070
2071 if (src_lo == dst_lo && src_hi == dst_hi) {
2072 return 0; // Self copy, no move.
2073 }
2074
2075 bool is64 = (src_lo & 1) == 0 && src_lo + 1 == src_hi &&
2076 (dst_lo & 1) == 0 && dst_lo + 1 == dst_hi;
2077 int src_offset = ra_->reg2offset(src_lo);
2078 int dst_offset = ra_->reg2offset(dst_lo);
2079
2080 if (bottom_type()->isa_vect() && !bottom_type()->isa_vectmask()) {
2081 uint ireg = ideal_reg();
2082 if (ireg == Op_VecA && cbuf) {
2083 C2_MacroAssembler _masm(cbuf);
2084 int sve_vector_reg_size_in_bytes = Matcher::scalable_vector_reg_size(T_BYTE);
2085 if (src_lo_rc == rc_stack && dst_lo_rc == rc_stack) {
2086 // stack->stack
2087 __ spill_copy_sve_vector_stack_to_stack(src_offset, dst_offset,
2088 sve_vector_reg_size_in_bytes);
2089 } else if (src_lo_rc == rc_float && dst_lo_rc == rc_stack) {
2090 __ spill_sve_vector(as_FloatRegister(Matcher::_regEncode[src_lo]), ra_->reg2offset(dst_lo),
2091 sve_vector_reg_size_in_bytes);
2092 } else if (src_lo_rc == rc_stack && dst_lo_rc == rc_float) {
2093 __ unspill_sve_vector(as_FloatRegister(Matcher::_regEncode[dst_lo]), ra_->reg2offset(src_lo),
2094 sve_vector_reg_size_in_bytes);
2095 } else if (src_lo_rc == rc_float && dst_lo_rc == rc_float) {
2096 __ sve_orr(as_FloatRegister(Matcher::_regEncode[dst_lo]),
2097 as_FloatRegister(Matcher::_regEncode[src_lo]),
2098 as_FloatRegister(Matcher::_regEncode[src_lo]));
2099 } else {
2100 ShouldNotReachHere();
2101 }
2102 } else if (cbuf) {
2103 assert(ireg == Op_VecD || ireg == Op_VecX, "must be 64 bit or 128 bit vector");
2104 C2_MacroAssembler _masm(cbuf);
2105 assert((src_lo_rc != rc_int && dst_lo_rc != rc_int), "sanity");
2106 if (src_lo_rc == rc_stack && dst_lo_rc == rc_stack) {
2107 // stack->stack
2108 assert((src_offset & 7) == 0 && (dst_offset & 7) == 0, "unaligned stack offset");
2109 if (ireg == Op_VecD) {
2110 __ unspill(rscratch1, true, src_offset);
2111 __ spill(rscratch1, true, dst_offset);
2112 } else {
2113 __ spill_copy128(src_offset, dst_offset);
2114 }
2115 } else if (src_lo_rc == rc_float && dst_lo_rc == rc_float) {
2116 __ mov(as_FloatRegister(Matcher::_regEncode[dst_lo]),
2117 ireg == Op_VecD ? __ T8B : __ T16B,
2118 as_FloatRegister(Matcher::_regEncode[src_lo]));
2119 } else if (src_lo_rc == rc_float && dst_lo_rc == rc_stack) {
2120 __ spill(as_FloatRegister(Matcher::_regEncode[src_lo]),
2121 ireg == Op_VecD ? __ D : __ Q,
2122 ra_->reg2offset(dst_lo));
2123 } else if (src_lo_rc == rc_stack && dst_lo_rc == rc_float) {
2124 __ unspill(as_FloatRegister(Matcher::_regEncode[dst_lo]),
2125 ireg == Op_VecD ? __ D : __ Q,
2126 ra_->reg2offset(src_lo));
2127 } else {
2128 ShouldNotReachHere();
2129 }
2130 }
2131 } else if (cbuf) {
2132 C2_MacroAssembler _masm(cbuf);
2133 switch (src_lo_rc) {
2134 case rc_int:
2135 if (dst_lo_rc == rc_int) { // gpr --> gpr copy
2136 if (is64) {
2137 __ mov(as_Register(Matcher::_regEncode[dst_lo]),
2138 as_Register(Matcher::_regEncode[src_lo]));
2139 } else {
2140 C2_MacroAssembler _masm(cbuf);
2141 __ movw(as_Register(Matcher::_regEncode[dst_lo]),
2142 as_Register(Matcher::_regEncode[src_lo]));
2143 }
2144 } else if (dst_lo_rc == rc_float) { // gpr --> fpr copy
2145 if (is64) {
2146 __ fmovd(as_FloatRegister(Matcher::_regEncode[dst_lo]),
2147 as_Register(Matcher::_regEncode[src_lo]));
2148 } else {
2149 __ fmovs(as_FloatRegister(Matcher::_regEncode[dst_lo]),
2150 as_Register(Matcher::_regEncode[src_lo]));
2151 }
2152 } else { // gpr --> stack spill
2153 assert(dst_lo_rc == rc_stack, "spill to bad register class");
2154 __ spill(as_Register(Matcher::_regEncode[src_lo]), is64, dst_offset);
2155 }
2156 break;
2157 case rc_float:
2158 if (dst_lo_rc == rc_int) { // fpr --> gpr copy
2159 if (is64) {
2160 __ fmovd(as_Register(Matcher::_regEncode[dst_lo]),
2161 as_FloatRegister(Matcher::_regEncode[src_lo]));
2162 } else {
2163 __ fmovs(as_Register(Matcher::_regEncode[dst_lo]),
2164 as_FloatRegister(Matcher::_regEncode[src_lo]));
2165 }
2166 } else if (dst_lo_rc == rc_float) { // fpr --> fpr copy
2167 if (is64) {
2168 __ fmovd(as_FloatRegister(Matcher::_regEncode[dst_lo]),
2169 as_FloatRegister(Matcher::_regEncode[src_lo]));
2170 } else {
2171 __ fmovs(as_FloatRegister(Matcher::_regEncode[dst_lo]),
2172 as_FloatRegister(Matcher::_regEncode[src_lo]));
2173 }
2174 } else { // fpr --> stack spill
2175 assert(dst_lo_rc == rc_stack, "spill to bad register class");
2176 __ spill(as_FloatRegister(Matcher::_regEncode[src_lo]),
2177 is64 ? __ D : __ S, dst_offset);
2178 }
2179 break;
2180 case rc_stack:
2181 if (dst_lo_rc == rc_int) { // stack --> gpr load
2182 __ unspill(as_Register(Matcher::_regEncode[dst_lo]), is64, src_offset);
2183 } else if (dst_lo_rc == rc_float) { // stack --> fpr load
2184 __ unspill(as_FloatRegister(Matcher::_regEncode[dst_lo]),
2185 is64 ? __ D : __ S, src_offset);
2186 } else if (dst_lo_rc == rc_predicate) {
2187 __ unspill_sve_predicate(as_PRegister(Matcher::_regEncode[dst_lo]), ra_->reg2offset(src_lo),
2188 Matcher::scalable_vector_reg_size(T_BYTE) >> 3);
2189 } else { // stack --> stack copy
2190 assert(dst_lo_rc == rc_stack, "spill to bad register class");
2191 if (ideal_reg() == Op_RegVectMask) {
2192 __ spill_copy_sve_predicate_stack_to_stack(src_offset, dst_offset,
2193 Matcher::scalable_vector_reg_size(T_BYTE) >> 3);
2194 } else {
2195 __ unspill(rscratch1, is64, src_offset);
2196 __ spill(rscratch1, is64, dst_offset);
2197 }
2198 }
2199 break;
2200 case rc_predicate:
2201 if (dst_lo_rc == rc_predicate) {
2202 __ sve_mov(as_PRegister(Matcher::_regEncode[dst_lo]), as_PRegister(Matcher::_regEncode[src_lo]));
2203 } else if (dst_lo_rc == rc_stack) {
2204 __ spill_sve_predicate(as_PRegister(Matcher::_regEncode[src_lo]), ra_->reg2offset(dst_lo),
2205 Matcher::scalable_vector_reg_size(T_BYTE) >> 3);
2206 } else {
2207 assert(false, "bad src and dst rc_class combination.");
2208 ShouldNotReachHere();
2209 }
2210 break;
2211 default:
2212 assert(false, "bad rc_class for spill");
2213 ShouldNotReachHere();
2214 }
2215 }
2216
2217 if (st) {
2218 st->print("spill ");
2219 if (src_lo_rc == rc_stack) {
2220 st->print("[sp, #%d] -> ", ra_->reg2offset(src_lo));
2221 } else {
2222 st->print("%s -> ", Matcher::regName[src_lo]);
2223 }
2224 if (dst_lo_rc == rc_stack) {
2225 st->print("[sp, #%d]", ra_->reg2offset(dst_lo));
2226 } else {
2227 st->print("%s", Matcher::regName[dst_lo]);
2228 }
2229 if (bottom_type()->isa_vect() && !bottom_type()->isa_vectmask()) {
2230 int vsize = 0;
2231 switch (ideal_reg()) {
2232 case Op_VecD:
2233 vsize = 64;
2234 break;
2235 case Op_VecX:
2236 vsize = 128;
2237 break;
2238 case Op_VecA:
2239 vsize = Matcher::scalable_vector_reg_size(T_BYTE) * 8;
2240 break;
2241 default:
2242 assert(false, "bad register type for spill");
2243 ShouldNotReachHere();
2244 }
2245 st->print("\t# vector spill size = %d", vsize);
2246 } else if (ideal_reg() == Op_RegVectMask) {
2247 assert(Matcher::supports_scalable_vector(), "bad register type for spill");
2248 int vsize = Matcher::scalable_predicate_reg_slots() * 32;
2249 st->print("\t# predicate spill size = %d", vsize);
2250 } else {
2251 st->print("\t# spill size = %d", is64 ? 64 : 32);
2252 }
2253 }
2254
2255 return 0;
2256
2257 }
2258
2259 #ifndef PRODUCT
2260 void MachSpillCopyNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
2261 if (!ra_)
2262 st->print("N%d = SpillCopy(N%d)", _idx, in(1)->_idx);
2263 else
2264 implementation(NULL, ra_, false, st);
2265 }
2266 #endif
2267
2268 void MachSpillCopyNode::emit(CodeBuffer &cbuf, PhaseRegAlloc *ra_) const {
2269 implementation(&cbuf, ra_, false, NULL);
2270 }
2271
2272 uint MachSpillCopyNode::size(PhaseRegAlloc *ra_) const {
2273 return MachNode::size(ra_);
2274 }
2275
2276 //=============================================================================
2277
2278 #ifndef PRODUCT
2279 void BoxLockNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
2280 int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
2281 int reg = ra_->get_reg_first(this);
2282 st->print("add %s, rsp, #%d]\t# box lock",
2283 Matcher::regName[reg], offset);
2284 }
2285 #endif
2286
2287 void BoxLockNode::emit(CodeBuffer &cbuf, PhaseRegAlloc *ra_) const {
2288 C2_MacroAssembler _masm(&cbuf);
2289
2290 int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
2291 int reg = ra_->get_encode(this);
2292
2293 // This add will handle any 24-bit signed offset. 24 bits allows an
2294 // 8 megabyte stack frame.
2295 __ add(as_Register(reg), sp, offset);
2296 }
2297
2298 uint BoxLockNode::size(PhaseRegAlloc *ra_) const {
2299 // BoxLockNode is not a MachNode, so we can't just call MachNode::size(ra_).
2300 int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
2301
2302 if (Assembler::operand_valid_for_add_sub_immediate(offset)) {
2303 return NativeInstruction::instruction_size;
2304 } else {
2305 return 2 * NativeInstruction::instruction_size;
2306 }
2307 }
2308
2309 ///=============================================================================
2310 #ifndef PRODUCT
2311 void MachVEPNode::format(PhaseRegAlloc* ra_, outputStream* st) const
2312 {
2313 st->print_cr("# MachVEPNode");
2314 if (!_verified) {
2315 st->print_cr("\t load_class");
2316 } else {
2317 st->print_cr("\t unpack_inline_arg");
2318 }
2319 }
2320 #endif
2321
2322 void MachVEPNode::emit(CodeBuffer& cbuf, PhaseRegAlloc* ra_) const
2323 {
2324 MacroAssembler _masm(&cbuf);
2325
2326 if (!_verified) {
2327 Label skip;
2328 __ cmp_klass(j_rarg0, rscratch2, rscratch1);
2329 __ br(Assembler::EQ, skip);
2330 __ far_jump(RuntimeAddress(SharedRuntime::get_ic_miss_stub()));
2331 __ bind(skip);
2332
2333 } else {
2334 // Unpack inline type args passed as oop and then jump to
2335 // the verified entry point (skipping the unverified entry).
2336 int sp_inc = __ unpack_inline_args(ra_->C, _receiver_only);
2337 // Emit code for verified entry and save increment for stack repair on return
2338 __ verified_entry(ra_->C, sp_inc);
2339 __ b(*_verified_entry);
2340 }
2341 }
2342
2343 //=============================================================================
2344 #ifndef PRODUCT
2345 void MachUEPNode::format(PhaseRegAlloc* ra_, outputStream* st) const
2346 {
2347 st->print_cr("# MachUEPNode");
2348 if (UseCompressedClassPointers) {
2349 st->print_cr("\tldrw rscratch1, j_rarg0 + oopDesc::klass_offset_in_bytes()]\t# compressed klass");
2350 if (CompressedKlassPointers::shift() != 0) {
2351 st->print_cr("\tdecode_klass_not_null rscratch1, rscratch1");
2352 }
2353 } else {
2354 st->print_cr("\tldr rscratch1, j_rarg0 + oopDesc::klass_offset_in_bytes()]\t# compressed klass");
2355 }
2356 st->print_cr("\tcmp r0, rscratch1\t # Inline cache check");
2357 st->print_cr("\tbne, SharedRuntime::_ic_miss_stub");
2358 }
2359 #endif
2360
2361 void MachUEPNode::emit(CodeBuffer& cbuf, PhaseRegAlloc* ra_) const
2362 {
2363 // This is the unverified entry point.
2364 C2_MacroAssembler _masm(&cbuf);
2365 Label skip;
2366
2367 // UseCompressedClassPointers logic are inside cmp_klass
2368 __ cmp_klass(j_rarg0, rscratch2, rscratch1);
2369
2370 // TODO
2371 // can we avoid this skip and still use a reloc?
2372 __ br(Assembler::EQ, skip);
2373 __ far_jump(RuntimeAddress(SharedRuntime::get_ic_miss_stub()));
2374 __ bind(skip);
2375 }
2376
2377 // REQUIRED EMIT CODE
2378
2379 //=============================================================================
2380
2381 // Emit exception handler code.
2382 int HandlerImpl::emit_exception_handler(CodeBuffer& cbuf)
2383 {
2384 // mov rscratch1 #exception_blob_entry_point
2385 // br rscratch1
2386 // Note that the code buffer's insts_mark is always relative to insts.
2387 // That's why we must use the macroassembler to generate a handler.
2388 C2_MacroAssembler _masm(&cbuf);
2389 address base = __ start_a_stub(size_exception_handler());
2390 if (base == NULL) {
2391 ciEnv::current()->record_failure("CodeCache is full");
2392 return 0; // CodeBuffer::expand failed
2393 }
2394 int offset = __ offset();
2395 __ far_jump(RuntimeAddress(OptoRuntime::exception_blob()->entry_point()));
2396 assert(__ offset() - offset <= (int) size_exception_handler(), "overflow");
2397 __ end_a_stub();
2398 return offset;
2399 }
2400
2401 // Emit deopt handler code.
2402 int HandlerImpl::emit_deopt_handler(CodeBuffer& cbuf)
2403 {
2404 // Note that the code buffer's insts_mark is always relative to insts.
2405 // That's why we must use the macroassembler to generate a handler.
2406 C2_MacroAssembler _masm(&cbuf);
2407 address base = __ start_a_stub(size_deopt_handler());
2408 if (base == NULL) {
2409 ciEnv::current()->record_failure("CodeCache is full");
2410 return 0; // CodeBuffer::expand failed
2411 }
2412 int offset = __ offset();
2413
2414 __ adr(lr, __ pc());
2415 __ far_jump(RuntimeAddress(SharedRuntime::deopt_blob()->unpack()));
2416
2417 assert(__ offset() - offset <= (int) size_deopt_handler(), "overflow");
2418 __ end_a_stub();
2419 return offset;
2420 }
2421
2422 // REQUIRED MATCHER CODE
2423
2424 //=============================================================================
2425
2426 const bool Matcher::match_rule_supported(int opcode) {
2427 if (!has_match_rule(opcode))
2428 return false;
2429
2430 bool ret_value = true;
2431 switch (opcode) {
2432 case Op_OnSpinWait:
2433 return VM_Version::supports_on_spin_wait();
2434 case Op_CacheWB:
2435 case Op_CacheWBPreSync:
2436 case Op_CacheWBPostSync:
2437 if (!VM_Version::supports_data_cache_line_flush()) {
2438 ret_value = false;
2439 }
2440 break;
2441 case Op_LoadVectorMasked:
2442 case Op_StoreVectorMasked:
2443 case Op_LoadVectorGatherMasked:
2444 case Op_StoreVectorScatterMasked:
2445 case Op_MaskAll:
2446 case Op_AndVMask:
2447 case Op_OrVMask:
2448 case Op_XorVMask:
2449 if (UseSVE == 0) {
2450 ret_value = false;
2451 }
2452 break;
2453 }
2454
2455 return ret_value; // Per default match rules are supported.
2456 }
2457
2458 // Identify extra cases that we might want to provide match rules for vector nodes and
2459 // other intrinsics guarded with vector length (vlen) and element type (bt).
2460 const bool Matcher::match_rule_supported_vector(int opcode, int vlen, BasicType bt) {
2461 if (!match_rule_supported(opcode)) {
2462 return false;
2463 }
2464 int bit_size = vlen * type2aelembytes(bt) * 8;
2465 if (UseSVE == 0 && bit_size > 128) {
2466 return false;
2467 }
2468 if (UseSVE > 0) {
2469 return op_sve_supported(opcode, vlen, bt);
2470 } else { // NEON
2471 // Special cases
2472 switch (opcode) {
2473 case Op_VectorMaskCmp:
2474 if (vlen < 2 || bit_size < 64) {
2475 return false;
2476 }
2477 break;
2478 case Op_MulAddVS2VI:
2479 if (bit_size < 128) {
2480 return false;
2481 }
2482 break;
2483 case Op_MulVL:
2484 return false;
2485 case Op_VectorLoadShuffle:
2486 case Op_VectorRearrange:
2487 if (vlen < 4) {
2488 return false;
2489 }
2490 break;
2491 case Op_LoadVectorGather:
2492 case Op_StoreVectorScatter:
2493 return false;
2494 default:
2495 break;
2496 }
2497 }
2498 return vector_size_supported(bt, vlen);
2499 }
2500
2501 const bool Matcher::match_rule_supported_vector_masked(int opcode, int vlen, BasicType bt) {
2502 // Only SVE supports masked operations.
2503 if (UseSVE == 0) {
2504 return false;
2505 }
2506 return match_rule_supported(opcode) &&
2507 masked_op_sve_supported(opcode, vlen, bt);
2508 }
2509
2510 const RegMask* Matcher::predicate_reg_mask(void) {
2511 return &_PR_REG_mask;
2512 }
2513
2514 const TypeVectMask* Matcher::predicate_reg_type(const Type* elemTy, int length) {
2515 return new TypeVectMask(elemTy, length);
2516 }
2517
2518 // Vector calling convention not yet implemented.
2519 const bool Matcher::supports_vector_calling_convention(void) {
2520 return false;
2521 }
2522
2523 OptoRegPair Matcher::vector_return_value(uint ideal_reg) {
2524 Unimplemented();
2525 return OptoRegPair(0, 0);
2526 }
2527
2528 // Is this branch offset short enough that a short branch can be used?
2529 //
2530 // NOTE: If the platform does not provide any short branch variants, then
2531 // this method should return false for offset 0.
2532 bool Matcher::is_short_branch_offset(int rule, int br_size, int offset) {
2533 // The passed offset is relative to address of the branch.
2534
2535 return (-32768 <= offset && offset < 32768);
2536 }
2537
2538 // Vector width in bytes.
2539 const int Matcher::vector_width_in_bytes(BasicType bt) {
2540 // The MaxVectorSize should have been set by detecting SVE max vector register size.
2541 int size = MIN2((UseSVE > 0) ? 256 : 16, (int)MaxVectorSize);
2542 // Minimum 2 values in vector
2543 if (size < 2*type2aelembytes(bt)) size = 0;
2544 // But never < 4
2545 if (size < 4) size = 0;
2546 return size;
2547 }
2548
2549 // Limits on vector size (number of elements) loaded into vector.
2550 const int Matcher::max_vector_size(const BasicType bt) {
2551 return vector_width_in_bytes(bt)/type2aelembytes(bt);
2552 }
2553
2554 const int Matcher::min_vector_size(const BasicType bt) {
2555 int max_size = max_vector_size(bt);
2556 // Limit the min vector size to 8 bytes.
2557 int size = 8 / type2aelembytes(bt);
2558 if (bt == T_BYTE) {
2559 // To support vector api shuffle/rearrange.
2560 size = 4;
2561 } else if (bt == T_BOOLEAN) {
2562 // To support vector api load/store mask.
2563 size = 2;
2564 }
2565 if (size < 2) size = 2;
2566 return MIN2(size, max_size);
2567 }
2568
2569 // Actual max scalable vector register length.
2570 const int Matcher::scalable_vector_reg_size(const BasicType bt) {
2571 return Matcher::max_vector_size(bt);
2572 }
2573
2574 // Vector ideal reg.
2575 const uint Matcher::vector_ideal_reg(int len) {
2576 if (UseSVE > 0 && 2 <= len && len <= 256) {
2577 return Op_VecA;
2578 }
2579 switch(len) {
2580 // For 16-bit/32-bit mask vector, reuse VecD.
2581 case 2:
2582 case 4:
2583 case 8: return Op_VecD;
2584 case 16: return Op_VecX;
2585 }
2586 ShouldNotReachHere();
2587 return 0;
2588 }
2589
2590 MachOper* Matcher::pd_specialize_generic_vector_operand(MachOper* original_opnd, uint ideal_reg, bool is_temp) {
2591 ShouldNotReachHere(); // generic vector operands not supported
2592 return NULL;
2593 }
2594
2595 bool Matcher::is_reg2reg_move(MachNode* m) {
2596 ShouldNotReachHere(); // generic vector operands not supported
2597 return false;
2598 }
2599
2600 bool Matcher::is_generic_vector(MachOper* opnd) {
2601 ShouldNotReachHere(); // generic vector operands not supported
2602 return false;
2603 }
2604
2605 // Return whether or not this register is ever used as an argument.
2606 // This function is used on startup to build the trampoline stubs in
2607 // generateOptoStub. Registers not mentioned will be killed by the VM
2608 // call in the trampoline, and arguments in those registers not be
2609 // available to the callee.
2610 bool Matcher::can_be_java_arg(int reg)
2611 {
2612 return
2613 reg == R0_num || reg == R0_H_num ||
2614 reg == R1_num || reg == R1_H_num ||
2615 reg == R2_num || reg == R2_H_num ||
2616 reg == R3_num || reg == R3_H_num ||
2617 reg == R4_num || reg == R4_H_num ||
2618 reg == R5_num || reg == R5_H_num ||
2619 reg == R6_num || reg == R6_H_num ||
2620 reg == R7_num || reg == R7_H_num ||
2621 reg == V0_num || reg == V0_H_num ||
2622 reg == V1_num || reg == V1_H_num ||
2623 reg == V2_num || reg == V2_H_num ||
2624 reg == V3_num || reg == V3_H_num ||
2625 reg == V4_num || reg == V4_H_num ||
2626 reg == V5_num || reg == V5_H_num ||
2627 reg == V6_num || reg == V6_H_num ||
2628 reg == V7_num || reg == V7_H_num;
2629 }
2630
2631 bool Matcher::is_spillable_arg(int reg)
2632 {
2633 return can_be_java_arg(reg);
2634 }
2635
2636 uint Matcher::int_pressure_limit()
2637 {
2638 // JDK-8183543: When taking the number of available registers as int
2639 // register pressure threshold, the jtreg test:
2640 // test/hotspot/jtreg/compiler/regalloc/TestC2IntPressure.java
2641 // failed due to C2 compilation failure with
2642 // "COMPILE SKIPPED: failed spill-split-recycle sanity check".
2643 //
2644 // A derived pointer is live at CallNode and then is flagged by RA
2645 // as a spilled LRG. Spilling heuristics(Spill-USE) explicitly skip
2646 // derived pointers and lastly fail to spill after reaching maximum
2647 // number of iterations. Lowering the default pressure threshold to
2648 // (_NO_SPECIAL_REG32_mask.Size() minus 1) forces CallNode to become
2649 // a high register pressure area of the code so that split_DEF can
2650 // generate DefinitionSpillCopy for the derived pointer.
2651 uint default_int_pressure_threshold = _NO_SPECIAL_REG32_mask.Size() - 1;
2652 if (!PreserveFramePointer) {
2653 // When PreserveFramePointer is off, frame pointer is allocatable,
2654 // but different from other SOC registers, it is excluded from
2655 // fatproj's mask because its save type is No-Save. Decrease 1 to
2656 // ensure high pressure at fatproj when PreserveFramePointer is off.
2657 // See check_pressure_at_fatproj().
2658 default_int_pressure_threshold--;
2659 }
2660 return (INTPRESSURE == -1) ? default_int_pressure_threshold : INTPRESSURE;
2661 }
2662
2663 uint Matcher::float_pressure_limit()
2664 {
2665 // _FLOAT_REG_mask is generated by adlc from the float_reg register class.
2666 return (FLOATPRESSURE == -1) ? _FLOAT_REG_mask.Size() : FLOATPRESSURE;
2667 }
2668
2669 bool Matcher::use_asm_for_ldiv_by_con(jlong divisor) {
2670 return false;
2671 }
2672
2673 RegMask Matcher::divI_proj_mask() {
2674 ShouldNotReachHere();
2675 return RegMask();
2676 }
2677
2678 // Register for MODI projection of divmodI.
2679 RegMask Matcher::modI_proj_mask() {
2680 ShouldNotReachHere();
2681 return RegMask();
2682 }
2683
2684 // Register for DIVL projection of divmodL.
2685 RegMask Matcher::divL_proj_mask() {
2686 ShouldNotReachHere();
2687 return RegMask();
2688 }
2689
2690 // Register for MODL projection of divmodL.
2691 RegMask Matcher::modL_proj_mask() {
2692 ShouldNotReachHere();
2693 return RegMask();
2694 }
2695
2696 const RegMask Matcher::method_handle_invoke_SP_save_mask() {
2697 return FP_REG_mask();
2698 }
2699
2700 bool size_fits_all_mem_uses(AddPNode* addp, int shift) {
2701 for (DUIterator_Fast imax, i = addp->fast_outs(imax); i < imax; i++) {
2702 Node* u = addp->fast_out(i);
2703 if (u->is_LoadStore()) {
2704 // On AArch64, LoadStoreNodes (i.e. compare and swap
2705 // instructions) only take register indirect as an operand, so
2706 // any attempt to use an AddPNode as an input to a LoadStoreNode
2707 // must fail.
2708 return false;
2709 }
2710 if (u->is_Mem()) {
2711 int opsize = u->as_Mem()->memory_size();
2712 assert(opsize > 0, "unexpected memory operand size");
2713 if (u->as_Mem()->memory_size() != (1<<shift)) {
2714 return false;
2715 }
2716 }
2717 }
2718 return true;
2719 }
2720
2721 bool can_combine_with_imm(Node* binary_node, Node* replicate_node) {
2722 if (UseSVE == 0 || !VectorNode::is_invariant_vector(replicate_node)){
2723 return false;
2724 }
2725 Node* imm_node = replicate_node->in(1);
2726 if (!imm_node->is_Con()) {
2727 return false;
2728 }
2729
2730 const Type* t = imm_node->bottom_type();
2731 if (!(t->isa_int() || t->isa_long())) {
2732 return false;
2733 }
2734
2735 switch (binary_node->Opcode()) {
2736 case Op_AndV:
2737 case Op_OrV:
2738 case Op_XorV: {
2739 Assembler::SIMD_RegVariant T = Assembler::elemType_to_regVariant(Matcher::vector_element_basic_type(binary_node));
2740 uint64_t value = t->isa_long() ? (uint64_t)imm_node->get_long() : (uint64_t)imm_node->get_int();
2741 return Assembler::operand_valid_for_sve_logical_immediate(Assembler::regVariant_to_elemBits(T), value);
2742 }
2743 case Op_AddVB:
2744 return (imm_node->get_int() <= 255 && imm_node->get_int() >= -255);
2745 case Op_AddVS:
2746 case Op_AddVI:
2747 return Assembler::operand_valid_for_sve_add_sub_immediate((int64_t)imm_node->get_int());
2748 case Op_AddVL:
2749 return Assembler::operand_valid_for_sve_add_sub_immediate(imm_node->get_long());
2750 default:
2751 return false;
2752 }
2753 }
2754
2755 bool is_vector_arith_imm_pattern(Node* n, Node* m) {
2756 if (n != NULL && m != NULL) {
2757 return can_combine_with_imm(n, m);
2758 }
2759 return false;
2760 }
2761
2762 // Should the matcher clone input 'm' of node 'n'?
2763 bool Matcher::pd_clone_node(Node* n, Node* m, Matcher::MStack& mstack) {
2764 // ShiftV src (ShiftCntV con)
2765 // Binary src (Replicate con)
2766 if (is_vshift_con_pattern(n, m) || is_vector_arith_imm_pattern(n, m)) {
2767 mstack.push(m, Visit);
2768 return true;
2769 }
2770
2771 return false;
2772 }
2773
2774 // Should the Matcher clone shifts on addressing modes, expecting them
2775 // to be subsumed into complex addressing expressions or compute them
2776 // into registers?
2777 bool Matcher::pd_clone_address_expressions(AddPNode* m, Matcher::MStack& mstack, VectorSet& address_visited) {
2778 if (clone_base_plus_offset_address(m, mstack, address_visited)) {
2779 return true;
2780 }
2781
2782 Node *off = m->in(AddPNode::Offset);
2783 if (off->Opcode() == Op_LShiftL && off->in(2)->is_Con() &&
2784 size_fits_all_mem_uses(m, off->in(2)->get_int()) &&
2785 // Are there other uses besides address expressions?
2786 !is_visited(off)) {
2787 address_visited.set(off->_idx); // Flag as address_visited
2788 mstack.push(off->in(2), Visit);
2789 Node *conv = off->in(1);
2790 if (conv->Opcode() == Op_ConvI2L &&
2791 // Are there other uses besides address expressions?
2792 !is_visited(conv)) {
2793 address_visited.set(conv->_idx); // Flag as address_visited
2794 mstack.push(conv->in(1), Pre_Visit);
2795 } else {
2796 mstack.push(conv, Pre_Visit);
2797 }
2798 address_visited.test_set(m->_idx); // Flag as address_visited
2799 mstack.push(m->in(AddPNode::Address), Pre_Visit);
2800 mstack.push(m->in(AddPNode::Base), Pre_Visit);
2801 return true;
2802 } else if (off->Opcode() == Op_ConvI2L &&
2803 // Are there other uses besides address expressions?
2804 !is_visited(off)) {
2805 address_visited.test_set(m->_idx); // Flag as address_visited
2806 address_visited.set(off->_idx); // Flag as address_visited
2807 mstack.push(off->in(1), Pre_Visit);
2808 mstack.push(m->in(AddPNode::Address), Pre_Visit);
2809 mstack.push(m->in(AddPNode::Base), Pre_Visit);
2810 return true;
2811 }
2812 return false;
2813 }
2814
2815 #define MOV_VOLATILE(REG, BASE, INDEX, SCALE, DISP, SCRATCH, INSN) \
2816 C2_MacroAssembler _masm(&cbuf); \
2817 { \
2818 guarantee(INDEX == -1, "mode not permitted for volatile"); \
2819 guarantee(DISP == 0, "mode not permitted for volatile"); \
2820 guarantee(SCALE == 0, "mode not permitted for volatile"); \
2821 __ INSN(REG, as_Register(BASE)); \
2822 }
2823
2824
2825 static Address mem2address(int opcode, Register base, int index, int size, int disp)
2826 {
2827 Address::extend scale;
2828
2829 // Hooboy, this is fugly. We need a way to communicate to the
2830 // encoder that the index needs to be sign extended, so we have to
2831 // enumerate all the cases.
2832 switch (opcode) {
2833 case INDINDEXSCALEDI2L:
2834 case INDINDEXSCALEDI2LN:
2835 case INDINDEXI2L:
2836 case INDINDEXI2LN:
2837 scale = Address::sxtw(size);
2838 break;
2839 default:
2840 scale = Address::lsl(size);
2841 }
2842
2843 if (index == -1) {
2844 return Address(base, disp);
2845 } else {
2846 assert(disp == 0, "unsupported address mode: disp = %d", disp);
2847 return Address(base, as_Register(index), scale);
2848 }
2849 }
2850
2851
2852 typedef void (MacroAssembler::* mem_insn)(Register Rt, const Address &adr);
2853 typedef void (MacroAssembler::* mem_insn2)(Register Rt, Register adr);
2854 typedef void (MacroAssembler::* mem_float_insn)(FloatRegister Rt, const Address &adr);
2855 typedef void (MacroAssembler::* mem_vector_insn)(FloatRegister Rt,
2856 MacroAssembler::SIMD_RegVariant T, const Address &adr);
2857
2858 // Used for all non-volatile memory accesses. The use of
2859 // $mem->opcode() to discover whether this pattern uses sign-extended
2860 // offsets is something of a kludge.
2861 static void loadStore(C2_MacroAssembler masm, mem_insn insn,
2862 Register reg, int opcode,
2863 Register base, int index, int scale, int disp,
2864 int size_in_memory)
2865 {
2866 Address addr = mem2address(opcode, base, index, scale, disp);
2867 if (addr.getMode() == Address::base_plus_offset) {
2868 /* If we get an out-of-range offset it is a bug in the compiler,
2869 so we assert here. */
2870 assert(Address::offset_ok_for_immed(addr.offset(), exact_log2(size_in_memory)),
2871 "c2 compiler bug");
2872 /* Fix up any out-of-range offsets. */
2873 assert_different_registers(rscratch1, base);
2874 assert_different_registers(rscratch1, reg);
2875 addr = masm.legitimize_address(addr, size_in_memory, rscratch1);
2876 }
2877 (masm.*insn)(reg, addr);
2878 }
2879
2880 static void loadStore(C2_MacroAssembler masm, mem_float_insn insn,
2881 FloatRegister reg, int opcode,
2882 Register base, int index, int size, int disp,
2883 int size_in_memory)
2884 {
2885 Address::extend scale;
2886
2887 switch (opcode) {
2888 case INDINDEXSCALEDI2L:
2889 case INDINDEXSCALEDI2LN:
2890 scale = Address::sxtw(size);
2891 break;
2892 default:
2893 scale = Address::lsl(size);
2894 }
2895
2896 if (index == -1) {
2897 /* If we get an out-of-range offset it is a bug in the compiler,
2898 so we assert here. */
2899 assert(Address::offset_ok_for_immed(disp, exact_log2(size_in_memory)), "c2 compiler bug");
2900 /* Fix up any out-of-range offsets. */
2901 assert_different_registers(rscratch1, base);
2902 Address addr = Address(base, disp);
2903 addr = masm.legitimize_address(addr, size_in_memory, rscratch1);
2904 (masm.*insn)(reg, addr);
2905 } else {
2906 assert(disp == 0, "unsupported address mode: disp = %d", disp);
2907 (masm.*insn)(reg, Address(base, as_Register(index), scale));
2908 }
2909 }
2910
2911 static void loadStore(C2_MacroAssembler masm, mem_vector_insn insn,
2912 FloatRegister reg, MacroAssembler::SIMD_RegVariant T,
2913 int opcode, Register base, int index, int size, int disp)
2914 {
2915 if (index == -1) {
2916 (masm.*insn)(reg, T, Address(base, disp));
2917 } else {
2918 assert(disp == 0, "unsupported address mode");
2919 (masm.*insn)(reg, T, Address(base, as_Register(index), Address::lsl(size)));
2920 }
2921 }
2922
2923 %}
2924
2925
2926
2927 //----------ENCODING BLOCK-----------------------------------------------------
2928 // This block specifies the encoding classes used by the compiler to
2929 // output byte streams. Encoding classes are parameterized macros
2930 // used by Machine Instruction Nodes in order to generate the bit
2931 // encoding of the instruction. Operands specify their base encoding
2932 // interface with the interface keyword. There are currently
2933 // supported four interfaces, REG_INTER, CONST_INTER, MEMORY_INTER, &
2934 // COND_INTER. REG_INTER causes an operand to generate a function
2935 // which returns its register number when queried. CONST_INTER causes
2936 // an operand to generate a function which returns the value of the
2937 // constant when queried. MEMORY_INTER causes an operand to generate
2938 // four functions which return the Base Register, the Index Register,
2939 // the Scale Value, and the Offset Value of the operand when queried.
2940 // COND_INTER causes an operand to generate six functions which return
2941 // the encoding code (ie - encoding bits for the instruction)
2942 // associated with each basic boolean condition for a conditional
2943 // instruction.
2944 //
2945 // Instructions specify two basic values for encoding. Again, a
2946 // function is available to check if the constant displacement is an
2947 // oop. They use the ins_encode keyword to specify their encoding
2948 // classes (which must be a sequence of enc_class names, and their
2949 // parameters, specified in the encoding block), and they use the
2950 // opcode keyword to specify, in order, their primary, secondary, and
2951 // tertiary opcode. Only the opcode sections which a particular
2952 // instruction needs for encoding need to be specified.
2953 encode %{
2954 // Build emit functions for each basic byte or larger field in the
2955 // intel encoding scheme (opcode, rm, sib, immediate), and call them
2956 // from C++ code in the enc_class source block. Emit functions will
2957 // live in the main source block for now. In future, we can
2958 // generalize this by adding a syntax that specifies the sizes of
2959 // fields in an order, so that the adlc can build the emit functions
2960 // automagically
2961
2962 // catch all for unimplemented encodings
2963 enc_class enc_unimplemented %{
2964 C2_MacroAssembler _masm(&cbuf);
2965 __ unimplemented("C2 catch all");
2966 %}
2967
2968 // BEGIN Non-volatile memory access
2969
2970 // This encoding class is generated automatically from ad_encode.m4.
2971 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2972 enc_class aarch64_enc_ldrsbw(iRegI dst, memory1 mem) %{
2973 Register dst_reg = as_Register($dst$$reg);
2974 loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrsbw, dst_reg, $mem->opcode(),
2975 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
2976 %}
2977
2978 // This encoding class is generated automatically from ad_encode.m4.
2979 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2980 enc_class aarch64_enc_ldrsb(iRegI dst, memory1 mem) %{
2981 Register dst_reg = as_Register($dst$$reg);
2982 loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrsb, dst_reg, $mem->opcode(),
2983 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
2984 %}
2985
2986 // This encoding class is generated automatically from ad_encode.m4.
2987 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2988 enc_class aarch64_enc_ldrb(iRegI dst, memory1 mem) %{
2989 Register dst_reg = as_Register($dst$$reg);
2990 loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrb, dst_reg, $mem->opcode(),
2991 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
2992 %}
2993
2994 // This encoding class is generated automatically from ad_encode.m4.
2995 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2996 enc_class aarch64_enc_ldrb(iRegL dst, memory1 mem) %{
2997 Register dst_reg = as_Register($dst$$reg);
2998 loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrb, dst_reg, $mem->opcode(),
2999 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
3000 %}
3001
3002 // This encoding class is generated automatically from ad_encode.m4.
3003 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
3004 enc_class aarch64_enc_ldrshw(iRegI dst, memory2 mem) %{
3005 Register dst_reg = as_Register($dst$$reg);
3006 loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrshw, dst_reg, $mem->opcode(),
3007 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
3008 %}
3009
3010 // This encoding class is generated automatically from ad_encode.m4.
3011 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
3012 enc_class aarch64_enc_ldrsh(iRegI dst, memory2 mem) %{
3013 Register dst_reg = as_Register($dst$$reg);
3014 loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrsh, dst_reg, $mem->opcode(),
3015 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
3016 %}
3017
3018 // This encoding class is generated automatically from ad_encode.m4.
3019 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
3020 enc_class aarch64_enc_ldrh(iRegI dst, memory2 mem) %{
3021 Register dst_reg = as_Register($dst$$reg);
3022 loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrh, dst_reg, $mem->opcode(),
3023 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
3024 %}
3025
3026 // This encoding class is generated automatically from ad_encode.m4.
3027 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
3028 enc_class aarch64_enc_ldrh(iRegL dst, memory2 mem) %{
3029 Register dst_reg = as_Register($dst$$reg);
3030 loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrh, dst_reg, $mem->opcode(),
3031 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
3032 %}
3033
3034 // This encoding class is generated automatically from ad_encode.m4.
3035 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
3036 enc_class aarch64_enc_ldrw(iRegI dst, memory4 mem) %{
3037 Register dst_reg = as_Register($dst$$reg);
3038 loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrw, dst_reg, $mem->opcode(),
3039 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
3040 %}
3041
3042 // This encoding class is generated automatically from ad_encode.m4.
3043 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
3044 enc_class aarch64_enc_ldrw(iRegL dst, memory4 mem) %{
3045 Register dst_reg = as_Register($dst$$reg);
3046 loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrw, dst_reg, $mem->opcode(),
3047 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
3048 %}
3049
3050 // This encoding class is generated automatically from ad_encode.m4.
3051 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
3052 enc_class aarch64_enc_ldrsw(iRegL dst, memory4 mem) %{
3053 Register dst_reg = as_Register($dst$$reg);
3054 loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrsw, dst_reg, $mem->opcode(),
3055 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
3056 %}
3057
3058 // This encoding class is generated automatically from ad_encode.m4.
3059 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
3060 enc_class aarch64_enc_ldr(iRegL dst, memory8 mem) %{
3061 Register dst_reg = as_Register($dst$$reg);
3062 loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldr, dst_reg, $mem->opcode(),
3063 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
3064 %}
3065
3066 // This encoding class is generated automatically from ad_encode.m4.
3067 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
3068 enc_class aarch64_enc_ldrs(vRegF dst, memory4 mem) %{
3069 FloatRegister dst_reg = as_FloatRegister($dst$$reg);
3070 loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrs, dst_reg, $mem->opcode(),
3071 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
3072 %}
3073
3074 // This encoding class is generated automatically from ad_encode.m4.
3075 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
3076 enc_class aarch64_enc_ldrd(vRegD dst, memory8 mem) %{
3077 FloatRegister dst_reg = as_FloatRegister($dst$$reg);
3078 loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrd, dst_reg, $mem->opcode(),
3079 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
3080 %}
3081
3082 // This encoding class is generated automatically from ad_encode.m4.
3083 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
3084 enc_class aarch64_enc_strb(iRegI src, memory1 mem) %{
3085 Register src_reg = as_Register($src$$reg);
3086 loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::strb, src_reg, $mem->opcode(),
3087 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
3088 %}
3089
3090 // This encoding class is generated automatically from ad_encode.m4.
3091 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
3092 enc_class aarch64_enc_strb0(memory1 mem) %{
3093 C2_MacroAssembler _masm(&cbuf);
3094 loadStore(_masm, &MacroAssembler::strb, zr, $mem->opcode(),
3095 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
3096 %}
3097
3098 // This encoding class is generated automatically from ad_encode.m4.
3099 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
3100 enc_class aarch64_enc_strh(iRegI src, memory2 mem) %{
3101 Register src_reg = as_Register($src$$reg);
3102 loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::strh, src_reg, $mem->opcode(),
3103 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
3104 %}
3105
3106 // This encoding class is generated automatically from ad_encode.m4.
3107 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
3108 enc_class aarch64_enc_strh0(memory2 mem) %{
3109 C2_MacroAssembler _masm(&cbuf);
3110 loadStore(_masm, &MacroAssembler::strh, zr, $mem->opcode(),
3111 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
3112 %}
3113
3114 // This encoding class is generated automatically from ad_encode.m4.
3115 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
3116 enc_class aarch64_enc_strw(iRegI src, memory4 mem) %{
3117 Register src_reg = as_Register($src$$reg);
3118 loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::strw, src_reg, $mem->opcode(),
3119 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
3120 %}
3121
3122 // This encoding class is generated automatically from ad_encode.m4.
3123 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
3124 enc_class aarch64_enc_strw0(memory4 mem) %{
3125 C2_MacroAssembler _masm(&cbuf);
3126 loadStore(_masm, &MacroAssembler::strw, zr, $mem->opcode(),
3127 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
3128 %}
3129
3130 // This encoding class is generated automatically from ad_encode.m4.
3131 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
3132 enc_class aarch64_enc_str(iRegL src, memory8 mem) %{
3133 Register src_reg = as_Register($src$$reg);
3134 // we sometimes get asked to store the stack pointer into the
3135 // current thread -- we cannot do that directly on AArch64
3136 if (src_reg == r31_sp) {
3137 C2_MacroAssembler _masm(&cbuf);
3138 assert(as_Register($mem$$base) == rthread, "unexpected store for sp");
3139 __ mov(rscratch2, sp);
3140 src_reg = rscratch2;
3141 }
3142 loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::str, src_reg, $mem->opcode(),
3143 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
3144 %}
3145
3146 // This encoding class is generated automatically from ad_encode.m4.
3147 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
3148 enc_class aarch64_enc_str0(memory8 mem) %{
3149 C2_MacroAssembler _masm(&cbuf);
3150 loadStore(_masm, &MacroAssembler::str, zr, $mem->opcode(),
3151 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
3152 %}
3153
3154 // This encoding class is generated automatically from ad_encode.m4.
3155 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
3156 enc_class aarch64_enc_strs(vRegF src, memory4 mem) %{
3157 FloatRegister src_reg = as_FloatRegister($src$$reg);
3158 loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::strs, src_reg, $mem->opcode(),
3159 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
3160 %}
3161
3162 // This encoding class is generated automatically from ad_encode.m4.
3163 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
3164 enc_class aarch64_enc_strd(vRegD src, memory8 mem) %{
3165 FloatRegister src_reg = as_FloatRegister($src$$reg);
3166 loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::strd, src_reg, $mem->opcode(),
3167 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
3168 %}
3169
3170 // This encoding class is generated automatically from ad_encode.m4.
3171 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
3172 enc_class aarch64_enc_strb0_ordered(memory4 mem) %{
3173 C2_MacroAssembler _masm(&cbuf);
3174 __ membar(Assembler::StoreStore);
3175 loadStore(_masm, &MacroAssembler::strb, zr, $mem->opcode(),
3176 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
3177 %}
3178
3179 // END Non-volatile memory access
3180
3181 // Vector loads and stores
3182 enc_class aarch64_enc_ldrvH(vecD dst, memory mem) %{
3183 FloatRegister dst_reg = as_FloatRegister($dst$$reg);
3184 loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldr, dst_reg, MacroAssembler::H,
3185 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3186 %}
3187
3188 enc_class aarch64_enc_ldrvS(vecD dst, memory mem) %{
3189 FloatRegister dst_reg = as_FloatRegister($dst$$reg);
3190 loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldr, dst_reg, MacroAssembler::S,
3191 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3192 %}
3193
3194 enc_class aarch64_enc_ldrvD(vecD dst, memory mem) %{
3195 FloatRegister dst_reg = as_FloatRegister($dst$$reg);
3196 loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldr, dst_reg, MacroAssembler::D,
3197 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3198 %}
3199
3200 enc_class aarch64_enc_ldrvQ(vecX dst, memory mem) %{
3201 FloatRegister dst_reg = as_FloatRegister($dst$$reg);
3202 loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldr, dst_reg, MacroAssembler::Q,
3203 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3204 %}
3205
3206 enc_class aarch64_enc_strvH(vecD src, memory mem) %{
3207 FloatRegister src_reg = as_FloatRegister($src$$reg);
3208 loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::str, src_reg, MacroAssembler::H,
3209 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3210 %}
3211
3212 enc_class aarch64_enc_strvS(vecD src, memory mem) %{
3213 FloatRegister src_reg = as_FloatRegister($src$$reg);
3214 loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::str, src_reg, MacroAssembler::S,
3215 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3216 %}
3217
3218 enc_class aarch64_enc_strvD(vecD src, memory mem) %{
3219 FloatRegister src_reg = as_FloatRegister($src$$reg);
3220 loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::str, src_reg, MacroAssembler::D,
3221 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3222 %}
3223
3224 enc_class aarch64_enc_strvQ(vecX src, memory mem) %{
3225 FloatRegister src_reg = as_FloatRegister($src$$reg);
3226 loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::str, src_reg, MacroAssembler::Q,
3227 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3228 %}
3229
3230 // volatile loads and stores
3231
3232 enc_class aarch64_enc_stlrb(iRegI src, memory mem) %{
3233 MOV_VOLATILE(as_Register($src$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3234 rscratch1, stlrb);
3235 %}
3236
3237 enc_class aarch64_enc_stlrb0(memory mem) %{
3238 MOV_VOLATILE(zr, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3239 rscratch1, stlrb);
3240 %}
3241
3242 enc_class aarch64_enc_stlrh(iRegI src, memory mem) %{
3243 MOV_VOLATILE(as_Register($src$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3244 rscratch1, stlrh);
3245 %}
3246
3247 enc_class aarch64_enc_stlrh0(memory mem) %{
3248 MOV_VOLATILE(zr, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3249 rscratch1, stlrh);
3250 %}
3251
3252 enc_class aarch64_enc_stlrw(iRegI src, memory mem) %{
3253 MOV_VOLATILE(as_Register($src$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3254 rscratch1, stlrw);
3255 %}
3256
3257 enc_class aarch64_enc_stlrw0(memory mem) %{
3258 MOV_VOLATILE(zr, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3259 rscratch1, stlrw);
3260 %}
3261
3262 enc_class aarch64_enc_ldarsbw(iRegI dst, memory mem) %{
3263 Register dst_reg = as_Register($dst$$reg);
3264 MOV_VOLATILE(dst_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3265 rscratch1, ldarb);
3266 __ sxtbw(dst_reg, dst_reg);
3267 %}
3268
3269 enc_class aarch64_enc_ldarsb(iRegL dst, memory mem) %{
3270 Register dst_reg = as_Register($dst$$reg);
3271 MOV_VOLATILE(dst_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3272 rscratch1, ldarb);
3273 __ sxtb(dst_reg, dst_reg);
3274 %}
3275
3276 enc_class aarch64_enc_ldarbw(iRegI dst, memory mem) %{
3277 MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3278 rscratch1, ldarb);
3279 %}
3280
3281 enc_class aarch64_enc_ldarb(iRegL dst, memory mem) %{
3282 MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3283 rscratch1, ldarb);
3284 %}
3285
3286 enc_class aarch64_enc_ldarshw(iRegI dst, memory mem) %{
3287 Register dst_reg = as_Register($dst$$reg);
3288 MOV_VOLATILE(dst_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3289 rscratch1, ldarh);
3290 __ sxthw(dst_reg, dst_reg);
3291 %}
3292
3293 enc_class aarch64_enc_ldarsh(iRegL dst, memory mem) %{
3294 Register dst_reg = as_Register($dst$$reg);
3295 MOV_VOLATILE(dst_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3296 rscratch1, ldarh);
3297 __ sxth(dst_reg, dst_reg);
3298 %}
3299
3300 enc_class aarch64_enc_ldarhw(iRegI dst, memory mem) %{
3301 MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3302 rscratch1, ldarh);
3303 %}
3304
3305 enc_class aarch64_enc_ldarh(iRegL dst, memory mem) %{
3306 MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3307 rscratch1, ldarh);
3308 %}
3309
3310 enc_class aarch64_enc_ldarw(iRegI dst, memory mem) %{
3311 MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3312 rscratch1, ldarw);
3313 %}
3314
3315 enc_class aarch64_enc_ldarw(iRegL dst, memory mem) %{
3316 MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3317 rscratch1, ldarw);
3318 %}
3319
3320 enc_class aarch64_enc_ldar(iRegL dst, memory mem) %{
3321 MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3322 rscratch1, ldar);
3323 %}
3324
3325 enc_class aarch64_enc_fldars(vRegF dst, memory mem) %{
3326 MOV_VOLATILE(rscratch1, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3327 rscratch1, ldarw);
3328 __ fmovs(as_FloatRegister($dst$$reg), rscratch1);
3329 %}
3330
3331 enc_class aarch64_enc_fldard(vRegD dst, memory mem) %{
3332 MOV_VOLATILE(rscratch1, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3333 rscratch1, ldar);
3334 __ fmovd(as_FloatRegister($dst$$reg), rscratch1);
3335 %}
3336
3337 enc_class aarch64_enc_stlr(iRegL src, memory mem) %{
3338 Register src_reg = as_Register($src$$reg);
3339 // we sometimes get asked to store the stack pointer into the
3340 // current thread -- we cannot do that directly on AArch64
3341 if (src_reg == r31_sp) {
3342 C2_MacroAssembler _masm(&cbuf);
3343 assert(as_Register($mem$$base) == rthread, "unexpected store for sp");
3344 __ mov(rscratch2, sp);
3345 src_reg = rscratch2;
3346 }
3347 MOV_VOLATILE(src_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3348 rscratch1, stlr);
3349 %}
3350
3351 enc_class aarch64_enc_stlr0(memory mem) %{
3352 MOV_VOLATILE(zr, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3353 rscratch1, stlr);
3354 %}
3355
3356 enc_class aarch64_enc_fstlrs(vRegF src, memory mem) %{
3357 {
3358 C2_MacroAssembler _masm(&cbuf);
3359 FloatRegister src_reg = as_FloatRegister($src$$reg);
3360 __ fmovs(rscratch2, src_reg);
3361 }
3362 MOV_VOLATILE(rscratch2, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3363 rscratch1, stlrw);
3364 %}
3365
3366 enc_class aarch64_enc_fstlrd(vRegD src, memory mem) %{
3367 {
3368 C2_MacroAssembler _masm(&cbuf);
3369 FloatRegister src_reg = as_FloatRegister($src$$reg);
3370 __ fmovd(rscratch2, src_reg);
3371 }
3372 MOV_VOLATILE(rscratch2, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3373 rscratch1, stlr);
3374 %}
3375
3376 // synchronized read/update encodings
3377
3378 enc_class aarch64_enc_ldaxr(iRegL dst, memory8 mem) %{
3379 C2_MacroAssembler _masm(&cbuf);
3380 Register dst_reg = as_Register($dst$$reg);
3381 Register base = as_Register($mem$$base);
3382 int index = $mem$$index;
3383 int scale = $mem$$scale;
3384 int disp = $mem$$disp;
3385 if (index == -1) {
3386 if (disp != 0) {
3387 __ lea(rscratch1, Address(base, disp));
3388 __ ldaxr(dst_reg, rscratch1);
3389 } else {
3390 // TODO
3391 // should we ever get anything other than this case?
3392 __ ldaxr(dst_reg, base);
3393 }
3394 } else {
3395 Register index_reg = as_Register(index);
3396 if (disp == 0) {
3397 __ lea(rscratch1, Address(base, index_reg, Address::lsl(scale)));
3398 __ ldaxr(dst_reg, rscratch1);
3399 } else {
3400 __ lea(rscratch1, Address(base, disp));
3401 __ lea(rscratch1, Address(rscratch1, index_reg, Address::lsl(scale)));
3402 __ ldaxr(dst_reg, rscratch1);
3403 }
3404 }
3405 %}
3406
3407 enc_class aarch64_enc_stlxr(iRegLNoSp src, memory8 mem) %{
3408 C2_MacroAssembler _masm(&cbuf);
3409 Register src_reg = as_Register($src$$reg);
3410 Register base = as_Register($mem$$base);
3411 int index = $mem$$index;
3412 int scale = $mem$$scale;
3413 int disp = $mem$$disp;
3414 if (index == -1) {
3415 if (disp != 0) {
3416 __ lea(rscratch2, Address(base, disp));
3417 __ stlxr(rscratch1, src_reg, rscratch2);
3418 } else {
3419 // TODO
3420 // should we ever get anything other than this case?
3421 __ stlxr(rscratch1, src_reg, base);
3422 }
3423 } else {
3424 Register index_reg = as_Register(index);
3425 if (disp == 0) {
3426 __ lea(rscratch2, Address(base, index_reg, Address::lsl(scale)));
3427 __ stlxr(rscratch1, src_reg, rscratch2);
3428 } else {
3429 __ lea(rscratch2, Address(base, disp));
3430 __ lea(rscratch2, Address(rscratch2, index_reg, Address::lsl(scale)));
3431 __ stlxr(rscratch1, src_reg, rscratch2);
3432 }
3433 }
3434 __ cmpw(rscratch1, zr);
3435 %}
3436
3437 enc_class aarch64_enc_cmpxchg(memory mem, iRegLNoSp oldval, iRegLNoSp newval) %{
3438 C2_MacroAssembler _masm(&cbuf);
3439 guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
3440 __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
3441 Assembler::xword, /*acquire*/ false, /*release*/ true,
3442 /*weak*/ false, noreg);
3443 %}
3444
3445 enc_class aarch64_enc_cmpxchgw(memory mem, iRegINoSp oldval, iRegINoSp newval) %{
3446 C2_MacroAssembler _masm(&cbuf);
3447 guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
3448 __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
3449 Assembler::word, /*acquire*/ false, /*release*/ true,
3450 /*weak*/ false, noreg);
3451 %}
3452
3453 enc_class aarch64_enc_cmpxchgs(memory mem, iRegINoSp oldval, iRegINoSp newval) %{
3454 C2_MacroAssembler _masm(&cbuf);
3455 guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
3456 __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
3457 Assembler::halfword, /*acquire*/ false, /*release*/ true,
3458 /*weak*/ false, noreg);
3459 %}
3460
3461 enc_class aarch64_enc_cmpxchgb(memory mem, iRegINoSp oldval, iRegINoSp newval) %{
3462 C2_MacroAssembler _masm(&cbuf);
3463 guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
3464 __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
3465 Assembler::byte, /*acquire*/ false, /*release*/ true,
3466 /*weak*/ false, noreg);
3467 %}
3468
3469
3470 // The only difference between aarch64_enc_cmpxchg and
3471 // aarch64_enc_cmpxchg_acq is that we use load-acquire in the
3472 // CompareAndSwap sequence to serve as a barrier on acquiring a
3473 // lock.
3474 enc_class aarch64_enc_cmpxchg_acq(memory mem, iRegLNoSp oldval, iRegLNoSp newval) %{
3475 C2_MacroAssembler _masm(&cbuf);
3476 guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
3477 __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
3478 Assembler::xword, /*acquire*/ true, /*release*/ true,
3479 /*weak*/ false, noreg);
3480 %}
3481
3482 enc_class aarch64_enc_cmpxchgw_acq(memory mem, iRegINoSp oldval, iRegINoSp newval) %{
3483 C2_MacroAssembler _masm(&cbuf);
3484 guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
3485 __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
3486 Assembler::word, /*acquire*/ true, /*release*/ true,
3487 /*weak*/ false, noreg);
3488 %}
3489
3490 enc_class aarch64_enc_cmpxchgs_acq(memory mem, iRegINoSp oldval, iRegINoSp newval) %{
3491 C2_MacroAssembler _masm(&cbuf);
3492 guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
3493 __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
3494 Assembler::halfword, /*acquire*/ true, /*release*/ true,
3495 /*weak*/ false, noreg);
3496 %}
3497
3498 enc_class aarch64_enc_cmpxchgb_acq(memory mem, iRegINoSp oldval, iRegINoSp newval) %{
3499 C2_MacroAssembler _masm(&cbuf);
3500 guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
3501 __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
3502 Assembler::byte, /*acquire*/ true, /*release*/ true,
3503 /*weak*/ false, noreg);
3504 %}
3505
3506 // auxiliary used for CompareAndSwapX to set result register
3507 enc_class aarch64_enc_cset_eq(iRegINoSp res) %{
3508 C2_MacroAssembler _masm(&cbuf);
3509 Register res_reg = as_Register($res$$reg);
3510 __ cset(res_reg, Assembler::EQ);
3511 %}
3512
3513 // prefetch encodings
3514
3515 enc_class aarch64_enc_prefetchw(memory mem) %{
3516 C2_MacroAssembler _masm(&cbuf);
3517 Register base = as_Register($mem$$base);
3518 int index = $mem$$index;
3519 int scale = $mem$$scale;
3520 int disp = $mem$$disp;
3521 if (index == -1) {
3522 __ prfm(Address(base, disp), PSTL1KEEP);
3523 } else {
3524 Register index_reg = as_Register(index);
3525 if (disp == 0) {
3526 __ prfm(Address(base, index_reg, Address::lsl(scale)), PSTL1KEEP);
3527 } else {
3528 __ lea(rscratch1, Address(base, disp));
3529 __ prfm(Address(rscratch1, index_reg, Address::lsl(scale)), PSTL1KEEP);
3530 }
3531 }
3532 %}
3533
3534 /// mov envcodings
3535
3536 enc_class aarch64_enc_movw_imm(iRegI dst, immI src) %{
3537 C2_MacroAssembler _masm(&cbuf);
3538 uint32_t con = (uint32_t)$src$$constant;
3539 Register dst_reg = as_Register($dst$$reg);
3540 if (con == 0) {
3541 __ movw(dst_reg, zr);
3542 } else {
3543 __ movw(dst_reg, con);
3544 }
3545 %}
3546
3547 enc_class aarch64_enc_mov_imm(iRegL dst, immL src) %{
3548 C2_MacroAssembler _masm(&cbuf);
3549 Register dst_reg = as_Register($dst$$reg);
3550 uint64_t con = (uint64_t)$src$$constant;
3551 if (con == 0) {
3552 __ mov(dst_reg, zr);
3553 } else {
3554 __ mov(dst_reg, con);
3555 }
3556 %}
3557
3558 enc_class aarch64_enc_mov_p(iRegP dst, immP src) %{
3559 C2_MacroAssembler _masm(&cbuf);
3560 Register dst_reg = as_Register($dst$$reg);
3561 address con = (address)$src$$constant;
3562 if (con == NULL || con == (address)1) {
3563 ShouldNotReachHere();
3564 } else {
3565 relocInfo::relocType rtype = $src->constant_reloc();
3566 if (rtype == relocInfo::oop_type) {
3567 __ movoop(dst_reg, (jobject)con, /*immediate*/true);
3568 } else if (rtype == relocInfo::metadata_type) {
3569 __ mov_metadata(dst_reg, (Metadata*)con);
3570 } else {
3571 assert(rtype == relocInfo::none, "unexpected reloc type");
3572 if (con < (address)(uintptr_t)os::vm_page_size()) {
3573 __ mov(dst_reg, con);
3574 } else {
3575 uint64_t offset;
3576 __ adrp(dst_reg, con, offset);
3577 __ add(dst_reg, dst_reg, offset);
3578 }
3579 }
3580 }
3581 %}
3582
3583 enc_class aarch64_enc_mov_p0(iRegP dst, immP0 src) %{
3584 C2_MacroAssembler _masm(&cbuf);
3585 Register dst_reg = as_Register($dst$$reg);
3586 __ mov(dst_reg, zr);
3587 %}
3588
3589 enc_class aarch64_enc_mov_p1(iRegP dst, immP_1 src) %{
3590 C2_MacroAssembler _masm(&cbuf);
3591 Register dst_reg = as_Register($dst$$reg);
3592 __ mov(dst_reg, (uint64_t)1);
3593 %}
3594
3595 enc_class aarch64_enc_mov_byte_map_base(iRegP dst, immByteMapBase src) %{
3596 C2_MacroAssembler _masm(&cbuf);
3597 __ load_byte_map_base($dst$$Register);
3598 %}
3599
3600 enc_class aarch64_enc_mov_n(iRegN dst, immN src) %{
3601 C2_MacroAssembler _masm(&cbuf);
3602 Register dst_reg = as_Register($dst$$reg);
3603 address con = (address)$src$$constant;
3604 if (con == NULL) {
3605 ShouldNotReachHere();
3606 } else {
3607 relocInfo::relocType rtype = $src->constant_reloc();
3608 assert(rtype == relocInfo::oop_type, "unexpected reloc type");
3609 __ set_narrow_oop(dst_reg, (jobject)con);
3610 }
3611 %}
3612
3613 enc_class aarch64_enc_mov_n0(iRegN dst, immN0 src) %{
3614 C2_MacroAssembler _masm(&cbuf);
3615 Register dst_reg = as_Register($dst$$reg);
3616 __ mov(dst_reg, zr);
3617 %}
3618
3619 enc_class aarch64_enc_mov_nk(iRegN dst, immNKlass src) %{
3620 C2_MacroAssembler _masm(&cbuf);
3621 Register dst_reg = as_Register($dst$$reg);
3622 address con = (address)$src$$constant;
3623 if (con == NULL) {
3624 ShouldNotReachHere();
3625 } else {
3626 relocInfo::relocType rtype = $src->constant_reloc();
3627 assert(rtype == relocInfo::metadata_type, "unexpected reloc type");
3628 __ set_narrow_klass(dst_reg, (Klass *)con);
3629 }
3630 %}
3631
3632 // arithmetic encodings
3633
3634 enc_class aarch64_enc_addsubw_imm(iRegI dst, iRegI src1, immIAddSub src2) %{
3635 C2_MacroAssembler _masm(&cbuf);
3636 Register dst_reg = as_Register($dst$$reg);
3637 Register src_reg = as_Register($src1$$reg);
3638 int32_t con = (int32_t)$src2$$constant;
3639 // add has primary == 0, subtract has primary == 1
3640 if ($primary) { con = -con; }
3641 if (con < 0) {
3642 __ subw(dst_reg, src_reg, -con);
3643 } else {
3644 __ addw(dst_reg, src_reg, con);
3645 }
3646 %}
3647
3648 enc_class aarch64_enc_addsub_imm(iRegL dst, iRegL src1, immLAddSub src2) %{
3649 C2_MacroAssembler _masm(&cbuf);
3650 Register dst_reg = as_Register($dst$$reg);
3651 Register src_reg = as_Register($src1$$reg);
3652 int32_t con = (int32_t)$src2$$constant;
3653 // add has primary == 0, subtract has primary == 1
3654 if ($primary) { con = -con; }
3655 if (con < 0) {
3656 __ sub(dst_reg, src_reg, -con);
3657 } else {
3658 __ add(dst_reg, src_reg, con);
3659 }
3660 %}
3661
3662 enc_class aarch64_enc_divw(iRegI dst, iRegI src1, iRegI src2) %{
3663 C2_MacroAssembler _masm(&cbuf);
3664 Register dst_reg = as_Register($dst$$reg);
3665 Register src1_reg = as_Register($src1$$reg);
3666 Register src2_reg = as_Register($src2$$reg);
3667 __ corrected_idivl(dst_reg, src1_reg, src2_reg, false, rscratch1);
3668 %}
3669
3670 enc_class aarch64_enc_div(iRegI dst, iRegI src1, iRegI src2) %{
3671 C2_MacroAssembler _masm(&cbuf);
3672 Register dst_reg = as_Register($dst$$reg);
3673 Register src1_reg = as_Register($src1$$reg);
3674 Register src2_reg = as_Register($src2$$reg);
3675 __ corrected_idivq(dst_reg, src1_reg, src2_reg, false, rscratch1);
3676 %}
3677
3678 enc_class aarch64_enc_modw(iRegI dst, iRegI src1, iRegI src2) %{
3679 C2_MacroAssembler _masm(&cbuf);
3680 Register dst_reg = as_Register($dst$$reg);
3681 Register src1_reg = as_Register($src1$$reg);
3682 Register src2_reg = as_Register($src2$$reg);
3683 __ corrected_idivl(dst_reg, src1_reg, src2_reg, true, rscratch1);
3684 %}
3685
3686 enc_class aarch64_enc_mod(iRegI dst, iRegI src1, iRegI src2) %{
3687 C2_MacroAssembler _masm(&cbuf);
3688 Register dst_reg = as_Register($dst$$reg);
3689 Register src1_reg = as_Register($src1$$reg);
3690 Register src2_reg = as_Register($src2$$reg);
3691 __ corrected_idivq(dst_reg, src1_reg, src2_reg, true, rscratch1);
3692 %}
3693
3694 // compare instruction encodings
3695
3696 enc_class aarch64_enc_cmpw(iRegI src1, iRegI src2) %{
3697 C2_MacroAssembler _masm(&cbuf);
3698 Register reg1 = as_Register($src1$$reg);
3699 Register reg2 = as_Register($src2$$reg);
3700 __ cmpw(reg1, reg2);
3701 %}
3702
3703 enc_class aarch64_enc_cmpw_imm_addsub(iRegI src1, immIAddSub src2) %{
3704 C2_MacroAssembler _masm(&cbuf);
3705 Register reg = as_Register($src1$$reg);
3706 int32_t val = $src2$$constant;
3707 if (val >= 0) {
3708 __ subsw(zr, reg, val);
3709 } else {
3710 __ addsw(zr, reg, -val);
3711 }
3712 %}
3713
3714 enc_class aarch64_enc_cmpw_imm(iRegI src1, immI src2) %{
3715 C2_MacroAssembler _masm(&cbuf);
3716 Register reg1 = as_Register($src1$$reg);
3717 uint32_t val = (uint32_t)$src2$$constant;
3718 __ movw(rscratch1, val);
3719 __ cmpw(reg1, rscratch1);
3720 %}
3721
3722 enc_class aarch64_enc_cmp(iRegL src1, iRegL src2) %{
3723 C2_MacroAssembler _masm(&cbuf);
3724 Register reg1 = as_Register($src1$$reg);
3725 Register reg2 = as_Register($src2$$reg);
3726 __ cmp(reg1, reg2);
3727 %}
3728
3729 enc_class aarch64_enc_cmp_imm_addsub(iRegL src1, immL12 src2) %{
3730 C2_MacroAssembler _masm(&cbuf);
3731 Register reg = as_Register($src1$$reg);
3732 int64_t val = $src2$$constant;
3733 if (val >= 0) {
3734 __ subs(zr, reg, val);
3735 } else if (val != -val) {
3736 __ adds(zr, reg, -val);
3737 } else {
3738 // aargh, Long.MIN_VALUE is a special case
3739 __ orr(rscratch1, zr, (uint64_t)val);
3740 __ subs(zr, reg, rscratch1);
3741 }
3742 %}
3743
3744 enc_class aarch64_enc_cmp_imm(iRegL src1, immL src2) %{
3745 C2_MacroAssembler _masm(&cbuf);
3746 Register reg1 = as_Register($src1$$reg);
3747 uint64_t val = (uint64_t)$src2$$constant;
3748 __ mov(rscratch1, val);
3749 __ cmp(reg1, rscratch1);
3750 %}
3751
3752 enc_class aarch64_enc_cmpp(iRegP src1, iRegP src2) %{
3753 C2_MacroAssembler _masm(&cbuf);
3754 Register reg1 = as_Register($src1$$reg);
3755 Register reg2 = as_Register($src2$$reg);
3756 __ cmp(reg1, reg2);
3757 %}
3758
3759 enc_class aarch64_enc_cmpn(iRegN src1, iRegN src2) %{
3760 C2_MacroAssembler _masm(&cbuf);
3761 Register reg1 = as_Register($src1$$reg);
3762 Register reg2 = as_Register($src2$$reg);
3763 __ cmpw(reg1, reg2);
3764 %}
3765
3766 enc_class aarch64_enc_testp(iRegP src) %{
3767 C2_MacroAssembler _masm(&cbuf);
3768 Register reg = as_Register($src$$reg);
3769 __ cmp(reg, zr);
3770 %}
3771
3772 enc_class aarch64_enc_testn(iRegN src) %{
3773 C2_MacroAssembler _masm(&cbuf);
3774 Register reg = as_Register($src$$reg);
3775 __ cmpw(reg, zr);
3776 %}
3777
3778 enc_class aarch64_enc_b(label lbl) %{
3779 C2_MacroAssembler _masm(&cbuf);
3780 Label *L = $lbl$$label;
3781 __ b(*L);
3782 %}
3783
3784 enc_class aarch64_enc_br_con(cmpOp cmp, label lbl) %{
3785 C2_MacroAssembler _masm(&cbuf);
3786 Label *L = $lbl$$label;
3787 __ br ((Assembler::Condition)$cmp$$cmpcode, *L);
3788 %}
3789
3790 enc_class aarch64_enc_br_conU(cmpOpU cmp, label lbl) %{
3791 C2_MacroAssembler _masm(&cbuf);
3792 Label *L = $lbl$$label;
3793 __ br ((Assembler::Condition)$cmp$$cmpcode, *L);
3794 %}
3795
3796 enc_class aarch64_enc_partial_subtype_check(iRegP sub, iRegP super, iRegP temp, iRegP result)
3797 %{
3798 Register sub_reg = as_Register($sub$$reg);
3799 Register super_reg = as_Register($super$$reg);
3800 Register temp_reg = as_Register($temp$$reg);
3801 Register result_reg = as_Register($result$$reg);
3802
3803 Label miss;
3804 C2_MacroAssembler _masm(&cbuf);
3805 __ check_klass_subtype_slow_path(sub_reg, super_reg, temp_reg, result_reg,
3806 NULL, &miss,
3807 /*set_cond_codes:*/ true);
3808 if ($primary) {
3809 __ mov(result_reg, zr);
3810 }
3811 __ bind(miss);
3812 %}
3813
3814 enc_class aarch64_enc_java_static_call(method meth) %{
3815 C2_MacroAssembler _masm(&cbuf);
3816
3817 address addr = (address)$meth$$method;
3818 address call;
3819 if (!_method) {
3820 // A call to a runtime wrapper, e.g. new, new_typeArray_Java, uncommon_trap.
3821 call = __ trampoline_call(Address(addr, relocInfo::runtime_call_type), &cbuf);
3822 if (call == NULL) {
3823 ciEnv::current()->record_failure("CodeCache is full");
3824 return;
3825 }
3826 } else {
3827 int method_index = resolved_method_index(cbuf);
3828 RelocationHolder rspec = _optimized_virtual ? opt_virtual_call_Relocation::spec(method_index)
3829 : static_call_Relocation::spec(method_index);
3830 call = __ trampoline_call(Address(addr, rspec), &cbuf);
3831 if (call == NULL) {
3832 ciEnv::current()->record_failure("CodeCache is full");
3833 return;
3834 }
3835 // Emit stub for static call
3836 address stub = CompiledStaticCall::emit_to_interp_stub(cbuf);
3837 if (stub == NULL) {
3838 ciEnv::current()->record_failure("CodeCache is full");
3839 return;
3840 }
3841 }
3842
3843 // Only non uncommon_trap calls need to reinitialize ptrue.
3844 if (Compile::current()->max_vector_size() > 0 && uncommon_trap_request() == 0) {
3845 __ reinitialize_ptrue();
3846 }
3847 %}
3848
3849 enc_class aarch64_enc_java_dynamic_call(method meth) %{
3850 C2_MacroAssembler _masm(&cbuf);
3851 int method_index = resolved_method_index(cbuf);
3852 address call = __ ic_call((address)$meth$$method, method_index);
3853 if (call == NULL) {
3854 ciEnv::current()->record_failure("CodeCache is full");
3855 return;
3856 } else if (Compile::current()->max_vector_size() > 0) {
3857 __ reinitialize_ptrue();
3858 }
3859 %}
3860
3861 enc_class aarch64_enc_call_epilog() %{
3862 C2_MacroAssembler _masm(&cbuf);
3863 if (VerifyStackAtCalls) {
3864 // Check that stack depth is unchanged: find majik cookie on stack
3865 __ call_Unimplemented();
3866 }
3867 if (tf()->returns_inline_type_as_fields() && !_method->is_method_handle_intrinsic()) {
3868 if (!_method->signature()->returns_null_free_inline_type()) {
3869 // The last return value is not set by the callee but used to pass IsInit information to compiled code.
3870 // Search for the corresponding projection, get the register and emit code that initialized it.
3871 uint con = (tf()->range_cc()->cnt() - 1);
3872 for (DUIterator_Fast imax, i = fast_outs(imax); i < imax; i++) {
3873 ProjNode* proj = fast_out(i)->as_Proj();
3874 if (proj->_con == con) {
3875 // Set IsInit if r0 is non-null (a non-null value is returned buffered or scalarized)
3876 OptoReg::Name optoReg = ra_->get_reg_first(proj);
3877 VMReg reg = OptoReg::as_VMReg(optoReg, ra_->_framesize, OptoReg::reg2stack(ra_->_matcher._new_SP));
3878 Register toReg = reg->is_reg() ? reg->as_Register() : rscratch1;
3879 __ cmp(r0, zr);
3880 __ cset(toReg, Assembler::NE);
3881 if (reg->is_stack()) {
3882 int st_off = reg->reg2stack() * VMRegImpl::stack_slot_size;
3883 __ str(toReg, Address(sp, st_off));
3884 }
3885 break;
3886 }
3887 }
3888 }
3889 if (return_value_is_used()) {
3890 // An inline type is returned as fields in multiple registers.
3891 // R0 either contains an oop if the inline type is buffered or a pointer
3892 // to the corresponding InlineKlass with the lowest bit set to 1. Zero r0
3893 // if the lowest bit is set to allow C2 to use the oop after null checking.
3894 // r0 &= (r0 & 1) - 1
3895 __ andr(rscratch1, r0, 0x1);
3896 __ sub(rscratch1, rscratch1, 0x1);
3897 __ andr(r0, r0, rscratch1);
3898 }
3899 }
3900 %}
3901
3902 enc_class aarch64_enc_java_to_runtime(method meth) %{
3903 C2_MacroAssembler _masm(&cbuf);
3904
3905 // some calls to generated routines (arraycopy code) are scheduled
3906 // by C2 as runtime calls. if so we can call them using a br (they
3907 // will be in a reachable segment) otherwise we have to use a blr
3908 // which loads the absolute address into a register.
3909 address entry = (address)$meth$$method;
3910 CodeBlob *cb = CodeCache::find_blob(entry);
3911 if (cb) {
3912 address call = __ trampoline_call(Address(entry, relocInfo::runtime_call_type));
3913 if (call == NULL) {
3914 ciEnv::current()->record_failure("CodeCache is full");
3915 return;
3916 }
3917 } else {
3918 Label retaddr;
3919 __ adr(rscratch2, retaddr);
3920 __ lea(rscratch1, RuntimeAddress(entry));
3921 // Leave a breadcrumb for JavaFrameAnchor::capture_last_Java_pc()
3922 __ stp(zr, rscratch2, Address(__ pre(sp, -2 * wordSize)));
3923 __ blr(rscratch1);
3924 __ bind(retaddr);
3925 __ add(sp, sp, 2 * wordSize);
3926 }
3927 if (Compile::current()->max_vector_size() > 0) {
3928 __ reinitialize_ptrue();
3929 }
3930 %}
3931
3932 enc_class aarch64_enc_rethrow() %{
3933 C2_MacroAssembler _masm(&cbuf);
3934 __ far_jump(RuntimeAddress(OptoRuntime::rethrow_stub()));
3935 %}
3936
3937 enc_class aarch64_enc_ret() %{
3938 C2_MacroAssembler _masm(&cbuf);
3939 #ifdef ASSERT
3940 if (Compile::current()->max_vector_size() > 0) {
3941 __ verify_ptrue();
3942 }
3943 #endif
3944 __ ret(lr);
3945 %}
3946
3947 enc_class aarch64_enc_tail_call(iRegP jump_target) %{
3948 C2_MacroAssembler _masm(&cbuf);
3949 Register target_reg = as_Register($jump_target$$reg);
3950 __ br(target_reg);
3951 %}
3952
3953 enc_class aarch64_enc_tail_jmp(iRegP jump_target) %{
3954 C2_MacroAssembler _masm(&cbuf);
3955 Register target_reg = as_Register($jump_target$$reg);
3956 // exception oop should be in r0
3957 // ret addr has been popped into lr
3958 // callee expects it in r3
3959 __ mov(r3, lr);
3960 __ br(target_reg);
3961 %}
3962
3963 enc_class aarch64_enc_fast_lock(iRegP object, iRegP box, iRegP tmp, iRegP tmp2) %{
3964 C2_MacroAssembler _masm(&cbuf);
3965 Register oop = as_Register($object$$reg);
3966 Register box = as_Register($box$$reg);
3967 Register disp_hdr = as_Register($tmp$$reg);
3968 Register tmp = as_Register($tmp2$$reg);
3969 Label cont;
3970 Label object_has_monitor;
3971 Label cas_failed;
3972
3973 assert_different_registers(oop, box, tmp, disp_hdr);
3974
3975 // Load markWord from object into displaced_header.
3976 __ ldr(disp_hdr, Address(oop, oopDesc::mark_offset_in_bytes()));
3977
3978 if (DiagnoseSyncOnValueBasedClasses != 0) {
3979 __ load_klass(tmp, oop);
3980 __ ldrw(tmp, Address(tmp, Klass::access_flags_offset()));
3981 __ tstw(tmp, JVM_ACC_IS_VALUE_BASED_CLASS);
3982 __ br(Assembler::NE, cont);
3983 }
3984
3985 // Check for existing monitor
3986 __ tbnz(disp_hdr, exact_log2(markWord::monitor_value), object_has_monitor);
3987
3988 if (!UseHeavyMonitors) {
3989 // Set tmp to be (markWord of object | UNLOCK_VALUE).
3990 __ orr(tmp, disp_hdr, markWord::unlocked_value);
3991
3992 if (EnableValhalla) {
3993 // Mask inline_type bit such that we go to the slow path if object is an inline type
3994 __ andr(tmp, tmp, ~((int) markWord::inline_type_bit_in_place));
3995 }
3996
3997 // Initialize the box. (Must happen before we update the object mark!)
3998 __ str(tmp, Address(box, BasicLock::displaced_header_offset_in_bytes()));
3999
4000 // Compare object markWord with an unlocked value (tmp) and if
4001 // equal exchange the stack address of our box with object markWord.
4002 // On failure disp_hdr contains the possibly locked markWord.
4003 __ cmpxchg(oop, tmp, box, Assembler::xword, /*acquire*/ true,
4004 /*release*/ true, /*weak*/ false, disp_hdr);
4005 __ br(Assembler::EQ, cont);
4006
4007 assert(oopDesc::mark_offset_in_bytes() == 0, "offset of _mark is not 0");
4008
4009 // If the compare-and-exchange succeeded, then we found an unlocked
4010 // object, will have now locked it will continue at label cont
4011
4012 __ bind(cas_failed);
4013 // We did not see an unlocked object so try the fast recursive case.
4014
4015 // Check if the owner is self by comparing the value in the
4016 // markWord of object (disp_hdr) with the stack pointer.
4017 __ mov(rscratch1, sp);
4018 __ sub(disp_hdr, disp_hdr, rscratch1);
4019 __ mov(tmp, (address) (~(os::vm_page_size()-1) | markWord::lock_mask_in_place));
4020 // If condition is true we are cont and hence we can store 0 as the
4021 // displaced header in the box, which indicates that it is a recursive lock.
4022 __ ands(tmp/*==0?*/, disp_hdr, tmp); // Sets flags for result
4023 __ str(tmp/*==0, perhaps*/, Address(box, BasicLock::displaced_header_offset_in_bytes()));
4024 } else {
4025 __ tst(oop, oop); // Set NE to indicate 'failure' -> take slow-path. We know that oop != 0.
4026 }
4027 __ b(cont);
4028
4029 // Handle existing monitor.
4030 __ bind(object_has_monitor);
4031
4032 // The object's monitor m is unlocked iff m->owner == NULL,
4033 // otherwise m->owner may contain a thread or a stack address.
4034 //
4035 // Try to CAS m->owner from NULL to current thread.
4036 __ add(tmp, disp_hdr, (ObjectMonitor::owner_offset_in_bytes()-markWord::monitor_value));
4037 __ cmpxchg(tmp, zr, rthread, Assembler::xword, /*acquire*/ true,
4038 /*release*/ true, /*weak*/ false, rscratch1); // Sets flags for result
4039
4040 // Store a non-null value into the box to avoid looking like a re-entrant
4041 // lock. The fast-path monitor unlock code checks for
4042 // markWord::monitor_value so use markWord::unused_mark which has the
4043 // relevant bit set, and also matches ObjectSynchronizer::enter.
4044 __ mov(tmp, (address)markWord::unused_mark().value());
4045 __ str(tmp, Address(box, BasicLock::displaced_header_offset_in_bytes()));
4046
4047 __ br(Assembler::EQ, cont); // CAS success means locking succeeded
4048
4049 __ cmp(rscratch1, rthread);
4050 __ br(Assembler::NE, cont); // Check for recursive locking
4051
4052 // Recursive lock case
4053 __ increment(Address(disp_hdr, ObjectMonitor::recursions_offset_in_bytes() - markWord::monitor_value), 1);
4054 // flag == EQ still from the cmp above, checking if this is a reentrant lock
4055
4056 __ bind(cont);
4057 // flag == EQ indicates success
4058 // flag == NE indicates failure
4059 %}
4060
4061 enc_class aarch64_enc_fast_unlock(iRegP object, iRegP box, iRegP tmp, iRegP tmp2) %{
4062 C2_MacroAssembler _masm(&cbuf);
4063 Register oop = as_Register($object$$reg);
4064 Register box = as_Register($box$$reg);
4065 Register disp_hdr = as_Register($tmp$$reg);
4066 Register tmp = as_Register($tmp2$$reg);
4067 Label cont;
4068 Label object_has_monitor;
4069
4070 assert_different_registers(oop, box, tmp, disp_hdr);
4071
4072 if (!UseHeavyMonitors) {
4073 // Find the lock address and load the displaced header from the stack.
4074 __ ldr(disp_hdr, Address(box, BasicLock::displaced_header_offset_in_bytes()));
4075
4076 // If the displaced header is 0, we have a recursive unlock.
4077 __ cmp(disp_hdr, zr);
4078 __ br(Assembler::EQ, cont);
4079 }
4080
4081 // Handle existing monitor.
4082 __ ldr(tmp, Address(oop, oopDesc::mark_offset_in_bytes()));
4083 __ tbnz(disp_hdr, exact_log2(markWord::monitor_value), object_has_monitor);
4084
4085 if (!UseHeavyMonitors) {
4086 // Check if it is still a light weight lock, this is is true if we
4087 // see the stack address of the basicLock in the markWord of the
4088 // object.
4089
4090 __ cmpxchg(oop, box, disp_hdr, Assembler::xword, /*acquire*/ false,
4091 /*release*/ true, /*weak*/ false, tmp);
4092 } else {
4093 __ tst(oop, oop); // Set NE to indicate 'failure' -> take slow-path. We know that oop != 0.
4094 }
4095 __ b(cont);
4096
4097 assert(oopDesc::mark_offset_in_bytes() == 0, "offset of _mark is not 0");
4098
4099 // Handle existing monitor.
4100 __ bind(object_has_monitor);
4101 STATIC_ASSERT(markWord::monitor_value <= INT_MAX);
4102 __ add(tmp, tmp, -(int)markWord::monitor_value); // monitor
4103 __ ldr(disp_hdr, Address(tmp, ObjectMonitor::recursions_offset_in_bytes()));
4104
4105 Label notRecursive;
4106 __ cbz(disp_hdr, notRecursive);
4107
4108 // Recursive lock
4109 __ sub(disp_hdr, disp_hdr, 1u);
4110 __ str(disp_hdr, Address(tmp, ObjectMonitor::recursions_offset_in_bytes()));
4111 __ cmp(disp_hdr, disp_hdr); // Sets flags for result
4112 __ b(cont);
4113
4114 __ bind(notRecursive);
4115 __ ldr(rscratch1, Address(tmp, ObjectMonitor::EntryList_offset_in_bytes()));
4116 __ ldr(disp_hdr, Address(tmp, ObjectMonitor::cxq_offset_in_bytes()));
4117 __ orr(rscratch1, rscratch1, disp_hdr); // Will be 0 if both are 0.
4118 __ cmp(rscratch1, zr); // Sets flags for result
4119 __ cbnz(rscratch1, cont);
4120 // need a release store here
4121 __ lea(tmp, Address(tmp, ObjectMonitor::owner_offset_in_bytes()));
4122 __ stlr(zr, tmp); // set unowned
4123
4124 __ bind(cont);
4125 // flag == EQ indicates success
4126 // flag == NE indicates failure
4127 %}
4128
4129 %}
4130
4131 //----------FRAME--------------------------------------------------------------
4132 // Definition of frame structure and management information.
4133 //
4134 // S T A C K L A Y O U T Allocators stack-slot number
4135 // | (to get allocators register number
4136 // G Owned by | | v add OptoReg::stack0())
4137 // r CALLER | |
4138 // o | +--------+ pad to even-align allocators stack-slot
4139 // w V | pad0 | numbers; owned by CALLER
4140 // t -----------+--------+----> Matcher::_in_arg_limit, unaligned
4141 // h ^ | in | 5
4142 // | | args | 4 Holes in incoming args owned by SELF
4143 // | | | | 3
4144 // | | +--------+
4145 // V | | old out| Empty on Intel, window on Sparc
4146 // | old |preserve| Must be even aligned.
4147 // | SP-+--------+----> Matcher::_old_SP, even aligned
4148 // | | in | 3 area for Intel ret address
4149 // Owned by |preserve| Empty on Sparc.
4150 // SELF +--------+
4151 // | | pad2 | 2 pad to align old SP
4152 // | +--------+ 1
4153 // | | locks | 0
4154 // | +--------+----> OptoReg::stack0(), even aligned
4155 // | | pad1 | 11 pad to align new SP
4156 // | +--------+
4157 // | | | 10
4158 // | | spills | 9 spills
4159 // V | | 8 (pad0 slot for callee)
4160 // -----------+--------+----> Matcher::_out_arg_limit, unaligned
4161 // ^ | out | 7
4162 // | | args | 6 Holes in outgoing args owned by CALLEE
4163 // Owned by +--------+
4164 // CALLEE | new out| 6 Empty on Intel, window on Sparc
4165 // | new |preserve| Must be even-aligned.
4166 // | SP-+--------+----> Matcher::_new_SP, even aligned
4167 // | | |
4168 //
4169 // Note 1: Only region 8-11 is determined by the allocator. Region 0-5 is
4170 // known from SELF's arguments and the Java calling convention.
4171 // Region 6-7 is determined per call site.
4172 // Note 2: If the calling convention leaves holes in the incoming argument
4173 // area, those holes are owned by SELF. Holes in the outgoing area
4174 // are owned by the CALLEE. Holes should not be nessecary in the
4175 // incoming area, as the Java calling convention is completely under
4176 // the control of the AD file. Doubles can be sorted and packed to
4177 // avoid holes. Holes in the outgoing arguments may be nessecary for
4178 // varargs C calling conventions.
4179 // Note 3: Region 0-3 is even aligned, with pad2 as needed. Region 3-5 is
4180 // even aligned with pad0 as needed.
4181 // Region 6 is even aligned. Region 6-7 is NOT even aligned;
4182 // (the latter is true on Intel but is it false on AArch64?)
4183 // region 6-11 is even aligned; it may be padded out more so that
4184 // the region from SP to FP meets the minimum stack alignment.
4185 // Note 4: For I2C adapters, the incoming FP may not meet the minimum stack
4186 // alignment. Region 11, pad1, may be dynamically extended so that
4187 // SP meets the minimum alignment.
4188
4189 frame %{
4190 // These three registers define part of the calling convention
4191 // between compiled code and the interpreter.
4192
4193 // Inline Cache Register or Method for I2C.
4194 inline_cache_reg(R12);
4195
4196 // Number of stack slots consumed by locking an object
4197 sync_stack_slots(2);
4198
4199 // Compiled code's Frame Pointer
4200 frame_pointer(R31);
4201
4202 // Interpreter stores its frame pointer in a register which is
4203 // stored to the stack by I2CAdaptors.
4204 // I2CAdaptors convert from interpreted java to compiled java.
4205 interpreter_frame_pointer(R29);
4206
4207 // Stack alignment requirement
4208 stack_alignment(StackAlignmentInBytes); // Alignment size in bytes (128-bit -> 16 bytes)
4209
4210 // Number of outgoing stack slots killed above the out_preserve_stack_slots
4211 // for calls to C. Supports the var-args backing area for register parms.
4212 varargs_C_out_slots_killed(frame::arg_reg_save_area_bytes/BytesPerInt);
4213
4214 // The after-PROLOG location of the return address. Location of
4215 // return address specifies a type (REG or STACK) and a number
4216 // representing the register number (i.e. - use a register name) or
4217 // stack slot.
4218 // Ret Addr is on stack in slot 0 if no locks or verification or alignment.
4219 // Otherwise, it is above the locks and verification slot and alignment word
4220 // TODO this may well be correct but need to check why that - 2 is there
4221 // ppc port uses 0 but we definitely need to allow for fixed_slots
4222 // which folds in the space used for monitors
4223 return_addr(STACK - 2 +
4224 align_up((Compile::current()->in_preserve_stack_slots() +
4225 Compile::current()->fixed_slots()),
4226 stack_alignment_in_slots()));
4227
4228 // Location of compiled Java return values. Same as C for now.
4229 return_value
4230 %{
4231 // TODO do we allow ideal_reg == Op_RegN???
4232 assert(ideal_reg >= Op_RegI && ideal_reg <= Op_RegL,
4233 "only return normal values");
4234
4235 static const int lo[Op_RegL + 1] = { // enum name
4236 0, // Op_Node
4237 0, // Op_Set
4238 R0_num, // Op_RegN
4239 R0_num, // Op_RegI
4240 R0_num, // Op_RegP
4241 V0_num, // Op_RegF
4242 V0_num, // Op_RegD
4243 R0_num // Op_RegL
4244 };
4245
4246 static const int hi[Op_RegL + 1] = { // enum name
4247 0, // Op_Node
4248 0, // Op_Set
4249 OptoReg::Bad, // Op_RegN
4250 OptoReg::Bad, // Op_RegI
4251 R0_H_num, // Op_RegP
4252 OptoReg::Bad, // Op_RegF
4253 V0_H_num, // Op_RegD
4254 R0_H_num // Op_RegL
4255 };
4256
4257 return OptoRegPair(hi[ideal_reg], lo[ideal_reg]);
4258 %}
4259 %}
4260
4261 //----------ATTRIBUTES---------------------------------------------------------
4262 //----------Operand Attributes-------------------------------------------------
4263 op_attrib op_cost(1); // Required cost attribute
4264
4265 //----------Instruction Attributes---------------------------------------------
4266 ins_attrib ins_cost(INSN_COST); // Required cost attribute
4267 ins_attrib ins_size(32); // Required size attribute (in bits)
4268 ins_attrib ins_short_branch(0); // Required flag: is this instruction
4269 // a non-matching short branch variant
4270 // of some long branch?
4271 ins_attrib ins_alignment(4); // Required alignment attribute (must
4272 // be a power of 2) specifies the
4273 // alignment that some part of the
4274 // instruction (not necessarily the
4275 // start) requires. If > 1, a
4276 // compute_padding() function must be
4277 // provided for the instruction
4278
4279 //----------OPERANDS-----------------------------------------------------------
4280 // Operand definitions must precede instruction definitions for correct parsing
4281 // in the ADLC because operands constitute user defined types which are used in
4282 // instruction definitions.
4283
4284 //----------Simple Operands----------------------------------------------------
4285
4286 // Integer operands 32 bit
4287 // 32 bit immediate
4288 operand immI()
4289 %{
4290 match(ConI);
4291
4292 op_cost(0);
4293 format %{ %}
4294 interface(CONST_INTER);
4295 %}
4296
4297 // 32 bit zero
4298 operand immI0()
4299 %{
4300 predicate(n->get_int() == 0);
4301 match(ConI);
4302
4303 op_cost(0);
4304 format %{ %}
4305 interface(CONST_INTER);
4306 %}
4307
4308 // 32 bit unit increment
4309 operand immI_1()
4310 %{
4311 predicate(n->get_int() == 1);
4312 match(ConI);
4313
4314 op_cost(0);
4315 format %{ %}
4316 interface(CONST_INTER);
4317 %}
4318
4319 // 32 bit unit decrement
4320 operand immI_M1()
4321 %{
4322 predicate(n->get_int() == -1);
4323 match(ConI);
4324
4325 op_cost(0);
4326 format %{ %}
4327 interface(CONST_INTER);
4328 %}
4329
4330 // Shift values for add/sub extension shift
4331 operand immIExt()
4332 %{
4333 predicate(0 <= n->get_int() && (n->get_int() <= 4));
4334 match(ConI);
4335
4336 op_cost(0);
4337 format %{ %}
4338 interface(CONST_INTER);
4339 %}
4340
4341 operand immI_gt_1()
4342 %{
4343 predicate(n->get_int() > 1);
4344 match(ConI);
4345
4346 op_cost(0);
4347 format %{ %}
4348 interface(CONST_INTER);
4349 %}
4350
4351 operand immI_le_4()
4352 %{
4353 predicate(n->get_int() <= 4);
4354 match(ConI);
4355
4356 op_cost(0);
4357 format %{ %}
4358 interface(CONST_INTER);
4359 %}
4360
4361 operand immI_31()
4362 %{
4363 predicate(n->get_int() == 31);
4364 match(ConI);
4365
4366 op_cost(0);
4367 format %{ %}
4368 interface(CONST_INTER);
4369 %}
4370
4371 operand immI_2()
4372 %{
4373 predicate(n->get_int() == 2);
4374 match(ConI);
4375
4376 op_cost(0);
4377 format %{ %}
4378 interface(CONST_INTER);
4379 %}
4380
4381 operand immI_4()
4382 %{
4383 predicate(n->get_int() == 4);
4384 match(ConI);
4385
4386 op_cost(0);
4387 format %{ %}
4388 interface(CONST_INTER);
4389 %}
4390
4391 operand immI_8()
4392 %{
4393 predicate(n->get_int() == 8);
4394 match(ConI);
4395
4396 op_cost(0);
4397 format %{ %}
4398 interface(CONST_INTER);
4399 %}
4400
4401 operand immI_16()
4402 %{
4403 predicate(n->get_int() == 16);
4404 match(ConI);
4405
4406 op_cost(0);
4407 format %{ %}
4408 interface(CONST_INTER);
4409 %}
4410
4411 operand immI_24()
4412 %{
4413 predicate(n->get_int() == 24);
4414 match(ConI);
4415
4416 op_cost(0);
4417 format %{ %}
4418 interface(CONST_INTER);
4419 %}
4420
4421 operand immI_32()
4422 %{
4423 predicate(n->get_int() == 32);
4424 match(ConI);
4425
4426 op_cost(0);
4427 format %{ %}
4428 interface(CONST_INTER);
4429 %}
4430
4431 operand immI_48()
4432 %{
4433 predicate(n->get_int() == 48);
4434 match(ConI);
4435
4436 op_cost(0);
4437 format %{ %}
4438 interface(CONST_INTER);
4439 %}
4440
4441 operand immI_56()
4442 %{
4443 predicate(n->get_int() == 56);
4444 match(ConI);
4445
4446 op_cost(0);
4447 format %{ %}
4448 interface(CONST_INTER);
4449 %}
4450
4451 operand immI_63()
4452 %{
4453 predicate(n->get_int() == 63);
4454 match(ConI);
4455
4456 op_cost(0);
4457 format %{ %}
4458 interface(CONST_INTER);
4459 %}
4460
4461 operand immI_64()
4462 %{
4463 predicate(n->get_int() == 64);
4464 match(ConI);
4465
4466 op_cost(0);
4467 format %{ %}
4468 interface(CONST_INTER);
4469 %}
4470
4471 operand immI_255()
4472 %{
4473 predicate(n->get_int() == 255);
4474 match(ConI);
4475
4476 op_cost(0);
4477 format %{ %}
4478 interface(CONST_INTER);
4479 %}
4480
4481 operand immI_65535()
4482 %{
4483 predicate(n->get_int() == 65535);
4484 match(ConI);
4485
4486 op_cost(0);
4487 format %{ %}
4488 interface(CONST_INTER);
4489 %}
4490
4491 operand immL_255()
4492 %{
4493 predicate(n->get_long() == 255L);
4494 match(ConL);
4495
4496 op_cost(0);
4497 format %{ %}
4498 interface(CONST_INTER);
4499 %}
4500
4501 operand immL_65535()
4502 %{
4503 predicate(n->get_long() == 65535L);
4504 match(ConL);
4505
4506 op_cost(0);
4507 format %{ %}
4508 interface(CONST_INTER);
4509 %}
4510
4511 operand immL_4294967295()
4512 %{
4513 predicate(n->get_long() == 4294967295L);
4514 match(ConL);
4515
4516 op_cost(0);
4517 format %{ %}
4518 interface(CONST_INTER);
4519 %}
4520
4521 operand immL_bitmask()
4522 %{
4523 predicate((n->get_long() != 0)
4524 && ((n->get_long() & 0xc000000000000000l) == 0)
4525 && is_power_of_2(n->get_long() + 1));
4526 match(ConL);
4527
4528 op_cost(0);
4529 format %{ %}
4530 interface(CONST_INTER);
4531 %}
4532
4533 operand immI_bitmask()
4534 %{
4535 predicate((n->get_int() != 0)
4536 && ((n->get_int() & 0xc0000000) == 0)
4537 && is_power_of_2(n->get_int() + 1));
4538 match(ConI);
4539
4540 op_cost(0);
4541 format %{ %}
4542 interface(CONST_INTER);
4543 %}
4544
4545 operand immL_positive_bitmaskI()
4546 %{
4547 predicate((n->get_long() != 0)
4548 && ((julong)n->get_long() < 0x80000000ULL)
4549 && is_power_of_2(n->get_long() + 1));
4550 match(ConL);
4551
4552 op_cost(0);
4553 format %{ %}
4554 interface(CONST_INTER);
4555 %}
4556
4557 // Scale values for scaled offset addressing modes (up to long but not quad)
4558 operand immIScale()
4559 %{
4560 predicate(0 <= n->get_int() && (n->get_int() <= 3));
4561 match(ConI);
4562
4563 op_cost(0);
4564 format %{ %}
4565 interface(CONST_INTER);
4566 %}
4567
4568 // 26 bit signed offset -- for pc-relative branches
4569 operand immI26()
4570 %{
4571 predicate(((-(1 << 25)) <= n->get_int()) && (n->get_int() < (1 << 25)));
4572 match(ConI);
4573
4574 op_cost(0);
4575 format %{ %}
4576 interface(CONST_INTER);
4577 %}
4578
4579 // 19 bit signed offset -- for pc-relative loads
4580 operand immI19()
4581 %{
4582 predicate(((-(1 << 18)) <= n->get_int()) && (n->get_int() < (1 << 18)));
4583 match(ConI);
4584
4585 op_cost(0);
4586 format %{ %}
4587 interface(CONST_INTER);
4588 %}
4589
4590 // 12 bit unsigned offset -- for base plus immediate loads
4591 operand immIU12()
4592 %{
4593 predicate((0 <= n->get_int()) && (n->get_int() < (1 << 12)));
4594 match(ConI);
4595
4596 op_cost(0);
4597 format %{ %}
4598 interface(CONST_INTER);
4599 %}
4600
4601 operand immLU12()
4602 %{
4603 predicate((0 <= n->get_long()) && (n->get_long() < (1 << 12)));
4604 match(ConL);
4605
4606 op_cost(0);
4607 format %{ %}
4608 interface(CONST_INTER);
4609 %}
4610
4611 // Offset for scaled or unscaled immediate loads and stores
4612 operand immIOffset()
4613 %{
4614 predicate(Address::offset_ok_for_immed(n->get_int(), 0));
4615 match(ConI);
4616
4617 op_cost(0);
4618 format %{ %}
4619 interface(CONST_INTER);
4620 %}
4621
4622 operand immIOffset1()
4623 %{
4624 predicate(Address::offset_ok_for_immed(n->get_int(), 0));
4625 match(ConI);
4626
4627 op_cost(0);
4628 format %{ %}
4629 interface(CONST_INTER);
4630 %}
4631
4632 operand immIOffset2()
4633 %{
4634 predicate(Address::offset_ok_for_immed(n->get_int(), 1));
4635 match(ConI);
4636
4637 op_cost(0);
4638 format %{ %}
4639 interface(CONST_INTER);
4640 %}
4641
4642 operand immIOffset4()
4643 %{
4644 predicate(Address::offset_ok_for_immed(n->get_int(), 2));
4645 match(ConI);
4646
4647 op_cost(0);
4648 format %{ %}
4649 interface(CONST_INTER);
4650 %}
4651
4652 operand immIOffset8()
4653 %{
4654 predicate(Address::offset_ok_for_immed(n->get_int(), 3));
4655 match(ConI);
4656
4657 op_cost(0);
4658 format %{ %}
4659 interface(CONST_INTER);
4660 %}
4661
4662 operand immIOffset16()
4663 %{
4664 predicate(Address::offset_ok_for_immed(n->get_int(), 4));
4665 match(ConI);
4666
4667 op_cost(0);
4668 format %{ %}
4669 interface(CONST_INTER);
4670 %}
4671
4672 operand immLoffset()
4673 %{
4674 predicate(Address::offset_ok_for_immed(n->get_long(), 0));
4675 match(ConL);
4676
4677 op_cost(0);
4678 format %{ %}
4679 interface(CONST_INTER);
4680 %}
4681
4682 operand immLoffset1()
4683 %{
4684 predicate(Address::offset_ok_for_immed(n->get_long(), 0));
4685 match(ConL);
4686
4687 op_cost(0);
4688 format %{ %}
4689 interface(CONST_INTER);
4690 %}
4691
4692 operand immLoffset2()
4693 %{
4694 predicate(Address::offset_ok_for_immed(n->get_long(), 1));
4695 match(ConL);
4696
4697 op_cost(0);
4698 format %{ %}
4699 interface(CONST_INTER);
4700 %}
4701
4702 operand immLoffset4()
4703 %{
4704 predicate(Address::offset_ok_for_immed(n->get_long(), 2));
4705 match(ConL);
4706
4707 op_cost(0);
4708 format %{ %}
4709 interface(CONST_INTER);
4710 %}
4711
4712 operand immLoffset8()
4713 %{
4714 predicate(Address::offset_ok_for_immed(n->get_long(), 3));
4715 match(ConL);
4716
4717 op_cost(0);
4718 format %{ %}
4719 interface(CONST_INTER);
4720 %}
4721
4722 operand immLoffset16()
4723 %{
4724 predicate(Address::offset_ok_for_immed(n->get_long(), 4));
4725 match(ConL);
4726
4727 op_cost(0);
4728 format %{ %}
4729 interface(CONST_INTER);
4730 %}
4731
4732 // 8 bit signed value.
4733 operand immI8()
4734 %{
4735 predicate(n->get_int() <= 127 && n->get_int() >= -128);
4736 match(ConI);
4737
4738 op_cost(0);
4739 format %{ %}
4740 interface(CONST_INTER);
4741 %}
4742
4743 // 8 bit signed value (simm8), or #simm8 LSL 8.
4744 operand immI8_shift8()
4745 %{
4746 predicate((n->get_int() <= 127 && n->get_int() >= -128) ||
4747 (n->get_int() <= 32512 && n->get_int() >= -32768 && (n->get_int() & 0xff) == 0));
4748 match(ConI);
4749
4750 op_cost(0);
4751 format %{ %}
4752 interface(CONST_INTER);
4753 %}
4754
4755 // 8 bit signed value (simm8), or #simm8 LSL 8.
4756 operand immL8_shift8()
4757 %{
4758 predicate((n->get_long() <= 127 && n->get_long() >= -128) ||
4759 (n->get_long() <= 32512 && n->get_long() >= -32768 && (n->get_long() & 0xff) == 0));
4760 match(ConL);
4761
4762 op_cost(0);
4763 format %{ %}
4764 interface(CONST_INTER);
4765 %}
4766
4767 // 8 bit integer valid for vector add sub immediate
4768 operand immBAddSubV()
4769 %{
4770 predicate(n->get_int() <= 255 && n->get_int() >= -255);
4771 match(ConI);
4772
4773 op_cost(0);
4774 format %{ %}
4775 interface(CONST_INTER);
4776 %}
4777
4778 // 32 bit integer valid for add sub immediate
4779 operand immIAddSub()
4780 %{
4781 predicate(Assembler::operand_valid_for_add_sub_immediate((int64_t)n->get_int()));
4782 match(ConI);
4783 op_cost(0);
4784 format %{ %}
4785 interface(CONST_INTER);
4786 %}
4787
4788 // 32 bit integer valid for vector add sub immediate
4789 operand immIAddSubV()
4790 %{
4791 predicate(Assembler::operand_valid_for_sve_add_sub_immediate((int64_t)n->get_int()));
4792 match(ConI);
4793
4794 op_cost(0);
4795 format %{ %}
4796 interface(CONST_INTER);
4797 %}
4798
4799 // 32 bit unsigned integer valid for logical immediate
4800
4801 operand immBLog()
4802 %{
4803 predicate(Assembler::operand_valid_for_sve_logical_immediate(BitsPerByte, (uint64_t)n->get_int()));
4804 match(ConI);
4805
4806 op_cost(0);
4807 format %{ %}
4808 interface(CONST_INTER);
4809 %}
4810
4811 operand immSLog()
4812 %{
4813 predicate(Assembler::operand_valid_for_sve_logical_immediate(BitsPerShort, (uint64_t)n->get_int()));
4814 match(ConI);
4815
4816 op_cost(0);
4817 format %{ %}
4818 interface(CONST_INTER);
4819 %}
4820
4821 operand immILog()
4822 %{
4823 predicate(Assembler::operand_valid_for_logical_immediate(/*is32*/true, (uint64_t)n->get_int()));
4824 match(ConI);
4825
4826 op_cost(0);
4827 format %{ %}
4828 interface(CONST_INTER);
4829 %}
4830
4831 // Integer operands 64 bit
4832 // 64 bit immediate
4833 operand immL()
4834 %{
4835 match(ConL);
4836
4837 op_cost(0);
4838 format %{ %}
4839 interface(CONST_INTER);
4840 %}
4841
4842 // 64 bit zero
4843 operand immL0()
4844 %{
4845 predicate(n->get_long() == 0);
4846 match(ConL);
4847
4848 op_cost(0);
4849 format %{ %}
4850 interface(CONST_INTER);
4851 %}
4852
4853 // 64 bit unit increment
4854 operand immL_1()
4855 %{
4856 predicate(n->get_long() == 1);
4857 match(ConL);
4858
4859 op_cost(0);
4860 format %{ %}
4861 interface(CONST_INTER);
4862 %}
4863
4864 // 64 bit unit decrement
4865 operand immL_M1()
4866 %{
4867 predicate(n->get_long() == -1);
4868 match(ConL);
4869
4870 op_cost(0);
4871 format %{ %}
4872 interface(CONST_INTER);
4873 %}
4874
4875 // 32 bit offset of pc in thread anchor
4876
4877 operand immL_pc_off()
4878 %{
4879 predicate(n->get_long() == in_bytes(JavaThread::frame_anchor_offset()) +
4880 in_bytes(JavaFrameAnchor::last_Java_pc_offset()));
4881 match(ConL);
4882
4883 op_cost(0);
4884 format %{ %}
4885 interface(CONST_INTER);
4886 %}
4887
4888 // 64 bit integer valid for add sub immediate
4889 operand immLAddSub()
4890 %{
4891 predicate(Assembler::operand_valid_for_add_sub_immediate(n->get_long()));
4892 match(ConL);
4893 op_cost(0);
4894 format %{ %}
4895 interface(CONST_INTER);
4896 %}
4897
4898 // 64 bit integer valid for addv subv immediate
4899 operand immLAddSubV()
4900 %{
4901 predicate(Assembler::operand_valid_for_sve_add_sub_immediate(n->get_long()));
4902 match(ConL);
4903
4904 op_cost(0);
4905 format %{ %}
4906 interface(CONST_INTER);
4907 %}
4908
4909 // 64 bit integer valid for logical immediate
4910 operand immLLog()
4911 %{
4912 predicate(Assembler::operand_valid_for_logical_immediate(/*is32*/false, (uint64_t)n->get_long()));
4913 match(ConL);
4914 op_cost(0);
4915 format %{ %}
4916 interface(CONST_INTER);
4917 %}
4918
4919 // Long Immediate: low 32-bit mask
4920 operand immL_32bits()
4921 %{
4922 predicate(n->get_long() == 0xFFFFFFFFL);
4923 match(ConL);
4924 op_cost(0);
4925 format %{ %}
4926 interface(CONST_INTER);
4927 %}
4928
4929 // Pointer operands
4930 // Pointer Immediate
4931 operand immP()
4932 %{
4933 match(ConP);
4934
4935 op_cost(0);
4936 format %{ %}
4937 interface(CONST_INTER);
4938 %}
4939
4940 // NULL Pointer Immediate
4941 operand immP0()
4942 %{
4943 predicate(n->get_ptr() == 0);
4944 match(ConP);
4945
4946 op_cost(0);
4947 format %{ %}
4948 interface(CONST_INTER);
4949 %}
4950
4951 // Pointer Immediate One
4952 // this is used in object initialization (initial object header)
4953 operand immP_1()
4954 %{
4955 predicate(n->get_ptr() == 1);
4956 match(ConP);
4957
4958 op_cost(0);
4959 format %{ %}
4960 interface(CONST_INTER);
4961 %}
4962
4963 // Card Table Byte Map Base
4964 operand immByteMapBase()
4965 %{
4966 // Get base of card map
4967 predicate(BarrierSet::barrier_set()->is_a(BarrierSet::CardTableBarrierSet) &&
4968 (CardTable::CardValue*)n->get_ptr() == ((CardTableBarrierSet*)(BarrierSet::barrier_set()))->card_table()->byte_map_base());
4969 match(ConP);
4970
4971 op_cost(0);
4972 format %{ %}
4973 interface(CONST_INTER);
4974 %}
4975
4976 // Pointer Immediate Minus One
4977 // this is used when we want to write the current PC to the thread anchor
4978 operand immP_M1()
4979 %{
4980 predicate(n->get_ptr() == -1);
4981 match(ConP);
4982
4983 op_cost(0);
4984 format %{ %}
4985 interface(CONST_INTER);
4986 %}
4987
4988 // Pointer Immediate Minus Two
4989 // this is used when we want to write the current PC to the thread anchor
4990 operand immP_M2()
4991 %{
4992 predicate(n->get_ptr() == -2);
4993 match(ConP);
4994
4995 op_cost(0);
4996 format %{ %}
4997 interface(CONST_INTER);
4998 %}
4999
5000 // Float and Double operands
5001 // Double Immediate
5002 operand immD()
5003 %{
5004 match(ConD);
5005 op_cost(0);
5006 format %{ %}
5007 interface(CONST_INTER);
5008 %}
5009
5010 // Double Immediate: +0.0d
5011 operand immD0()
5012 %{
5013 predicate(jlong_cast(n->getd()) == 0);
5014 match(ConD);
5015
5016 op_cost(0);
5017 format %{ %}
5018 interface(CONST_INTER);
5019 %}
5020
5021 // constant 'double +0.0'.
5022 operand immDPacked()
5023 %{
5024 predicate(Assembler::operand_valid_for_float_immediate(n->getd()));
5025 match(ConD);
5026 op_cost(0);
5027 format %{ %}
5028 interface(CONST_INTER);
5029 %}
5030
5031 // Float Immediate
5032 operand immF()
5033 %{
5034 match(ConF);
5035 op_cost(0);
5036 format %{ %}
5037 interface(CONST_INTER);
5038 %}
5039
5040 // Float Immediate: +0.0f.
5041 operand immF0()
5042 %{
5043 predicate(jint_cast(n->getf()) == 0);
5044 match(ConF);
5045
5046 op_cost(0);
5047 format %{ %}
5048 interface(CONST_INTER);
5049 %}
5050
5051 //
5052 operand immFPacked()
5053 %{
5054 predicate(Assembler::operand_valid_for_float_immediate((double)n->getf()));
5055 match(ConF);
5056 op_cost(0);
5057 format %{ %}
5058 interface(CONST_INTER);
5059 %}
5060
5061 // Narrow pointer operands
5062 // Narrow Pointer Immediate
5063 operand immN()
5064 %{
5065 match(ConN);
5066
5067 op_cost(0);
5068 format %{ %}
5069 interface(CONST_INTER);
5070 %}
5071
5072 // Narrow NULL Pointer Immediate
5073 operand immN0()
5074 %{
5075 predicate(n->get_narrowcon() == 0);
5076 match(ConN);
5077
5078 op_cost(0);
5079 format %{ %}
5080 interface(CONST_INTER);
5081 %}
5082
5083 operand immNKlass()
5084 %{
5085 match(ConNKlass);
5086
5087 op_cost(0);
5088 format %{ %}
5089 interface(CONST_INTER);
5090 %}
5091
5092 // Integer 32 bit Register Operands
5093 // Integer 32 bitRegister (excludes SP)
5094 operand iRegI()
5095 %{
5096 constraint(ALLOC_IN_RC(any_reg32));
5097 match(RegI);
5098 match(iRegINoSp);
5099 op_cost(0);
5100 format %{ %}
5101 interface(REG_INTER);
5102 %}
5103
5104 // Integer 32 bit Register not Special
5105 operand iRegINoSp()
5106 %{
5107 constraint(ALLOC_IN_RC(no_special_reg32));
5108 match(RegI);
5109 op_cost(0);
5110 format %{ %}
5111 interface(REG_INTER);
5112 %}
5113
5114 // Integer 64 bit Register Operands
5115 // Integer 64 bit Register (includes SP)
5116 operand iRegL()
5117 %{
5118 constraint(ALLOC_IN_RC(any_reg));
5119 match(RegL);
5120 match(iRegLNoSp);
5121 op_cost(0);
5122 format %{ %}
5123 interface(REG_INTER);
5124 %}
5125
5126 // Integer 64 bit Register not Special
5127 operand iRegLNoSp()
5128 %{
5129 constraint(ALLOC_IN_RC(no_special_reg));
5130 match(RegL);
5131 match(iRegL_R0);
5132 format %{ %}
5133 interface(REG_INTER);
5134 %}
5135
5136 // Pointer Register Operands
5137 // Pointer Register
5138 operand iRegP()
5139 %{
5140 constraint(ALLOC_IN_RC(ptr_reg));
5141 match(RegP);
5142 match(iRegPNoSp);
5143 match(iRegP_R0);
5144 //match(iRegP_R2);
5145 //match(iRegP_R4);
5146 //match(iRegP_R5);
5147 match(thread_RegP);
5148 op_cost(0);
5149 format %{ %}
5150 interface(REG_INTER);
5151 %}
5152
5153 // Pointer 64 bit Register not Special
5154 operand iRegPNoSp()
5155 %{
5156 constraint(ALLOC_IN_RC(no_special_ptr_reg));
5157 match(RegP);
5158 // match(iRegP);
5159 // match(iRegP_R0);
5160 // match(iRegP_R2);
5161 // match(iRegP_R4);
5162 // match(iRegP_R5);
5163 // match(thread_RegP);
5164 op_cost(0);
5165 format %{ %}
5166 interface(REG_INTER);
5167 %}
5168
5169 // Pointer 64 bit Register R0 only
5170 operand iRegP_R0()
5171 %{
5172 constraint(ALLOC_IN_RC(r0_reg));
5173 match(RegP);
5174 // match(iRegP);
5175 match(iRegPNoSp);
5176 op_cost(0);
5177 format %{ %}
5178 interface(REG_INTER);
5179 %}
5180
5181 // Pointer 64 bit Register R1 only
5182 operand iRegP_R1()
5183 %{
5184 constraint(ALLOC_IN_RC(r1_reg));
5185 match(RegP);
5186 // match(iRegP);
5187 match(iRegPNoSp);
5188 op_cost(0);
5189 format %{ %}
5190 interface(REG_INTER);
5191 %}
5192
5193 // Pointer 64 bit Register R2 only
5194 operand iRegP_R2()
5195 %{
5196 constraint(ALLOC_IN_RC(r2_reg));
5197 match(RegP);
5198 // match(iRegP);
5199 match(iRegPNoSp);
5200 op_cost(0);
5201 format %{ %}
5202 interface(REG_INTER);
5203 %}
5204
5205 // Pointer 64 bit Register R3 only
5206 operand iRegP_R3()
5207 %{
5208 constraint(ALLOC_IN_RC(r3_reg));
5209 match(RegP);
5210 // match(iRegP);
5211 match(iRegPNoSp);
5212 op_cost(0);
5213 format %{ %}
5214 interface(REG_INTER);
5215 %}
5216
5217 // Pointer 64 bit Register R4 only
5218 operand iRegP_R4()
5219 %{
5220 constraint(ALLOC_IN_RC(r4_reg));
5221 match(RegP);
5222 // match(iRegP);
5223 match(iRegPNoSp);
5224 op_cost(0);
5225 format %{ %}
5226 interface(REG_INTER);
5227 %}
5228
5229 // Pointer 64 bit Register R5 only
5230 operand iRegP_R5()
5231 %{
5232 constraint(ALLOC_IN_RC(r5_reg));
5233 match(RegP);
5234 // match(iRegP);
5235 match(iRegPNoSp);
5236 op_cost(0);
5237 format %{ %}
5238 interface(REG_INTER);
5239 %}
5240
5241 // Pointer 64 bit Register R10 only
5242 operand iRegP_R10()
5243 %{
5244 constraint(ALLOC_IN_RC(r10_reg));
5245 match(RegP);
5246 // match(iRegP);
5247 match(iRegPNoSp);
5248 op_cost(0);
5249 format %{ %}
5250 interface(REG_INTER);
5251 %}
5252
5253 // Long 64 bit Register R0 only
5254 operand iRegL_R0()
5255 %{
5256 constraint(ALLOC_IN_RC(r0_reg));
5257 match(RegL);
5258 match(iRegLNoSp);
5259 op_cost(0);
5260 format %{ %}
5261 interface(REG_INTER);
5262 %}
5263
5264 // Long 64 bit Register R2 only
5265 operand iRegL_R2()
5266 %{
5267 constraint(ALLOC_IN_RC(r2_reg));
5268 match(RegL);
5269 match(iRegLNoSp);
5270 op_cost(0);
5271 format %{ %}
5272 interface(REG_INTER);
5273 %}
5274
5275 // Long 64 bit Register R3 only
5276 operand iRegL_R3()
5277 %{
5278 constraint(ALLOC_IN_RC(r3_reg));
5279 match(RegL);
5280 match(iRegLNoSp);
5281 op_cost(0);
5282 format %{ %}
5283 interface(REG_INTER);
5284 %}
5285
5286 // Long 64 bit Register R11 only
5287 operand iRegL_R11()
5288 %{
5289 constraint(ALLOC_IN_RC(r11_reg));
5290 match(RegL);
5291 match(iRegLNoSp);
5292 op_cost(0);
5293 format %{ %}
5294 interface(REG_INTER);
5295 %}
5296
5297 // Pointer 64 bit Register FP only
5298 operand iRegP_FP()
5299 %{
5300 constraint(ALLOC_IN_RC(fp_reg));
5301 match(RegP);
5302 // match(iRegP);
5303 op_cost(0);
5304 format %{ %}
5305 interface(REG_INTER);
5306 %}
5307
5308 // Register R0 only
5309 operand iRegI_R0()
5310 %{
5311 constraint(ALLOC_IN_RC(int_r0_reg));
5312 match(RegI);
5313 match(iRegINoSp);
5314 op_cost(0);
5315 format %{ %}
5316 interface(REG_INTER);
5317 %}
5318
5319 // Register R2 only
5320 operand iRegI_R2()
5321 %{
5322 constraint(ALLOC_IN_RC(int_r2_reg));
5323 match(RegI);
5324 match(iRegINoSp);
5325 op_cost(0);
5326 format %{ %}
5327 interface(REG_INTER);
5328 %}
5329
5330 // Register R3 only
5331 operand iRegI_R3()
5332 %{
5333 constraint(ALLOC_IN_RC(int_r3_reg));
5334 match(RegI);
5335 match(iRegINoSp);
5336 op_cost(0);
5337 format %{ %}
5338 interface(REG_INTER);
5339 %}
5340
5341
5342 // Register R4 only
5343 operand iRegI_R4()
5344 %{
5345 constraint(ALLOC_IN_RC(int_r4_reg));
5346 match(RegI);
5347 match(iRegINoSp);
5348 op_cost(0);
5349 format %{ %}
5350 interface(REG_INTER);
5351 %}
5352
5353
5354 // Pointer Register Operands
5355 // Narrow Pointer Register
5356 operand iRegN()
5357 %{
5358 constraint(ALLOC_IN_RC(any_reg32));
5359 match(RegN);
5360 match(iRegNNoSp);
5361 op_cost(0);
5362 format %{ %}
5363 interface(REG_INTER);
5364 %}
5365
5366 operand iRegN_R0()
5367 %{
5368 constraint(ALLOC_IN_RC(r0_reg));
5369 match(iRegN);
5370 op_cost(0);
5371 format %{ %}
5372 interface(REG_INTER);
5373 %}
5374
5375 operand iRegN_R2()
5376 %{
5377 constraint(ALLOC_IN_RC(r2_reg));
5378 match(iRegN);
5379 op_cost(0);
5380 format %{ %}
5381 interface(REG_INTER);
5382 %}
5383
5384 operand iRegN_R3()
5385 %{
5386 constraint(ALLOC_IN_RC(r3_reg));
5387 match(iRegN);
5388 op_cost(0);
5389 format %{ %}
5390 interface(REG_INTER);
5391 %}
5392
5393 // Integer 64 bit Register not Special
5394 operand iRegNNoSp()
5395 %{
5396 constraint(ALLOC_IN_RC(no_special_reg32));
5397 match(RegN);
5398 op_cost(0);
5399 format %{ %}
5400 interface(REG_INTER);
5401 %}
5402
5403 // heap base register -- used for encoding immN0
5404
5405 operand iRegIHeapbase()
5406 %{
5407 constraint(ALLOC_IN_RC(heapbase_reg));
5408 match(RegI);
5409 op_cost(0);
5410 format %{ %}
5411 interface(REG_INTER);
5412 %}
5413
5414 // Float Register
5415 // Float register operands
5416 operand vRegF()
5417 %{
5418 constraint(ALLOC_IN_RC(float_reg));
5419 match(RegF);
5420
5421 op_cost(0);
5422 format %{ %}
5423 interface(REG_INTER);
5424 %}
5425
5426 // Double Register
5427 // Double register operands
5428 operand vRegD()
5429 %{
5430 constraint(ALLOC_IN_RC(double_reg));
5431 match(RegD);
5432
5433 op_cost(0);
5434 format %{ %}
5435 interface(REG_INTER);
5436 %}
5437
5438 // Generic vector class. This will be used for
5439 // all vector operands, including NEON and SVE,
5440 // but currently only used for SVE VecA.
5441 operand vReg()
5442 %{
5443 constraint(ALLOC_IN_RC(vectora_reg));
5444 match(VecA);
5445 op_cost(0);
5446 format %{ %}
5447 interface(REG_INTER);
5448 %}
5449
5450 operand vecD()
5451 %{
5452 constraint(ALLOC_IN_RC(vectord_reg));
5453 match(VecD);
5454
5455 op_cost(0);
5456 format %{ %}
5457 interface(REG_INTER);
5458 %}
5459
5460 operand vecX()
5461 %{
5462 constraint(ALLOC_IN_RC(vectorx_reg));
5463 match(VecX);
5464
5465 op_cost(0);
5466 format %{ %}
5467 interface(REG_INTER);
5468 %}
5469
5470 operand vRegD_V0()
5471 %{
5472 constraint(ALLOC_IN_RC(v0_reg));
5473 match(RegD);
5474 op_cost(0);
5475 format %{ %}
5476 interface(REG_INTER);
5477 %}
5478
5479 operand vRegD_V1()
5480 %{
5481 constraint(ALLOC_IN_RC(v1_reg));
5482 match(RegD);
5483 op_cost(0);
5484 format %{ %}
5485 interface(REG_INTER);
5486 %}
5487
5488 operand vRegD_V2()
5489 %{
5490 constraint(ALLOC_IN_RC(v2_reg));
5491 match(RegD);
5492 op_cost(0);
5493 format %{ %}
5494 interface(REG_INTER);
5495 %}
5496
5497 operand vRegD_V3()
5498 %{
5499 constraint(ALLOC_IN_RC(v3_reg));
5500 match(RegD);
5501 op_cost(0);
5502 format %{ %}
5503 interface(REG_INTER);
5504 %}
5505
5506 operand vRegD_V4()
5507 %{
5508 constraint(ALLOC_IN_RC(v4_reg));
5509 match(RegD);
5510 op_cost(0);
5511 format %{ %}
5512 interface(REG_INTER);
5513 %}
5514
5515 operand vRegD_V5()
5516 %{
5517 constraint(ALLOC_IN_RC(v5_reg));
5518 match(RegD);
5519 op_cost(0);
5520 format %{ %}
5521 interface(REG_INTER);
5522 %}
5523
5524 operand vRegD_V6()
5525 %{
5526 constraint(ALLOC_IN_RC(v6_reg));
5527 match(RegD);
5528 op_cost(0);
5529 format %{ %}
5530 interface(REG_INTER);
5531 %}
5532
5533 operand vRegD_V7()
5534 %{
5535 constraint(ALLOC_IN_RC(v7_reg));
5536 match(RegD);
5537 op_cost(0);
5538 format %{ %}
5539 interface(REG_INTER);
5540 %}
5541
5542 operand vRegD_V8()
5543 %{
5544 constraint(ALLOC_IN_RC(v8_reg));
5545 match(RegD);
5546 op_cost(0);
5547 format %{ %}
5548 interface(REG_INTER);
5549 %}
5550
5551 operand vRegD_V9()
5552 %{
5553 constraint(ALLOC_IN_RC(v9_reg));
5554 match(RegD);
5555 op_cost(0);
5556 format %{ %}
5557 interface(REG_INTER);
5558 %}
5559
5560 operand vRegD_V10()
5561 %{
5562 constraint(ALLOC_IN_RC(v10_reg));
5563 match(RegD);
5564 op_cost(0);
5565 format %{ %}
5566 interface(REG_INTER);
5567 %}
5568
5569 operand vRegD_V11()
5570 %{
5571 constraint(ALLOC_IN_RC(v11_reg));
5572 match(RegD);
5573 op_cost(0);
5574 format %{ %}
5575 interface(REG_INTER);
5576 %}
5577
5578 operand vRegD_V12()
5579 %{
5580 constraint(ALLOC_IN_RC(v12_reg));
5581 match(RegD);
5582 op_cost(0);
5583 format %{ %}
5584 interface(REG_INTER);
5585 %}
5586
5587 operand vRegD_V13()
5588 %{
5589 constraint(ALLOC_IN_RC(v13_reg));
5590 match(RegD);
5591 op_cost(0);
5592 format %{ %}
5593 interface(REG_INTER);
5594 %}
5595
5596 operand vRegD_V14()
5597 %{
5598 constraint(ALLOC_IN_RC(v14_reg));
5599 match(RegD);
5600 op_cost(0);
5601 format %{ %}
5602 interface(REG_INTER);
5603 %}
5604
5605 operand vRegD_V15()
5606 %{
5607 constraint(ALLOC_IN_RC(v15_reg));
5608 match(RegD);
5609 op_cost(0);
5610 format %{ %}
5611 interface(REG_INTER);
5612 %}
5613
5614 operand vRegD_V16()
5615 %{
5616 constraint(ALLOC_IN_RC(v16_reg));
5617 match(RegD);
5618 op_cost(0);
5619 format %{ %}
5620 interface(REG_INTER);
5621 %}
5622
5623 operand vRegD_V17()
5624 %{
5625 constraint(ALLOC_IN_RC(v17_reg));
5626 match(RegD);
5627 op_cost(0);
5628 format %{ %}
5629 interface(REG_INTER);
5630 %}
5631
5632 operand vRegD_V18()
5633 %{
5634 constraint(ALLOC_IN_RC(v18_reg));
5635 match(RegD);
5636 op_cost(0);
5637 format %{ %}
5638 interface(REG_INTER);
5639 %}
5640
5641 operand vRegD_V19()
5642 %{
5643 constraint(ALLOC_IN_RC(v19_reg));
5644 match(RegD);
5645 op_cost(0);
5646 format %{ %}
5647 interface(REG_INTER);
5648 %}
5649
5650 operand vRegD_V20()
5651 %{
5652 constraint(ALLOC_IN_RC(v20_reg));
5653 match(RegD);
5654 op_cost(0);
5655 format %{ %}
5656 interface(REG_INTER);
5657 %}
5658
5659 operand vRegD_V21()
5660 %{
5661 constraint(ALLOC_IN_RC(v21_reg));
5662 match(RegD);
5663 op_cost(0);
5664 format %{ %}
5665 interface(REG_INTER);
5666 %}
5667
5668 operand vRegD_V22()
5669 %{
5670 constraint(ALLOC_IN_RC(v22_reg));
5671 match(RegD);
5672 op_cost(0);
5673 format %{ %}
5674 interface(REG_INTER);
5675 %}
5676
5677 operand vRegD_V23()
5678 %{
5679 constraint(ALLOC_IN_RC(v23_reg));
5680 match(RegD);
5681 op_cost(0);
5682 format %{ %}
5683 interface(REG_INTER);
5684 %}
5685
5686 operand vRegD_V24()
5687 %{
5688 constraint(ALLOC_IN_RC(v24_reg));
5689 match(RegD);
5690 op_cost(0);
5691 format %{ %}
5692 interface(REG_INTER);
5693 %}
5694
5695 operand vRegD_V25()
5696 %{
5697 constraint(ALLOC_IN_RC(v25_reg));
5698 match(RegD);
5699 op_cost(0);
5700 format %{ %}
5701 interface(REG_INTER);
5702 %}
5703
5704 operand vRegD_V26()
5705 %{
5706 constraint(ALLOC_IN_RC(v26_reg));
5707 match(RegD);
5708 op_cost(0);
5709 format %{ %}
5710 interface(REG_INTER);
5711 %}
5712
5713 operand vRegD_V27()
5714 %{
5715 constraint(ALLOC_IN_RC(v27_reg));
5716 match(RegD);
5717 op_cost(0);
5718 format %{ %}
5719 interface(REG_INTER);
5720 %}
5721
5722 operand vRegD_V28()
5723 %{
5724 constraint(ALLOC_IN_RC(v28_reg));
5725 match(RegD);
5726 op_cost(0);
5727 format %{ %}
5728 interface(REG_INTER);
5729 %}
5730
5731 operand vRegD_V29()
5732 %{
5733 constraint(ALLOC_IN_RC(v29_reg));
5734 match(RegD);
5735 op_cost(0);
5736 format %{ %}
5737 interface(REG_INTER);
5738 %}
5739
5740 operand vRegD_V30()
5741 %{
5742 constraint(ALLOC_IN_RC(v30_reg));
5743 match(RegD);
5744 op_cost(0);
5745 format %{ %}
5746 interface(REG_INTER);
5747 %}
5748
5749 operand vRegD_V31()
5750 %{
5751 constraint(ALLOC_IN_RC(v31_reg));
5752 match(RegD);
5753 op_cost(0);
5754 format %{ %}
5755 interface(REG_INTER);
5756 %}
5757
5758 operand pReg()
5759 %{
5760 constraint(ALLOC_IN_RC(pr_reg));
5761 match(RegVectMask);
5762 match(pRegGov);
5763 op_cost(0);
5764 format %{ %}
5765 interface(REG_INTER);
5766 %}
5767
5768 operand pRegGov()
5769 %{
5770 constraint(ALLOC_IN_RC(gov_pr));
5771 match(RegVectMask);
5772 op_cost(0);
5773 format %{ %}
5774 interface(REG_INTER);
5775 %}
5776
5777 // Flags register, used as output of signed compare instructions
5778
5779 // note that on AArch64 we also use this register as the output for
5780 // for floating point compare instructions (CmpF CmpD). this ensures
5781 // that ordered inequality tests use GT, GE, LT or LE none of which
5782 // pass through cases where the result is unordered i.e. one or both
5783 // inputs to the compare is a NaN. this means that the ideal code can
5784 // replace e.g. a GT with an LE and not end up capturing the NaN case
5785 // (where the comparison should always fail). EQ and NE tests are
5786 // always generated in ideal code so that unordered folds into the NE
5787 // case, matching the behaviour of AArch64 NE.
5788 //
5789 // This differs from x86 where the outputs of FP compares use a
5790 // special FP flags registers and where compares based on this
5791 // register are distinguished into ordered inequalities (cmpOpUCF) and
5792 // EQ/NEQ tests (cmpOpUCF2). x86 has to special case the latter tests
5793 // to explicitly handle the unordered case in branches. x86 also has
5794 // to include extra CMoveX rules to accept a cmpOpUCF input.
5795
5796 operand rFlagsReg()
5797 %{
5798 constraint(ALLOC_IN_RC(int_flags));
5799 match(RegFlags);
5800
5801 op_cost(0);
5802 format %{ "RFLAGS" %}
5803 interface(REG_INTER);
5804 %}
5805
5806 // Flags register, used as output of unsigned compare instructions
5807 operand rFlagsRegU()
5808 %{
5809 constraint(ALLOC_IN_RC(int_flags));
5810 match(RegFlags);
5811
5812 op_cost(0);
5813 format %{ "RFLAGSU" %}
5814 interface(REG_INTER);
5815 %}
5816
5817 // Special Registers
5818
5819 // Method Register
5820 operand inline_cache_RegP(iRegP reg)
5821 %{
5822 constraint(ALLOC_IN_RC(method_reg)); // inline_cache_reg
5823 match(reg);
5824 match(iRegPNoSp);
5825 op_cost(0);
5826 format %{ %}
5827 interface(REG_INTER);
5828 %}
5829
5830 // Thread Register
5831 operand thread_RegP(iRegP reg)
5832 %{
5833 constraint(ALLOC_IN_RC(thread_reg)); // link_reg
5834 match(reg);
5835 op_cost(0);
5836 format %{ %}
5837 interface(REG_INTER);
5838 %}
5839
5840 operand lr_RegP(iRegP reg)
5841 %{
5842 constraint(ALLOC_IN_RC(lr_reg)); // link_reg
5843 match(reg);
5844 op_cost(0);
5845 format %{ %}
5846 interface(REG_INTER);
5847 %}
5848
5849 //----------Memory Operands----------------------------------------------------
5850
5851 operand indirect(iRegP reg)
5852 %{
5853 constraint(ALLOC_IN_RC(ptr_reg));
5854 match(reg);
5855 op_cost(0);
5856 format %{ "[$reg]" %}
5857 interface(MEMORY_INTER) %{
5858 base($reg);
5859 index(0xffffffff);
5860 scale(0x0);
5861 disp(0x0);
5862 %}
5863 %}
5864
5865 operand indIndexScaledI2L(iRegP reg, iRegI ireg, immIScale scale)
5866 %{
5867 constraint(ALLOC_IN_RC(ptr_reg));
5868 predicate(size_fits_all_mem_uses(n->as_AddP(), n->in(AddPNode::Offset)->in(2)->get_int()));
5869 match(AddP reg (LShiftL (ConvI2L ireg) scale));
5870 op_cost(0);
5871 format %{ "$reg, $ireg sxtw($scale), 0, I2L" %}
5872 interface(MEMORY_INTER) %{
5873 base($reg);
5874 index($ireg);
5875 scale($scale);
5876 disp(0x0);
5877 %}
5878 %}
5879
5880 operand indIndexScaled(iRegP reg, iRegL lreg, immIScale scale)
5881 %{
5882 constraint(ALLOC_IN_RC(ptr_reg));
5883 predicate(size_fits_all_mem_uses(n->as_AddP(), n->in(AddPNode::Offset)->in(2)->get_int()));
5884 match(AddP reg (LShiftL lreg scale));
5885 op_cost(0);
5886 format %{ "$reg, $lreg lsl($scale)" %}
5887 interface(MEMORY_INTER) %{
5888 base($reg);
5889 index($lreg);
5890 scale($scale);
5891 disp(0x0);
5892 %}
5893 %}
5894
5895 operand indIndexI2L(iRegP reg, iRegI ireg)
5896 %{
5897 constraint(ALLOC_IN_RC(ptr_reg));
5898 match(AddP reg (ConvI2L ireg));
5899 op_cost(0);
5900 format %{ "$reg, $ireg, 0, I2L" %}
5901 interface(MEMORY_INTER) %{
5902 base($reg);
5903 index($ireg);
5904 scale(0x0);
5905 disp(0x0);
5906 %}
5907 %}
5908
5909 operand indIndex(iRegP reg, iRegL lreg)
5910 %{
5911 constraint(ALLOC_IN_RC(ptr_reg));
5912 match(AddP reg lreg);
5913 op_cost(0);
5914 format %{ "$reg, $lreg" %}
5915 interface(MEMORY_INTER) %{
5916 base($reg);
5917 index($lreg);
5918 scale(0x0);
5919 disp(0x0);
5920 %}
5921 %}
5922
5923 operand indOffI(iRegP reg, immIOffset off)
5924 %{
5925 constraint(ALLOC_IN_RC(ptr_reg));
5926 match(AddP reg off);
5927 op_cost(0);
5928 format %{ "[$reg, $off]" %}
5929 interface(MEMORY_INTER) %{
5930 base($reg);
5931 index(0xffffffff);
5932 scale(0x0);
5933 disp($off);
5934 %}
5935 %}
5936
5937 operand indOffI1(iRegP reg, immIOffset1 off)
5938 %{
5939 constraint(ALLOC_IN_RC(ptr_reg));
5940 match(AddP reg off);
5941 op_cost(0);
5942 format %{ "[$reg, $off]" %}
5943 interface(MEMORY_INTER) %{
5944 base($reg);
5945 index(0xffffffff);
5946 scale(0x0);
5947 disp($off);
5948 %}
5949 %}
5950
5951 operand indOffI2(iRegP reg, immIOffset2 off)
5952 %{
5953 constraint(ALLOC_IN_RC(ptr_reg));
5954 match(AddP reg off);
5955 op_cost(0);
5956 format %{ "[$reg, $off]" %}
5957 interface(MEMORY_INTER) %{
5958 base($reg);
5959 index(0xffffffff);
5960 scale(0x0);
5961 disp($off);
5962 %}
5963 %}
5964
5965 operand indOffI4(iRegP reg, immIOffset4 off)
5966 %{
5967 constraint(ALLOC_IN_RC(ptr_reg));
5968 match(AddP reg off);
5969 op_cost(0);
5970 format %{ "[$reg, $off]" %}
5971 interface(MEMORY_INTER) %{
5972 base($reg);
5973 index(0xffffffff);
5974 scale(0x0);
5975 disp($off);
5976 %}
5977 %}
5978
5979 operand indOffI8(iRegP reg, immIOffset8 off)
5980 %{
5981 constraint(ALLOC_IN_RC(ptr_reg));
5982 match(AddP reg off);
5983 op_cost(0);
5984 format %{ "[$reg, $off]" %}
5985 interface(MEMORY_INTER) %{
5986 base($reg);
5987 index(0xffffffff);
5988 scale(0x0);
5989 disp($off);
5990 %}
5991 %}
5992
5993 operand indOffI16(iRegP reg, immIOffset16 off)
5994 %{
5995 constraint(ALLOC_IN_RC(ptr_reg));
5996 match(AddP reg off);
5997 op_cost(0);
5998 format %{ "[$reg, $off]" %}
5999 interface(MEMORY_INTER) %{
6000 base($reg);
6001 index(0xffffffff);
6002 scale(0x0);
6003 disp($off);
6004 %}
6005 %}
6006
6007 operand indOffL(iRegP reg, immLoffset off)
6008 %{
6009 constraint(ALLOC_IN_RC(ptr_reg));
6010 match(AddP reg off);
6011 op_cost(0);
6012 format %{ "[$reg, $off]" %}
6013 interface(MEMORY_INTER) %{
6014 base($reg);
6015 index(0xffffffff);
6016 scale(0x0);
6017 disp($off);
6018 %}
6019 %}
6020
6021 operand indOffL1(iRegP reg, immLoffset1 off)
6022 %{
6023 constraint(ALLOC_IN_RC(ptr_reg));
6024 match(AddP reg off);
6025 op_cost(0);
6026 format %{ "[$reg, $off]" %}
6027 interface(MEMORY_INTER) %{
6028 base($reg);
6029 index(0xffffffff);
6030 scale(0x0);
6031 disp($off);
6032 %}
6033 %}
6034
6035 operand indOffL2(iRegP reg, immLoffset2 off)
6036 %{
6037 constraint(ALLOC_IN_RC(ptr_reg));
6038 match(AddP reg off);
6039 op_cost(0);
6040 format %{ "[$reg, $off]" %}
6041 interface(MEMORY_INTER) %{
6042 base($reg);
6043 index(0xffffffff);
6044 scale(0x0);
6045 disp($off);
6046 %}
6047 %}
6048
6049 operand indOffL4(iRegP reg, immLoffset4 off)
6050 %{
6051 constraint(ALLOC_IN_RC(ptr_reg));
6052 match(AddP reg off);
6053 op_cost(0);
6054 format %{ "[$reg, $off]" %}
6055 interface(MEMORY_INTER) %{
6056 base($reg);
6057 index(0xffffffff);
6058 scale(0x0);
6059 disp($off);
6060 %}
6061 %}
6062
6063 operand indOffL8(iRegP reg, immLoffset8 off)
6064 %{
6065 constraint(ALLOC_IN_RC(ptr_reg));
6066 match(AddP reg off);
6067 op_cost(0);
6068 format %{ "[$reg, $off]" %}
6069 interface(MEMORY_INTER) %{
6070 base($reg);
6071 index(0xffffffff);
6072 scale(0x0);
6073 disp($off);
6074 %}
6075 %}
6076
6077 operand indOffL16(iRegP reg, immLoffset16 off)
6078 %{
6079 constraint(ALLOC_IN_RC(ptr_reg));
6080 match(AddP reg off);
6081 op_cost(0);
6082 format %{ "[$reg, $off]" %}
6083 interface(MEMORY_INTER) %{
6084 base($reg);
6085 index(0xffffffff);
6086 scale(0x0);
6087 disp($off);
6088 %}
6089 %}
6090
6091 operand indirectN(iRegN reg)
6092 %{
6093 predicate(CompressedOops::shift() == 0);
6094 constraint(ALLOC_IN_RC(ptr_reg));
6095 match(DecodeN reg);
6096 op_cost(0);
6097 format %{ "[$reg]\t# narrow" %}
6098 interface(MEMORY_INTER) %{
6099 base($reg);
6100 index(0xffffffff);
6101 scale(0x0);
6102 disp(0x0);
6103 %}
6104 %}
6105
6106 operand indIndexScaledI2LN(iRegN reg, iRegI ireg, immIScale scale)
6107 %{
6108 predicate(CompressedOops::shift() == 0 && size_fits_all_mem_uses(n->as_AddP(), n->in(AddPNode::Offset)->in(2)->get_int()));
6109 constraint(ALLOC_IN_RC(ptr_reg));
6110 match(AddP (DecodeN reg) (LShiftL (ConvI2L ireg) scale));
6111 op_cost(0);
6112 format %{ "$reg, $ireg sxtw($scale), 0, I2L\t# narrow" %}
6113 interface(MEMORY_INTER) %{
6114 base($reg);
6115 index($ireg);
6116 scale($scale);
6117 disp(0x0);
6118 %}
6119 %}
6120
6121 operand indIndexScaledN(iRegN reg, iRegL lreg, immIScale scale)
6122 %{
6123 predicate(CompressedOops::shift() == 0 && size_fits_all_mem_uses(n->as_AddP(), n->in(AddPNode::Offset)->in(2)->get_int()));
6124 constraint(ALLOC_IN_RC(ptr_reg));
6125 match(AddP (DecodeN reg) (LShiftL lreg scale));
6126 op_cost(0);
6127 format %{ "$reg, $lreg lsl($scale)\t# narrow" %}
6128 interface(MEMORY_INTER) %{
6129 base($reg);
6130 index($lreg);
6131 scale($scale);
6132 disp(0x0);
6133 %}
6134 %}
6135
6136 operand indIndexI2LN(iRegN reg, iRegI ireg)
6137 %{
6138 predicate(CompressedOops::shift() == 0);
6139 constraint(ALLOC_IN_RC(ptr_reg));
6140 match(AddP (DecodeN reg) (ConvI2L ireg));
6141 op_cost(0);
6142 format %{ "$reg, $ireg, 0, I2L\t# narrow" %}
6143 interface(MEMORY_INTER) %{
6144 base($reg);
6145 index($ireg);
6146 scale(0x0);
6147 disp(0x0);
6148 %}
6149 %}
6150
6151 operand indIndexN(iRegN reg, iRegL lreg)
6152 %{
6153 predicate(CompressedOops::shift() == 0);
6154 constraint(ALLOC_IN_RC(ptr_reg));
6155 match(AddP (DecodeN reg) lreg);
6156 op_cost(0);
6157 format %{ "$reg, $lreg\t# narrow" %}
6158 interface(MEMORY_INTER) %{
6159 base($reg);
6160 index($lreg);
6161 scale(0x0);
6162 disp(0x0);
6163 %}
6164 %}
6165
6166 operand indOffIN(iRegN reg, immIOffset off)
6167 %{
6168 predicate(CompressedOops::shift() == 0);
6169 constraint(ALLOC_IN_RC(ptr_reg));
6170 match(AddP (DecodeN reg) off);
6171 op_cost(0);
6172 format %{ "[$reg, $off]\t# narrow" %}
6173 interface(MEMORY_INTER) %{
6174 base($reg);
6175 index(0xffffffff);
6176 scale(0x0);
6177 disp($off);
6178 %}
6179 %}
6180
6181 operand indOffLN(iRegN reg, immLoffset off)
6182 %{
6183 predicate(CompressedOops::shift() == 0);
6184 constraint(ALLOC_IN_RC(ptr_reg));
6185 match(AddP (DecodeN reg) off);
6186 op_cost(0);
6187 format %{ "[$reg, $off]\t# narrow" %}
6188 interface(MEMORY_INTER) %{
6189 base($reg);
6190 index(0xffffffff);
6191 scale(0x0);
6192 disp($off);
6193 %}
6194 %}
6195
6196
6197
6198 // AArch64 opto stubs need to write to the pc slot in the thread anchor
6199 operand thread_anchor_pc(thread_RegP reg, immL_pc_off off)
6200 %{
6201 constraint(ALLOC_IN_RC(ptr_reg));
6202 match(AddP reg off);
6203 op_cost(0);
6204 format %{ "[$reg, $off]" %}
6205 interface(MEMORY_INTER) %{
6206 base($reg);
6207 index(0xffffffff);
6208 scale(0x0);
6209 disp($off);
6210 %}
6211 %}
6212
6213 //----------Special Memory Operands--------------------------------------------
6214 // Stack Slot Operand - This operand is used for loading and storing temporary
6215 // values on the stack where a match requires a value to
6216 // flow through memory.
6217 operand stackSlotP(sRegP reg)
6218 %{
6219 constraint(ALLOC_IN_RC(stack_slots));
6220 op_cost(100);
6221 // No match rule because this operand is only generated in matching
6222 // match(RegP);
6223 format %{ "[$reg]" %}
6224 interface(MEMORY_INTER) %{
6225 base(0x1e); // RSP
6226 index(0x0); // No Index
6227 scale(0x0); // No Scale
6228 disp($reg); // Stack Offset
6229 %}
6230 %}
6231
6232 operand stackSlotI(sRegI reg)
6233 %{
6234 constraint(ALLOC_IN_RC(stack_slots));
6235 // No match rule because this operand is only generated in matching
6236 // match(RegI);
6237 format %{ "[$reg]" %}
6238 interface(MEMORY_INTER) %{
6239 base(0x1e); // RSP
6240 index(0x0); // No Index
6241 scale(0x0); // No Scale
6242 disp($reg); // Stack Offset
6243 %}
6244 %}
6245
6246 operand stackSlotF(sRegF reg)
6247 %{
6248 constraint(ALLOC_IN_RC(stack_slots));
6249 // No match rule because this operand is only generated in matching
6250 // match(RegF);
6251 format %{ "[$reg]" %}
6252 interface(MEMORY_INTER) %{
6253 base(0x1e); // RSP
6254 index(0x0); // No Index
6255 scale(0x0); // No Scale
6256 disp($reg); // Stack Offset
6257 %}
6258 %}
6259
6260 operand stackSlotD(sRegD reg)
6261 %{
6262 constraint(ALLOC_IN_RC(stack_slots));
6263 // No match rule because this operand is only generated in matching
6264 // match(RegD);
6265 format %{ "[$reg]" %}
6266 interface(MEMORY_INTER) %{
6267 base(0x1e); // RSP
6268 index(0x0); // No Index
6269 scale(0x0); // No Scale
6270 disp($reg); // Stack Offset
6271 %}
6272 %}
6273
6274 operand stackSlotL(sRegL reg)
6275 %{
6276 constraint(ALLOC_IN_RC(stack_slots));
6277 // No match rule because this operand is only generated in matching
6278 // match(RegL);
6279 format %{ "[$reg]" %}
6280 interface(MEMORY_INTER) %{
6281 base(0x1e); // RSP
6282 index(0x0); // No Index
6283 scale(0x0); // No Scale
6284 disp($reg); // Stack Offset
6285 %}
6286 %}
6287
6288 // Operands for expressing Control Flow
6289 // NOTE: Label is a predefined operand which should not be redefined in
6290 // the AD file. It is generically handled within the ADLC.
6291
6292 //----------Conditional Branch Operands----------------------------------------
6293 // Comparison Op - This is the operation of the comparison, and is limited to
6294 // the following set of codes:
6295 // L (<), LE (<=), G (>), GE (>=), E (==), NE (!=)
6296 //
6297 // Other attributes of the comparison, such as unsignedness, are specified
6298 // by the comparison instruction that sets a condition code flags register.
6299 // That result is represented by a flags operand whose subtype is appropriate
6300 // to the unsignedness (etc.) of the comparison.
6301 //
6302 // Later, the instruction which matches both the Comparison Op (a Bool) and
6303 // the flags (produced by the Cmp) specifies the coding of the comparison op
6304 // by matching a specific subtype of Bool operand below, such as cmpOpU.
6305
6306 // used for signed integral comparisons and fp comparisons
6307
6308 operand cmpOp()
6309 %{
6310 match(Bool);
6311
6312 format %{ "" %}
6313 interface(COND_INTER) %{
6314 equal(0x0, "eq");
6315 not_equal(0x1, "ne");
6316 less(0xb, "lt");
6317 greater_equal(0xa, "ge");
6318 less_equal(0xd, "le");
6319 greater(0xc, "gt");
6320 overflow(0x6, "vs");
6321 no_overflow(0x7, "vc");
6322 %}
6323 %}
6324
6325 // used for unsigned integral comparisons
6326
6327 operand cmpOpU()
6328 %{
6329 match(Bool);
6330
6331 format %{ "" %}
6332 interface(COND_INTER) %{
6333 equal(0x0, "eq");
6334 not_equal(0x1, "ne");
6335 less(0x3, "lo");
6336 greater_equal(0x2, "hs");
6337 less_equal(0x9, "ls");
6338 greater(0x8, "hi");
6339 overflow(0x6, "vs");
6340 no_overflow(0x7, "vc");
6341 %}
6342 %}
6343
6344 // used for certain integral comparisons which can be
6345 // converted to cbxx or tbxx instructions
6346
6347 operand cmpOpEqNe()
6348 %{
6349 match(Bool);
6350 op_cost(0);
6351 predicate(n->as_Bool()->_test._test == BoolTest::ne
6352 || n->as_Bool()->_test._test == BoolTest::eq);
6353
6354 format %{ "" %}
6355 interface(COND_INTER) %{
6356 equal(0x0, "eq");
6357 not_equal(0x1, "ne");
6358 less(0xb, "lt");
6359 greater_equal(0xa, "ge");
6360 less_equal(0xd, "le");
6361 greater(0xc, "gt");
6362 overflow(0x6, "vs");
6363 no_overflow(0x7, "vc");
6364 %}
6365 %}
6366
6367 // used for certain integral comparisons which can be
6368 // converted to cbxx or tbxx instructions
6369
6370 operand cmpOpLtGe()
6371 %{
6372 match(Bool);
6373 op_cost(0);
6374
6375 predicate(n->as_Bool()->_test._test == BoolTest::lt
6376 || n->as_Bool()->_test._test == BoolTest::ge);
6377
6378 format %{ "" %}
6379 interface(COND_INTER) %{
6380 equal(0x0, "eq");
6381 not_equal(0x1, "ne");
6382 less(0xb, "lt");
6383 greater_equal(0xa, "ge");
6384 less_equal(0xd, "le");
6385 greater(0xc, "gt");
6386 overflow(0x6, "vs");
6387 no_overflow(0x7, "vc");
6388 %}
6389 %}
6390
6391 // used for certain unsigned integral comparisons which can be
6392 // converted to cbxx or tbxx instructions
6393
6394 operand cmpOpUEqNeLtGe()
6395 %{
6396 match(Bool);
6397 op_cost(0);
6398
6399 predicate(n->as_Bool()->_test._test == BoolTest::eq
6400 || n->as_Bool()->_test._test == BoolTest::ne
6401 || n->as_Bool()->_test._test == BoolTest::lt
6402 || n->as_Bool()->_test._test == BoolTest::ge);
6403
6404 format %{ "" %}
6405 interface(COND_INTER) %{
6406 equal(0x0, "eq");
6407 not_equal(0x1, "ne");
6408 less(0xb, "lt");
6409 greater_equal(0xa, "ge");
6410 less_equal(0xd, "le");
6411 greater(0xc, "gt");
6412 overflow(0x6, "vs");
6413 no_overflow(0x7, "vc");
6414 %}
6415 %}
6416
6417 // Special operand allowing long args to int ops to be truncated for free
6418
6419 operand iRegL2I(iRegL reg) %{
6420
6421 op_cost(0);
6422
6423 match(ConvL2I reg);
6424
6425 format %{ "l2i($reg)" %}
6426
6427 interface(REG_INTER)
6428 %}
6429
6430 opclass vmem2(indirect, indIndex, indOffI2, indOffL2);
6431 opclass vmem4(indirect, indIndex, indOffI4, indOffL4);
6432 opclass vmem8(indirect, indIndex, indOffI8, indOffL8);
6433 opclass vmem16(indirect, indIndex, indOffI16, indOffL16);
6434
6435 //----------OPERAND CLASSES----------------------------------------------------
6436 // Operand Classes are groups of operands that are used as to simplify
6437 // instruction definitions by not requiring the AD writer to specify
6438 // separate instructions for every form of operand when the
6439 // instruction accepts multiple operand types with the same basic
6440 // encoding and format. The classic case of this is memory operands.
6441
6442 // memory is used to define read/write location for load/store
6443 // instruction defs. we can turn a memory op into an Address
6444
6445 opclass memory1(indirect, indIndexScaled, indIndexScaledI2L, indIndexI2L, indIndex, indOffI1, indOffL1,
6446 indirectN, indIndexScaledN, indIndexScaledI2LN, indIndexI2LN, indIndexN);
6447
6448 opclass memory2(indirect, indIndexScaled, indIndexScaledI2L, indIndexI2L, indIndex, indOffI2, indOffL2,
6449 indirectN, indIndexScaledN, indIndexScaledI2LN, indIndexI2LN, indIndexN);
6450
6451 opclass memory4(indirect, indIndexScaled, indIndexScaledI2L, indIndexI2L, indIndex, indOffI4, indOffL4,
6452 indirectN, indIndexScaledN, indIndexScaledI2LN, indIndexI2LN, indIndexN, indOffIN, indOffLN);
6453
6454 opclass memory8(indirect, indIndexScaled, indIndexScaledI2L, indIndexI2L, indIndex, indOffI8, indOffL8,
6455 indirectN, indIndexScaledN, indIndexScaledI2LN, indIndexI2LN, indIndexN, indOffIN, indOffLN);
6456
6457 // All of the memory operands. For the pipeline description.
6458 opclass memory(indirect, indIndexScaled, indIndexScaledI2L, indIndexI2L, indIndex,
6459 indOffI1, indOffL1, indOffI2, indOffL2, indOffI4, indOffL4, indOffI8, indOffL8,
6460 indirectN, indIndexScaledN, indIndexScaledI2LN, indIndexI2LN, indIndexN, indOffIN, indOffLN);
6461
6462
6463 // iRegIorL2I is used for src inputs in rules for 32 bit int (I)
6464 // operations. it allows the src to be either an iRegI or a (ConvL2I
6465 // iRegL). in the latter case the l2i normally planted for a ConvL2I
6466 // can be elided because the 32-bit instruction will just employ the
6467 // lower 32 bits anyway.
6468 //
6469 // n.b. this does not elide all L2I conversions. if the truncated
6470 // value is consumed by more than one operation then the ConvL2I
6471 // cannot be bundled into the consuming nodes so an l2i gets planted
6472 // (actually a movw $dst $src) and the downstream instructions consume
6473 // the result of the l2i as an iRegI input. That's a shame since the
6474 // movw is actually redundant but its not too costly.
6475
6476 opclass iRegIorL2I(iRegI, iRegL2I);
6477
6478 //----------PIPELINE-----------------------------------------------------------
6479 // Rules which define the behavior of the target architectures pipeline.
6480
6481 // For specific pipelines, eg A53, define the stages of that pipeline
6482 //pipe_desc(ISS, EX1, EX2, WR);
6483 #define ISS S0
6484 #define EX1 S1
6485 #define EX2 S2
6486 #define WR S3
6487
6488 // Integer ALU reg operation
6489 pipeline %{
6490
6491 attributes %{
6492 // ARM instructions are of fixed length
6493 fixed_size_instructions; // Fixed size instructions TODO does
6494 max_instructions_per_bundle = 4; // A53 = 2, A57 = 4
6495 // ARM instructions come in 32-bit word units
6496 instruction_unit_size = 4; // An instruction is 4 bytes long
6497 instruction_fetch_unit_size = 64; // The processor fetches one line
6498 instruction_fetch_units = 1; // of 64 bytes
6499
6500 // List of nop instructions
6501 nops( MachNop );
6502 %}
6503
6504 // We don't use an actual pipeline model so don't care about resources
6505 // or description. we do use pipeline classes to introduce fixed
6506 // latencies
6507
6508 //----------RESOURCES----------------------------------------------------------
6509 // Resources are the functional units available to the machine
6510
6511 resources( INS0, INS1, INS01 = INS0 | INS1,
6512 ALU0, ALU1, ALU = ALU0 | ALU1,
6513 MAC,
6514 DIV,
6515 BRANCH,
6516 LDST,
6517 NEON_FP);
6518
6519 //----------PIPELINE DESCRIPTION-----------------------------------------------
6520 // Pipeline Description specifies the stages in the machine's pipeline
6521
6522 // Define the pipeline as a generic 6 stage pipeline
6523 pipe_desc(S0, S1, S2, S3, S4, S5);
6524
6525 //----------PIPELINE CLASSES---------------------------------------------------
6526 // Pipeline Classes describe the stages in which input and output are
6527 // referenced by the hardware pipeline.
6528
6529 pipe_class fp_dop_reg_reg_s(vRegF dst, vRegF src1, vRegF src2)
6530 %{
6531 single_instruction;
6532 src1 : S1(read);
6533 src2 : S2(read);
6534 dst : S5(write);
6535 INS01 : ISS;
6536 NEON_FP : S5;
6537 %}
6538
6539 pipe_class fp_dop_reg_reg_d(vRegD dst, vRegD src1, vRegD src2)
6540 %{
6541 single_instruction;
6542 src1 : S1(read);
6543 src2 : S2(read);
6544 dst : S5(write);
6545 INS01 : ISS;
6546 NEON_FP : S5;
6547 %}
6548
6549 pipe_class fp_uop_s(vRegF dst, vRegF src)
6550 %{
6551 single_instruction;
6552 src : S1(read);
6553 dst : S5(write);
6554 INS01 : ISS;
6555 NEON_FP : S5;
6556 %}
6557
6558 pipe_class fp_uop_d(vRegD dst, vRegD src)
6559 %{
6560 single_instruction;
6561 src : S1(read);
6562 dst : S5(write);
6563 INS01 : ISS;
6564 NEON_FP : S5;
6565 %}
6566
6567 pipe_class fp_d2f(vRegF dst, vRegD src)
6568 %{
6569 single_instruction;
6570 src : S1(read);
6571 dst : S5(write);
6572 INS01 : ISS;
6573 NEON_FP : S5;
6574 %}
6575
6576 pipe_class fp_f2d(vRegD dst, vRegF src)
6577 %{
6578 single_instruction;
6579 src : S1(read);
6580 dst : S5(write);
6581 INS01 : ISS;
6582 NEON_FP : S5;
6583 %}
6584
6585 pipe_class fp_f2i(iRegINoSp dst, vRegF src)
6586 %{
6587 single_instruction;
6588 src : S1(read);
6589 dst : S5(write);
6590 INS01 : ISS;
6591 NEON_FP : S5;
6592 %}
6593
6594 pipe_class fp_f2l(iRegLNoSp dst, vRegF src)
6595 %{
6596 single_instruction;
6597 src : S1(read);
6598 dst : S5(write);
6599 INS01 : ISS;
6600 NEON_FP : S5;
6601 %}
6602
6603 pipe_class fp_i2f(vRegF dst, iRegIorL2I src)
6604 %{
6605 single_instruction;
6606 src : S1(read);
6607 dst : S5(write);
6608 INS01 : ISS;
6609 NEON_FP : S5;
6610 %}
6611
6612 pipe_class fp_l2f(vRegF dst, iRegL src)
6613 %{
6614 single_instruction;
6615 src : S1(read);
6616 dst : S5(write);
6617 INS01 : ISS;
6618 NEON_FP : S5;
6619 %}
6620
6621 pipe_class fp_d2i(iRegINoSp dst, vRegD src)
6622 %{
6623 single_instruction;
6624 src : S1(read);
6625 dst : S5(write);
6626 INS01 : ISS;
6627 NEON_FP : S5;
6628 %}
6629
6630 pipe_class fp_d2l(iRegLNoSp dst, vRegD src)
6631 %{
6632 single_instruction;
6633 src : S1(read);
6634 dst : S5(write);
6635 INS01 : ISS;
6636 NEON_FP : S5;
6637 %}
6638
6639 pipe_class fp_i2d(vRegD dst, iRegIorL2I src)
6640 %{
6641 single_instruction;
6642 src : S1(read);
6643 dst : S5(write);
6644 INS01 : ISS;
6645 NEON_FP : S5;
6646 %}
6647
6648 pipe_class fp_l2d(vRegD dst, iRegIorL2I src)
6649 %{
6650 single_instruction;
6651 src : S1(read);
6652 dst : S5(write);
6653 INS01 : ISS;
6654 NEON_FP : S5;
6655 %}
6656
6657 pipe_class fp_div_s(vRegF dst, vRegF src1, vRegF src2)
6658 %{
6659 single_instruction;
6660 src1 : S1(read);
6661 src2 : S2(read);
6662 dst : S5(write);
6663 INS0 : ISS;
6664 NEON_FP : S5;
6665 %}
6666
6667 pipe_class fp_div_d(vRegD dst, vRegD src1, vRegD src2)
6668 %{
6669 single_instruction;
6670 src1 : S1(read);
6671 src2 : S2(read);
6672 dst : S5(write);
6673 INS0 : ISS;
6674 NEON_FP : S5;
6675 %}
6676
6677 pipe_class fp_cond_reg_reg_s(vRegF dst, vRegF src1, vRegF src2, rFlagsReg cr)
6678 %{
6679 single_instruction;
6680 cr : S1(read);
6681 src1 : S1(read);
6682 src2 : S1(read);
6683 dst : S3(write);
6684 INS01 : ISS;
6685 NEON_FP : S3;
6686 %}
6687
6688 pipe_class fp_cond_reg_reg_d(vRegD dst, vRegD src1, vRegD src2, rFlagsReg cr)
6689 %{
6690 single_instruction;
6691 cr : S1(read);
6692 src1 : S1(read);
6693 src2 : S1(read);
6694 dst : S3(write);
6695 INS01 : ISS;
6696 NEON_FP : S3;
6697 %}
6698
6699 pipe_class fp_imm_s(vRegF dst)
6700 %{
6701 single_instruction;
6702 dst : S3(write);
6703 INS01 : ISS;
6704 NEON_FP : S3;
6705 %}
6706
6707 pipe_class fp_imm_d(vRegD dst)
6708 %{
6709 single_instruction;
6710 dst : S3(write);
6711 INS01 : ISS;
6712 NEON_FP : S3;
6713 %}
6714
6715 pipe_class fp_load_constant_s(vRegF dst)
6716 %{
6717 single_instruction;
6718 dst : S4(write);
6719 INS01 : ISS;
6720 NEON_FP : S4;
6721 %}
6722
6723 pipe_class fp_load_constant_d(vRegD dst)
6724 %{
6725 single_instruction;
6726 dst : S4(write);
6727 INS01 : ISS;
6728 NEON_FP : S4;
6729 %}
6730
6731 pipe_class vmul64(vecD dst, vecD src1, vecD src2)
6732 %{
6733 single_instruction;
6734 dst : S5(write);
6735 src1 : S1(read);
6736 src2 : S1(read);
6737 INS01 : ISS;
6738 NEON_FP : S5;
6739 %}
6740
6741 pipe_class vmul128(vecX dst, vecX src1, vecX src2)
6742 %{
6743 single_instruction;
6744 dst : S5(write);
6745 src1 : S1(read);
6746 src2 : S1(read);
6747 INS0 : ISS;
6748 NEON_FP : S5;
6749 %}
6750
6751 pipe_class vmla64(vecD dst, vecD src1, vecD src2)
6752 %{
6753 single_instruction;
6754 dst : S5(write);
6755 src1 : S1(read);
6756 src2 : S1(read);
6757 dst : S1(read);
6758 INS01 : ISS;
6759 NEON_FP : S5;
6760 %}
6761
6762 pipe_class vmla128(vecX dst, vecX src1, vecX src2)
6763 %{
6764 single_instruction;
6765 dst : S5(write);
6766 src1 : S1(read);
6767 src2 : S1(read);
6768 dst : S1(read);
6769 INS0 : ISS;
6770 NEON_FP : S5;
6771 %}
6772
6773 pipe_class vdop64(vecD dst, vecD src1, vecD src2)
6774 %{
6775 single_instruction;
6776 dst : S4(write);
6777 src1 : S2(read);
6778 src2 : S2(read);
6779 INS01 : ISS;
6780 NEON_FP : S4;
6781 %}
6782
6783 pipe_class vdop128(vecX dst, vecX src1, vecX src2)
6784 %{
6785 single_instruction;
6786 dst : S4(write);
6787 src1 : S2(read);
6788 src2 : S2(read);
6789 INS0 : ISS;
6790 NEON_FP : S4;
6791 %}
6792
6793 pipe_class vlogical64(vecD dst, vecD src1, vecD src2)
6794 %{
6795 single_instruction;
6796 dst : S3(write);
6797 src1 : S2(read);
6798 src2 : S2(read);
6799 INS01 : ISS;
6800 NEON_FP : S3;
6801 %}
6802
6803 pipe_class vlogical128(vecX dst, vecX src1, vecX src2)
6804 %{
6805 single_instruction;
6806 dst : S3(write);
6807 src1 : S2(read);
6808 src2 : S2(read);
6809 INS0 : ISS;
6810 NEON_FP : S3;
6811 %}
6812
6813 pipe_class vshift64(vecD dst, vecD src, vecX shift)
6814 %{
6815 single_instruction;
6816 dst : S3(write);
6817 src : S1(read);
6818 shift : S1(read);
6819 INS01 : ISS;
6820 NEON_FP : S3;
6821 %}
6822
6823 pipe_class vshift128(vecX dst, vecX src, vecX shift)
6824 %{
6825 single_instruction;
6826 dst : S3(write);
6827 src : S1(read);
6828 shift : S1(read);
6829 INS0 : ISS;
6830 NEON_FP : S3;
6831 %}
6832
6833 pipe_class vshift64_imm(vecD dst, vecD src, immI shift)
6834 %{
6835 single_instruction;
6836 dst : S3(write);
6837 src : S1(read);
6838 INS01 : ISS;
6839 NEON_FP : S3;
6840 %}
6841
6842 pipe_class vshift128_imm(vecX dst, vecX src, immI shift)
6843 %{
6844 single_instruction;
6845 dst : S3(write);
6846 src : S1(read);
6847 INS0 : ISS;
6848 NEON_FP : S3;
6849 %}
6850
6851 pipe_class vdop_fp64(vecD dst, vecD src1, vecD src2)
6852 %{
6853 single_instruction;
6854 dst : S5(write);
6855 src1 : S1(read);
6856 src2 : S1(read);
6857 INS01 : ISS;
6858 NEON_FP : S5;
6859 %}
6860
6861 pipe_class vdop_fp128(vecX dst, vecX src1, vecX src2)
6862 %{
6863 single_instruction;
6864 dst : S5(write);
6865 src1 : S1(read);
6866 src2 : S1(read);
6867 INS0 : ISS;
6868 NEON_FP : S5;
6869 %}
6870
6871 pipe_class vmuldiv_fp64(vecD dst, vecD src1, vecD src2)
6872 %{
6873 single_instruction;
6874 dst : S5(write);
6875 src1 : S1(read);
6876 src2 : S1(read);
6877 INS0 : ISS;
6878 NEON_FP : S5;
6879 %}
6880
6881 pipe_class vmuldiv_fp128(vecX dst, vecX src1, vecX src2)
6882 %{
6883 single_instruction;
6884 dst : S5(write);
6885 src1 : S1(read);
6886 src2 : S1(read);
6887 INS0 : ISS;
6888 NEON_FP : S5;
6889 %}
6890
6891 pipe_class vsqrt_fp128(vecX dst, vecX src)
6892 %{
6893 single_instruction;
6894 dst : S5(write);
6895 src : S1(read);
6896 INS0 : ISS;
6897 NEON_FP : S5;
6898 %}
6899
6900 pipe_class vunop_fp64(vecD dst, vecD src)
6901 %{
6902 single_instruction;
6903 dst : S5(write);
6904 src : S1(read);
6905 INS01 : ISS;
6906 NEON_FP : S5;
6907 %}
6908
6909 pipe_class vunop_fp128(vecX dst, vecX src)
6910 %{
6911 single_instruction;
6912 dst : S5(write);
6913 src : S1(read);
6914 INS0 : ISS;
6915 NEON_FP : S5;
6916 %}
6917
6918 pipe_class vdup_reg_reg64(vecD dst, iRegI src)
6919 %{
6920 single_instruction;
6921 dst : S3(write);
6922 src : S1(read);
6923 INS01 : ISS;
6924 NEON_FP : S3;
6925 %}
6926
6927 pipe_class vdup_reg_reg128(vecX dst, iRegI src)
6928 %{
6929 single_instruction;
6930 dst : S3(write);
6931 src : S1(read);
6932 INS01 : ISS;
6933 NEON_FP : S3;
6934 %}
6935
6936 pipe_class vdup_reg_freg64(vecD dst, vRegF src)
6937 %{
6938 single_instruction;
6939 dst : S3(write);
6940 src : S1(read);
6941 INS01 : ISS;
6942 NEON_FP : S3;
6943 %}
6944
6945 pipe_class vdup_reg_freg128(vecX dst, vRegF src)
6946 %{
6947 single_instruction;
6948 dst : S3(write);
6949 src : S1(read);
6950 INS01 : ISS;
6951 NEON_FP : S3;
6952 %}
6953
6954 pipe_class vdup_reg_dreg128(vecX dst, vRegD src)
6955 %{
6956 single_instruction;
6957 dst : S3(write);
6958 src : S1(read);
6959 INS01 : ISS;
6960 NEON_FP : S3;
6961 %}
6962
6963 pipe_class vmovi_reg_imm64(vecD dst)
6964 %{
6965 single_instruction;
6966 dst : S3(write);
6967 INS01 : ISS;
6968 NEON_FP : S3;
6969 %}
6970
6971 pipe_class vmovi_reg_imm128(vecX dst)
6972 %{
6973 single_instruction;
6974 dst : S3(write);
6975 INS0 : ISS;
6976 NEON_FP : S3;
6977 %}
6978
6979 pipe_class vload_reg_mem64(vecD dst, vmem8 mem)
6980 %{
6981 single_instruction;
6982 dst : S5(write);
6983 mem : ISS(read);
6984 INS01 : ISS;
6985 NEON_FP : S3;
6986 %}
6987
6988 pipe_class vload_reg_mem128(vecX dst, vmem16 mem)
6989 %{
6990 single_instruction;
6991 dst : S5(write);
6992 mem : ISS(read);
6993 INS01 : ISS;
6994 NEON_FP : S3;
6995 %}
6996
6997 pipe_class vstore_reg_mem64(vecD src, vmem8 mem)
6998 %{
6999 single_instruction;
7000 mem : ISS(read);
7001 src : S2(read);
7002 INS01 : ISS;
7003 NEON_FP : S3;
7004 %}
7005
7006 pipe_class vstore_reg_mem128(vecD src, vmem16 mem)
7007 %{
7008 single_instruction;
7009 mem : ISS(read);
7010 src : S2(read);
7011 INS01 : ISS;
7012 NEON_FP : S3;
7013 %}
7014
7015 //------- Integer ALU operations --------------------------
7016
7017 // Integer ALU reg-reg operation
7018 // Operands needed in EX1, result generated in EX2
7019 // Eg. ADD x0, x1, x2
7020 pipe_class ialu_reg_reg(iRegI dst, iRegI src1, iRegI src2)
7021 %{
7022 single_instruction;
7023 dst : EX2(write);
7024 src1 : EX1(read);
7025 src2 : EX1(read);
7026 INS01 : ISS; // Dual issue as instruction 0 or 1
7027 ALU : EX2;
7028 %}
7029
7030 // Integer ALU reg-reg operation with constant shift
7031 // Shifted register must be available in LATE_ISS instead of EX1
7032 // Eg. ADD x0, x1, x2, LSL #2
7033 pipe_class ialu_reg_reg_shift(iRegI dst, iRegI src1, iRegI src2, immI shift)
7034 %{
7035 single_instruction;
7036 dst : EX2(write);
7037 src1 : EX1(read);
7038 src2 : ISS(read);
7039 INS01 : ISS;
7040 ALU : EX2;
7041 %}
7042
7043 // Integer ALU reg operation with constant shift
7044 // Eg. LSL x0, x1, #shift
7045 pipe_class ialu_reg_shift(iRegI dst, iRegI src1)
7046 %{
7047 single_instruction;
7048 dst : EX2(write);
7049 src1 : ISS(read);
7050 INS01 : ISS;
7051 ALU : EX2;
7052 %}
7053
7054 // Integer ALU reg-reg operation with variable shift
7055 // Both operands must be available in LATE_ISS instead of EX1
7056 // Result is available in EX1 instead of EX2
7057 // Eg. LSLV x0, x1, x2
7058 pipe_class ialu_reg_reg_vshift(iRegI dst, iRegI src1, iRegI src2)
7059 %{
7060 single_instruction;
7061 dst : EX1(write);
7062 src1 : ISS(read);
7063 src2 : ISS(read);
7064 INS01 : ISS;
7065 ALU : EX1;
7066 %}
7067
7068 // Integer ALU reg-reg operation with extract
7069 // As for _vshift above, but result generated in EX2
7070 // Eg. EXTR x0, x1, x2, #N
7071 pipe_class ialu_reg_reg_extr(iRegI dst, iRegI src1, iRegI src2)
7072 %{
7073 single_instruction;
7074 dst : EX2(write);
7075 src1 : ISS(read);
7076 src2 : ISS(read);
7077 INS1 : ISS; // Can only dual issue as Instruction 1
7078 ALU : EX1;
7079 %}
7080
7081 // Integer ALU reg operation
7082 // Eg. NEG x0, x1
7083 pipe_class ialu_reg(iRegI dst, iRegI src)
7084 %{
7085 single_instruction;
7086 dst : EX2(write);
7087 src : EX1(read);
7088 INS01 : ISS;
7089 ALU : EX2;
7090 %}
7091
7092 // Integer ALU reg mmediate operation
7093 // Eg. ADD x0, x1, #N
7094 pipe_class ialu_reg_imm(iRegI dst, iRegI src1)
7095 %{
7096 single_instruction;
7097 dst : EX2(write);
7098 src1 : EX1(read);
7099 INS01 : ISS;
7100 ALU : EX2;
7101 %}
7102
7103 // Integer ALU immediate operation (no source operands)
7104 // Eg. MOV x0, #N
7105 pipe_class ialu_imm(iRegI dst)
7106 %{
7107 single_instruction;
7108 dst : EX1(write);
7109 INS01 : ISS;
7110 ALU : EX1;
7111 %}
7112
7113 //------- Compare operation -------------------------------
7114
7115 // Compare reg-reg
7116 // Eg. CMP x0, x1
7117 pipe_class icmp_reg_reg(rFlagsReg cr, iRegI op1, iRegI op2)
7118 %{
7119 single_instruction;
7120 // fixed_latency(16);
7121 cr : EX2(write);
7122 op1 : EX1(read);
7123 op2 : EX1(read);
7124 INS01 : ISS;
7125 ALU : EX2;
7126 %}
7127
7128 // Compare reg-reg
7129 // Eg. CMP x0, #N
7130 pipe_class icmp_reg_imm(rFlagsReg cr, iRegI op1)
7131 %{
7132 single_instruction;
7133 // fixed_latency(16);
7134 cr : EX2(write);
7135 op1 : EX1(read);
7136 INS01 : ISS;
7137 ALU : EX2;
7138 %}
7139
7140 //------- Conditional instructions ------------------------
7141
7142 // Conditional no operands
7143 // Eg. CSINC x0, zr, zr, <cond>
7144 pipe_class icond_none(iRegI dst, rFlagsReg cr)
7145 %{
7146 single_instruction;
7147 cr : EX1(read);
7148 dst : EX2(write);
7149 INS01 : ISS;
7150 ALU : EX2;
7151 %}
7152
7153 // Conditional 2 operand
7154 // EG. CSEL X0, X1, X2, <cond>
7155 pipe_class icond_reg_reg(iRegI dst, iRegI src1, iRegI src2, rFlagsReg cr)
7156 %{
7157 single_instruction;
7158 cr : EX1(read);
7159 src1 : EX1(read);
7160 src2 : EX1(read);
7161 dst : EX2(write);
7162 INS01 : ISS;
7163 ALU : EX2;
7164 %}
7165
7166 // Conditional 2 operand
7167 // EG. CSEL X0, X1, X2, <cond>
7168 pipe_class icond_reg(iRegI dst, iRegI src, rFlagsReg cr)
7169 %{
7170 single_instruction;
7171 cr : EX1(read);
7172 src : EX1(read);
7173 dst : EX2(write);
7174 INS01 : ISS;
7175 ALU : EX2;
7176 %}
7177
7178 //------- Multiply pipeline operations --------------------
7179
7180 // Multiply reg-reg
7181 // Eg. MUL w0, w1, w2
7182 pipe_class imul_reg_reg(iRegI dst, iRegI src1, iRegI src2)
7183 %{
7184 single_instruction;
7185 dst : WR(write);
7186 src1 : ISS(read);
7187 src2 : ISS(read);
7188 INS01 : ISS;
7189 MAC : WR;
7190 %}
7191
7192 // Multiply accumulate
7193 // Eg. MADD w0, w1, w2, w3
7194 pipe_class imac_reg_reg(iRegI dst, iRegI src1, iRegI src2, iRegI src3)
7195 %{
7196 single_instruction;
7197 dst : WR(write);
7198 src1 : ISS(read);
7199 src2 : ISS(read);
7200 src3 : ISS(read);
7201 INS01 : ISS;
7202 MAC : WR;
7203 %}
7204
7205 // Eg. MUL w0, w1, w2
7206 pipe_class lmul_reg_reg(iRegI dst, iRegI src1, iRegI src2)
7207 %{
7208 single_instruction;
7209 fixed_latency(3); // Maximum latency for 64 bit mul
7210 dst : WR(write);
7211 src1 : ISS(read);
7212 src2 : ISS(read);
7213 INS01 : ISS;
7214 MAC : WR;
7215 %}
7216
7217 // Multiply accumulate
7218 // Eg. MADD w0, w1, w2, w3
7219 pipe_class lmac_reg_reg(iRegI dst, iRegI src1, iRegI src2, iRegI src3)
7220 %{
7221 single_instruction;
7222 fixed_latency(3); // Maximum latency for 64 bit mul
7223 dst : WR(write);
7224 src1 : ISS(read);
7225 src2 : ISS(read);
7226 src3 : ISS(read);
7227 INS01 : ISS;
7228 MAC : WR;
7229 %}
7230
7231 //------- Divide pipeline operations --------------------
7232
7233 // Eg. SDIV w0, w1, w2
7234 pipe_class idiv_reg_reg(iRegI dst, iRegI src1, iRegI src2)
7235 %{
7236 single_instruction;
7237 fixed_latency(8); // Maximum latency for 32 bit divide
7238 dst : WR(write);
7239 src1 : ISS(read);
7240 src2 : ISS(read);
7241 INS0 : ISS; // Can only dual issue as instruction 0
7242 DIV : WR;
7243 %}
7244
7245 // Eg. SDIV x0, x1, x2
7246 pipe_class ldiv_reg_reg(iRegI dst, iRegI src1, iRegI src2)
7247 %{
7248 single_instruction;
7249 fixed_latency(16); // Maximum latency for 64 bit divide
7250 dst : WR(write);
7251 src1 : ISS(read);
7252 src2 : ISS(read);
7253 INS0 : ISS; // Can only dual issue as instruction 0
7254 DIV : WR;
7255 %}
7256
7257 //------- Load pipeline operations ------------------------
7258
7259 // Load - prefetch
7260 // Eg. PFRM <mem>
7261 pipe_class iload_prefetch(memory mem)
7262 %{
7263 single_instruction;
7264 mem : ISS(read);
7265 INS01 : ISS;
7266 LDST : WR;
7267 %}
7268
7269 // Load - reg, mem
7270 // Eg. LDR x0, <mem>
7271 pipe_class iload_reg_mem(iRegI dst, memory mem)
7272 %{
7273 single_instruction;
7274 dst : WR(write);
7275 mem : ISS(read);
7276 INS01 : ISS;
7277 LDST : WR;
7278 %}
7279
7280 // Load - reg, reg
7281 // Eg. LDR x0, [sp, x1]
7282 pipe_class iload_reg_reg(iRegI dst, iRegI src)
7283 %{
7284 single_instruction;
7285 dst : WR(write);
7286 src : ISS(read);
7287 INS01 : ISS;
7288 LDST : WR;
7289 %}
7290
7291 //------- Store pipeline operations -----------------------
7292
7293 // Store - zr, mem
7294 // Eg. STR zr, <mem>
7295 pipe_class istore_mem(memory mem)
7296 %{
7297 single_instruction;
7298 mem : ISS(read);
7299 INS01 : ISS;
7300 LDST : WR;
7301 %}
7302
7303 // Store - reg, mem
7304 // Eg. STR x0, <mem>
7305 pipe_class istore_reg_mem(iRegI src, memory mem)
7306 %{
7307 single_instruction;
7308 mem : ISS(read);
7309 src : EX2(read);
7310 INS01 : ISS;
7311 LDST : WR;
7312 %}
7313
7314 // Store - reg, reg
7315 // Eg. STR x0, [sp, x1]
7316 pipe_class istore_reg_reg(iRegI dst, iRegI src)
7317 %{
7318 single_instruction;
7319 dst : ISS(read);
7320 src : EX2(read);
7321 INS01 : ISS;
7322 LDST : WR;
7323 %}
7324
7325 //------- Store pipeline operations -----------------------
7326
7327 // Branch
7328 pipe_class pipe_branch()
7329 %{
7330 single_instruction;
7331 INS01 : ISS;
7332 BRANCH : EX1;
7333 %}
7334
7335 // Conditional branch
7336 pipe_class pipe_branch_cond(rFlagsReg cr)
7337 %{
7338 single_instruction;
7339 cr : EX1(read);
7340 INS01 : ISS;
7341 BRANCH : EX1;
7342 %}
7343
7344 // Compare & Branch
7345 // EG. CBZ/CBNZ
7346 pipe_class pipe_cmp_branch(iRegI op1)
7347 %{
7348 single_instruction;
7349 op1 : EX1(read);
7350 INS01 : ISS;
7351 BRANCH : EX1;
7352 %}
7353
7354 //------- Synchronisation operations ----------------------
7355
7356 // Any operation requiring serialization.
7357 // EG. DMB/Atomic Ops/Load Acquire/Str Release
7358 pipe_class pipe_serial()
7359 %{
7360 single_instruction;
7361 force_serialization;
7362 fixed_latency(16);
7363 INS01 : ISS(2); // Cannot dual issue with any other instruction
7364 LDST : WR;
7365 %}
7366
7367 // Generic big/slow expanded idiom - also serialized
7368 pipe_class pipe_slow()
7369 %{
7370 instruction_count(10);
7371 multiple_bundles;
7372 force_serialization;
7373 fixed_latency(16);
7374 INS01 : ISS(2); // Cannot dual issue with any other instruction
7375 LDST : WR;
7376 %}
7377
7378 // Empty pipeline class
7379 pipe_class pipe_class_empty()
7380 %{
7381 single_instruction;
7382 fixed_latency(0);
7383 %}
7384
7385 // Default pipeline class.
7386 pipe_class pipe_class_default()
7387 %{
7388 single_instruction;
7389 fixed_latency(2);
7390 %}
7391
7392 // Pipeline class for compares.
7393 pipe_class pipe_class_compare()
7394 %{
7395 single_instruction;
7396 fixed_latency(16);
7397 %}
7398
7399 // Pipeline class for memory operations.
7400 pipe_class pipe_class_memory()
7401 %{
7402 single_instruction;
7403 fixed_latency(16);
7404 %}
7405
7406 // Pipeline class for call.
7407 pipe_class pipe_class_call()
7408 %{
7409 single_instruction;
7410 fixed_latency(100);
7411 %}
7412
7413 // Define the class for the Nop node.
7414 define %{
7415 MachNop = pipe_class_empty;
7416 %}
7417
7418 %}
7419 //----------INSTRUCTIONS-------------------------------------------------------
7420 //
7421 // match -- States which machine-independent subtree may be replaced
7422 // by this instruction.
7423 // ins_cost -- The estimated cost of this instruction is used by instruction
7424 // selection to identify a minimum cost tree of machine
7425 // instructions that matches a tree of machine-independent
7426 // instructions.
7427 // format -- A string providing the disassembly for this instruction.
7428 // The value of an instruction's operand may be inserted
7429 // by referring to it with a '$' prefix.
7430 // opcode -- Three instruction opcodes may be provided. These are referred
7431 // to within an encode class as $primary, $secondary, and $tertiary
7432 // rrspectively. The primary opcode is commonly used to
7433 // indicate the type of machine instruction, while secondary
7434 // and tertiary are often used for prefix options or addressing
7435 // modes.
7436 // ins_encode -- A list of encode classes with parameters. The encode class
7437 // name must have been defined in an 'enc_class' specification
7438 // in the encode section of the architecture description.
7439
7440 // ============================================================================
7441 // Memory (Load/Store) Instructions
7442
7443 // Load Instructions
7444
7445 // Load Byte (8 bit signed)
7446 instruct loadB(iRegINoSp dst, memory1 mem)
7447 %{
7448 match(Set dst (LoadB mem));
7449 predicate(!needs_acquiring_load(n));
7450
7451 ins_cost(4 * INSN_COST);
7452 format %{ "ldrsbw $dst, $mem\t# byte" %}
7453
7454 ins_encode(aarch64_enc_ldrsbw(dst, mem));
7455
7456 ins_pipe(iload_reg_mem);
7457 %}
7458
7459 // Load Byte (8 bit signed) into long
7460 instruct loadB2L(iRegLNoSp dst, memory1 mem)
7461 %{
7462 match(Set dst (ConvI2L (LoadB mem)));
7463 predicate(!needs_acquiring_load(n->in(1)));
7464
7465 ins_cost(4 * INSN_COST);
7466 format %{ "ldrsb $dst, $mem\t# byte" %}
7467
7468 ins_encode(aarch64_enc_ldrsb(dst, mem));
7469
7470 ins_pipe(iload_reg_mem);
7471 %}
7472
7473 // Load Byte (8 bit unsigned)
7474 instruct loadUB(iRegINoSp dst, memory1 mem)
7475 %{
7476 match(Set dst (LoadUB mem));
7477 predicate(!needs_acquiring_load(n));
7478
7479 ins_cost(4 * INSN_COST);
7480 format %{ "ldrbw $dst, $mem\t# byte" %}
7481
7482 ins_encode(aarch64_enc_ldrb(dst, mem));
7483
7484 ins_pipe(iload_reg_mem);
7485 %}
7486
7487 // Load Byte (8 bit unsigned) into long
7488 instruct loadUB2L(iRegLNoSp dst, memory1 mem)
7489 %{
7490 match(Set dst (ConvI2L (LoadUB mem)));
7491 predicate(!needs_acquiring_load(n->in(1)));
7492
7493 ins_cost(4 * INSN_COST);
7494 format %{ "ldrb $dst, $mem\t# byte" %}
7495
7496 ins_encode(aarch64_enc_ldrb(dst, mem));
7497
7498 ins_pipe(iload_reg_mem);
7499 %}
7500
7501 // Load Short (16 bit signed)
7502 instruct loadS(iRegINoSp dst, memory2 mem)
7503 %{
7504 match(Set dst (LoadS mem));
7505 predicate(!needs_acquiring_load(n));
7506
7507 ins_cost(4 * INSN_COST);
7508 format %{ "ldrshw $dst, $mem\t# short" %}
7509
7510 ins_encode(aarch64_enc_ldrshw(dst, mem));
7511
7512 ins_pipe(iload_reg_mem);
7513 %}
7514
7515 // Load Short (16 bit signed) into long
7516 instruct loadS2L(iRegLNoSp dst, memory2 mem)
7517 %{
7518 match(Set dst (ConvI2L (LoadS mem)));
7519 predicate(!needs_acquiring_load(n->in(1)));
7520
7521 ins_cost(4 * INSN_COST);
7522 format %{ "ldrsh $dst, $mem\t# short" %}
7523
7524 ins_encode(aarch64_enc_ldrsh(dst, mem));
7525
7526 ins_pipe(iload_reg_mem);
7527 %}
7528
7529 // Load Char (16 bit unsigned)
7530 instruct loadUS(iRegINoSp dst, memory2 mem)
7531 %{
7532 match(Set dst (LoadUS mem));
7533 predicate(!needs_acquiring_load(n));
7534
7535 ins_cost(4 * INSN_COST);
7536 format %{ "ldrh $dst, $mem\t# short" %}
7537
7538 ins_encode(aarch64_enc_ldrh(dst, mem));
7539
7540 ins_pipe(iload_reg_mem);
7541 %}
7542
7543 // Load Short/Char (16 bit unsigned) into long
7544 instruct loadUS2L(iRegLNoSp dst, memory2 mem)
7545 %{
7546 match(Set dst (ConvI2L (LoadUS mem)));
7547 predicate(!needs_acquiring_load(n->in(1)));
7548
7549 ins_cost(4 * INSN_COST);
7550 format %{ "ldrh $dst, $mem\t# short" %}
7551
7552 ins_encode(aarch64_enc_ldrh(dst, mem));
7553
7554 ins_pipe(iload_reg_mem);
7555 %}
7556
7557 // Load Integer (32 bit signed)
7558 instruct loadI(iRegINoSp dst, memory4 mem)
7559 %{
7560 match(Set dst (LoadI mem));
7561 predicate(!needs_acquiring_load(n));
7562
7563 ins_cost(4 * INSN_COST);
7564 format %{ "ldrw $dst, $mem\t# int" %}
7565
7566 ins_encode(aarch64_enc_ldrw(dst, mem));
7567
7568 ins_pipe(iload_reg_mem);
7569 %}
7570
7571 // Load Integer (32 bit signed) into long
7572 instruct loadI2L(iRegLNoSp dst, memory4 mem)
7573 %{
7574 match(Set dst (ConvI2L (LoadI mem)));
7575 predicate(!needs_acquiring_load(n->in(1)));
7576
7577 ins_cost(4 * INSN_COST);
7578 format %{ "ldrsw $dst, $mem\t# int" %}
7579
7580 ins_encode(aarch64_enc_ldrsw(dst, mem));
7581
7582 ins_pipe(iload_reg_mem);
7583 %}
7584
7585 // Load Integer (32 bit unsigned) into long
7586 instruct loadUI2L(iRegLNoSp dst, memory4 mem, immL_32bits mask)
7587 %{
7588 match(Set dst (AndL (ConvI2L (LoadI mem)) mask));
7589 predicate(!needs_acquiring_load(n->in(1)->in(1)->as_Load()));
7590
7591 ins_cost(4 * INSN_COST);
7592 format %{ "ldrw $dst, $mem\t# int" %}
7593
7594 ins_encode(aarch64_enc_ldrw(dst, mem));
7595
7596 ins_pipe(iload_reg_mem);
7597 %}
7598
7599 // Load Long (64 bit signed)
7600 instruct loadL(iRegLNoSp dst, memory8 mem)
7601 %{
7602 match(Set dst (LoadL mem));
7603 predicate(!needs_acquiring_load(n));
7604
7605 ins_cost(4 * INSN_COST);
7606 format %{ "ldr $dst, $mem\t# int" %}
7607
7608 ins_encode(aarch64_enc_ldr(dst, mem));
7609
7610 ins_pipe(iload_reg_mem);
7611 %}
7612
7613 // Load Range
7614 instruct loadRange(iRegINoSp dst, memory4 mem)
7615 %{
7616 match(Set dst (LoadRange mem));
7617
7618 ins_cost(4 * INSN_COST);
7619 format %{ "ldrw $dst, $mem\t# range" %}
7620
7621 ins_encode(aarch64_enc_ldrw(dst, mem));
7622
7623 ins_pipe(iload_reg_mem);
7624 %}
7625
7626 // Load Pointer
7627 instruct loadP(iRegPNoSp dst, memory8 mem)
7628 %{
7629 match(Set dst (LoadP mem));
7630 predicate(!needs_acquiring_load(n) && (n->as_Load()->barrier_data() == 0));
7631
7632 ins_cost(4 * INSN_COST);
7633 format %{ "ldr $dst, $mem\t# ptr" %}
7634
7635 ins_encode(aarch64_enc_ldr(dst, mem));
7636
7637 ins_pipe(iload_reg_mem);
7638 %}
7639
7640 // Load Compressed Pointer
7641 instruct loadN(iRegNNoSp dst, memory4 mem)
7642 %{
7643 match(Set dst (LoadN mem));
7644 predicate(!needs_acquiring_load(n));
7645
7646 ins_cost(4 * INSN_COST);
7647 format %{ "ldrw $dst, $mem\t# compressed ptr" %}
7648
7649 ins_encode(aarch64_enc_ldrw(dst, mem));
7650
7651 ins_pipe(iload_reg_mem);
7652 %}
7653
7654 // Load Klass Pointer
7655 instruct loadKlass(iRegPNoSp dst, memory8 mem)
7656 %{
7657 match(Set dst (LoadKlass mem));
7658 predicate(!needs_acquiring_load(n));
7659
7660 ins_cost(4 * INSN_COST);
7661 format %{ "ldr $dst, $mem\t# class" %}
7662
7663 ins_encode(aarch64_enc_ldr(dst, mem));
7664
7665 ins_pipe(iload_reg_mem);
7666 %}
7667
7668 // Load Narrow Klass Pointer
7669 instruct loadNKlass(iRegNNoSp dst, memory4 mem)
7670 %{
7671 match(Set dst (LoadNKlass mem));
7672 predicate(!needs_acquiring_load(n));
7673
7674 ins_cost(4 * INSN_COST);
7675 format %{ "ldrw $dst, $mem\t# compressed class ptr" %}
7676
7677 ins_encode(aarch64_enc_ldrw(dst, mem));
7678
7679 ins_pipe(iload_reg_mem);
7680 %}
7681
7682 // Load Float
7683 instruct loadF(vRegF dst, memory4 mem)
7684 %{
7685 match(Set dst (LoadF mem));
7686 predicate(!needs_acquiring_load(n));
7687
7688 ins_cost(4 * INSN_COST);
7689 format %{ "ldrs $dst, $mem\t# float" %}
7690
7691 ins_encode( aarch64_enc_ldrs(dst, mem) );
7692
7693 ins_pipe(pipe_class_memory);
7694 %}
7695
7696 // Load Double
7697 instruct loadD(vRegD dst, memory8 mem)
7698 %{
7699 match(Set dst (LoadD mem));
7700 predicate(!needs_acquiring_load(n));
7701
7702 ins_cost(4 * INSN_COST);
7703 format %{ "ldrd $dst, $mem\t# double" %}
7704
7705 ins_encode( aarch64_enc_ldrd(dst, mem) );
7706
7707 ins_pipe(pipe_class_memory);
7708 %}
7709
7710
7711 // Load Int Constant
7712 instruct loadConI(iRegINoSp dst, immI src)
7713 %{
7714 match(Set dst src);
7715
7716 ins_cost(INSN_COST);
7717 format %{ "mov $dst, $src\t# int" %}
7718
7719 ins_encode( aarch64_enc_movw_imm(dst, src) );
7720
7721 ins_pipe(ialu_imm);
7722 %}
7723
7724 // Load Long Constant
7725 instruct loadConL(iRegLNoSp dst, immL src)
7726 %{
7727 match(Set dst src);
7728
7729 ins_cost(INSN_COST);
7730 format %{ "mov $dst, $src\t# long" %}
7731
7732 ins_encode( aarch64_enc_mov_imm(dst, src) );
7733
7734 ins_pipe(ialu_imm);
7735 %}
7736
7737 // Load Pointer Constant
7738
7739 instruct loadConP(iRegPNoSp dst, immP con)
7740 %{
7741 match(Set dst con);
7742
7743 ins_cost(INSN_COST * 4);
7744 format %{
7745 "mov $dst, $con\t# ptr"
7746 %}
7747
7748 ins_encode(aarch64_enc_mov_p(dst, con));
7749
7750 ins_pipe(ialu_imm);
7751 %}
7752
7753 // Load Null Pointer Constant
7754
7755 instruct loadConP0(iRegPNoSp dst, immP0 con)
7756 %{
7757 match(Set dst con);
7758
7759 ins_cost(INSN_COST);
7760 format %{ "mov $dst, $con\t# NULL ptr" %}
7761
7762 ins_encode(aarch64_enc_mov_p0(dst, con));
7763
7764 ins_pipe(ialu_imm);
7765 %}
7766
7767 // Load Pointer Constant One
7768
7769 instruct loadConP1(iRegPNoSp dst, immP_1 con)
7770 %{
7771 match(Set dst con);
7772
7773 ins_cost(INSN_COST);
7774 format %{ "mov $dst, $con\t# NULL ptr" %}
7775
7776 ins_encode(aarch64_enc_mov_p1(dst, con));
7777
7778 ins_pipe(ialu_imm);
7779 %}
7780
7781 // Load Byte Map Base Constant
7782
7783 instruct loadByteMapBase(iRegPNoSp dst, immByteMapBase con)
7784 %{
7785 match(Set dst con);
7786
7787 ins_cost(INSN_COST);
7788 format %{ "adr $dst, $con\t# Byte Map Base" %}
7789
7790 ins_encode(aarch64_enc_mov_byte_map_base(dst, con));
7791
7792 ins_pipe(ialu_imm);
7793 %}
7794
7795 // Load Narrow Pointer Constant
7796
7797 instruct loadConN(iRegNNoSp dst, immN con)
7798 %{
7799 match(Set dst con);
7800
7801 ins_cost(INSN_COST * 4);
7802 format %{ "mov $dst, $con\t# compressed ptr" %}
7803
7804 ins_encode(aarch64_enc_mov_n(dst, con));
7805
7806 ins_pipe(ialu_imm);
7807 %}
7808
7809 // Load Narrow Null Pointer Constant
7810
7811 instruct loadConN0(iRegNNoSp dst, immN0 con)
7812 %{
7813 match(Set dst con);
7814
7815 ins_cost(INSN_COST);
7816 format %{ "mov $dst, $con\t# compressed NULL ptr" %}
7817
7818 ins_encode(aarch64_enc_mov_n0(dst, con));
7819
7820 ins_pipe(ialu_imm);
7821 %}
7822
7823 // Load Narrow Klass Constant
7824
7825 instruct loadConNKlass(iRegNNoSp dst, immNKlass con)
7826 %{
7827 match(Set dst con);
7828
7829 ins_cost(INSN_COST);
7830 format %{ "mov $dst, $con\t# compressed klass ptr" %}
7831
7832 ins_encode(aarch64_enc_mov_nk(dst, con));
7833
7834 ins_pipe(ialu_imm);
7835 %}
7836
7837 // Load Packed Float Constant
7838
7839 instruct loadConF_packed(vRegF dst, immFPacked con) %{
7840 match(Set dst con);
7841 ins_cost(INSN_COST * 4);
7842 format %{ "fmovs $dst, $con"%}
7843 ins_encode %{
7844 __ fmovs(as_FloatRegister($dst$$reg), (double)$con$$constant);
7845 %}
7846
7847 ins_pipe(fp_imm_s);
7848 %}
7849
7850 // Load Float Constant
7851
7852 instruct loadConF(vRegF dst, immF con) %{
7853 match(Set dst con);
7854
7855 ins_cost(INSN_COST * 4);
7856
7857 format %{
7858 "ldrs $dst, [$constantaddress]\t# load from constant table: float=$con\n\t"
7859 %}
7860
7861 ins_encode %{
7862 __ ldrs(as_FloatRegister($dst$$reg), $constantaddress($con));
7863 %}
7864
7865 ins_pipe(fp_load_constant_s);
7866 %}
7867
7868 // Load Packed Double Constant
7869
7870 instruct loadConD_packed(vRegD dst, immDPacked con) %{
7871 match(Set dst con);
7872 ins_cost(INSN_COST);
7873 format %{ "fmovd $dst, $con"%}
7874 ins_encode %{
7875 __ fmovd(as_FloatRegister($dst$$reg), $con$$constant);
7876 %}
7877
7878 ins_pipe(fp_imm_d);
7879 %}
7880
7881 // Load Double Constant
7882
7883 instruct loadConD(vRegD dst, immD con) %{
7884 match(Set dst con);
7885
7886 ins_cost(INSN_COST * 5);
7887 format %{
7888 "ldrd $dst, [$constantaddress]\t# load from constant table: float=$con\n\t"
7889 %}
7890
7891 ins_encode %{
7892 __ ldrd(as_FloatRegister($dst$$reg), $constantaddress($con));
7893 %}
7894
7895 ins_pipe(fp_load_constant_d);
7896 %}
7897
7898 // Store Instructions
7899
7900 // Store CMS card-mark Immediate
7901 instruct storeimmCM0(immI0 zero, memory1 mem)
7902 %{
7903 match(Set mem (StoreCM mem zero));
7904
7905 ins_cost(INSN_COST);
7906 format %{ "storestore (elided)\n\t"
7907 "strb zr, $mem\t# byte" %}
7908
7909 ins_encode(aarch64_enc_strb0(mem));
7910
7911 ins_pipe(istore_mem);
7912 %}
7913
7914 // Store CMS card-mark Immediate with intervening StoreStore
7915 // needed when using CMS with no conditional card marking
7916 instruct storeimmCM0_ordered(immI0 zero, memory1 mem)
7917 %{
7918 match(Set mem (StoreCM mem zero));
7919
7920 ins_cost(INSN_COST * 2);
7921 format %{ "storestore\n\t"
7922 "dmb ishst"
7923 "\n\tstrb zr, $mem\t# byte" %}
7924
7925 ins_encode(aarch64_enc_strb0_ordered(mem));
7926
7927 ins_pipe(istore_mem);
7928 %}
7929
7930 // Store Byte
7931 instruct storeB(iRegIorL2I src, memory1 mem)
7932 %{
7933 match(Set mem (StoreB mem src));
7934 predicate(!needs_releasing_store(n));
7935
7936 ins_cost(INSN_COST);
7937 format %{ "strb $src, $mem\t# byte" %}
7938
7939 ins_encode(aarch64_enc_strb(src, mem));
7940
7941 ins_pipe(istore_reg_mem);
7942 %}
7943
7944
7945 instruct storeimmB0(immI0 zero, memory1 mem)
7946 %{
7947 match(Set mem (StoreB mem zero));
7948 predicate(!needs_releasing_store(n));
7949
7950 ins_cost(INSN_COST);
7951 format %{ "strb rscractch2, $mem\t# byte" %}
7952
7953 ins_encode(aarch64_enc_strb0(mem));
7954
7955 ins_pipe(istore_mem);
7956 %}
7957
7958 // Store Char/Short
7959 instruct storeC(iRegIorL2I src, memory2 mem)
7960 %{
7961 match(Set mem (StoreC mem src));
7962 predicate(!needs_releasing_store(n));
7963
7964 ins_cost(INSN_COST);
7965 format %{ "strh $src, $mem\t# short" %}
7966
7967 ins_encode(aarch64_enc_strh(src, mem));
7968
7969 ins_pipe(istore_reg_mem);
7970 %}
7971
7972 instruct storeimmC0(immI0 zero, memory2 mem)
7973 %{
7974 match(Set mem (StoreC mem zero));
7975 predicate(!needs_releasing_store(n));
7976
7977 ins_cost(INSN_COST);
7978 format %{ "strh zr, $mem\t# short" %}
7979
7980 ins_encode(aarch64_enc_strh0(mem));
7981
7982 ins_pipe(istore_mem);
7983 %}
7984
7985 // Store Integer
7986
7987 instruct storeI(iRegIorL2I src, memory4 mem)
7988 %{
7989 match(Set mem(StoreI mem src));
7990 predicate(!needs_releasing_store(n));
7991
7992 ins_cost(INSN_COST);
7993 format %{ "strw $src, $mem\t# int" %}
7994
7995 ins_encode(aarch64_enc_strw(src, mem));
7996
7997 ins_pipe(istore_reg_mem);
7998 %}
7999
8000 instruct storeimmI0(immI0 zero, memory4 mem)
8001 %{
8002 match(Set mem(StoreI mem zero));
8003 predicate(!needs_releasing_store(n));
8004
8005 ins_cost(INSN_COST);
8006 format %{ "strw zr, $mem\t# int" %}
8007
8008 ins_encode(aarch64_enc_strw0(mem));
8009
8010 ins_pipe(istore_mem);
8011 %}
8012
8013 // Store Long (64 bit signed)
8014 instruct storeL(iRegL src, memory8 mem)
8015 %{
8016 match(Set mem (StoreL mem src));
8017 predicate(!needs_releasing_store(n));
8018
8019 ins_cost(INSN_COST);
8020 format %{ "str $src, $mem\t# int" %}
8021
8022 ins_encode(aarch64_enc_str(src, mem));
8023
8024 ins_pipe(istore_reg_mem);
8025 %}
8026
8027 // Store Long (64 bit signed)
8028 instruct storeimmL0(immL0 zero, memory8 mem)
8029 %{
8030 match(Set mem (StoreL mem zero));
8031 predicate(!needs_releasing_store(n));
8032
8033 ins_cost(INSN_COST);
8034 format %{ "str zr, $mem\t# int" %}
8035
8036 ins_encode(aarch64_enc_str0(mem));
8037
8038 ins_pipe(istore_mem);
8039 %}
8040
8041 // Store Pointer
8042 instruct storeP(iRegP src, memory8 mem)
8043 %{
8044 match(Set mem (StoreP mem src));
8045 predicate(!needs_releasing_store(n));
8046
8047 ins_cost(INSN_COST);
8048 format %{ "str $src, $mem\t# ptr" %}
8049
8050 ins_encode(aarch64_enc_str(src, mem));
8051
8052 ins_pipe(istore_reg_mem);
8053 %}
8054
8055 // Store Pointer
8056 instruct storeimmP0(immP0 zero, memory8 mem)
8057 %{
8058 match(Set mem (StoreP mem zero));
8059 predicate(!needs_releasing_store(n));
8060
8061 ins_cost(INSN_COST);
8062 format %{ "str zr, $mem\t# ptr" %}
8063
8064 ins_encode(aarch64_enc_str0(mem));
8065
8066 ins_pipe(istore_mem);
8067 %}
8068
8069 // Store Compressed Pointer
8070 instruct storeN(iRegN src, memory4 mem)
8071 %{
8072 match(Set mem (StoreN mem src));
8073 predicate(!needs_releasing_store(n));
8074
8075 ins_cost(INSN_COST);
8076 format %{ "strw $src, $mem\t# compressed ptr" %}
8077
8078 ins_encode(aarch64_enc_strw(src, mem));
8079
8080 ins_pipe(istore_reg_mem);
8081 %}
8082
8083 instruct storeImmN0(immN0 zero, memory4 mem)
8084 %{
8085 match(Set mem (StoreN mem zero));
8086 predicate(!needs_releasing_store(n));
8087
8088 ins_cost(INSN_COST);
8089 format %{ "strw zr, $mem\t# compressed ptr" %}
8090
8091 ins_encode(aarch64_enc_strw0(mem));
8092
8093 ins_pipe(istore_mem);
8094 %}
8095
8096 // Store Float
8097 instruct storeF(vRegF src, memory4 mem)
8098 %{
8099 match(Set mem (StoreF mem src));
8100 predicate(!needs_releasing_store(n));
8101
8102 ins_cost(INSN_COST);
8103 format %{ "strs $src, $mem\t# float" %}
8104
8105 ins_encode( aarch64_enc_strs(src, mem) );
8106
8107 ins_pipe(pipe_class_memory);
8108 %}
8109
8110 // TODO
8111 // implement storeImmF0 and storeFImmPacked
8112
8113 // Store Double
8114 instruct storeD(vRegD src, memory8 mem)
8115 %{
8116 match(Set mem (StoreD mem src));
8117 predicate(!needs_releasing_store(n));
8118
8119 ins_cost(INSN_COST);
8120 format %{ "strd $src, $mem\t# double" %}
8121
8122 ins_encode( aarch64_enc_strd(src, mem) );
8123
8124 ins_pipe(pipe_class_memory);
8125 %}
8126
8127 // Store Compressed Klass Pointer
8128 instruct storeNKlass(iRegN src, memory4 mem)
8129 %{
8130 predicate(!needs_releasing_store(n));
8131 match(Set mem (StoreNKlass mem src));
8132
8133 ins_cost(INSN_COST);
8134 format %{ "strw $src, $mem\t# compressed klass ptr" %}
8135
8136 ins_encode(aarch64_enc_strw(src, mem));
8137
8138 ins_pipe(istore_reg_mem);
8139 %}
8140
8141 // TODO
8142 // implement storeImmD0 and storeDImmPacked
8143
8144 // prefetch instructions
8145 // Must be safe to execute with invalid address (cannot fault).
8146
8147 instruct prefetchalloc( memory8 mem ) %{
8148 match(PrefetchAllocation mem);
8149
8150 ins_cost(INSN_COST);
8151 format %{ "prfm $mem, PSTL1KEEP\t# Prefetch into level 1 cache write keep" %}
8152
8153 ins_encode( aarch64_enc_prefetchw(mem) );
8154
8155 ins_pipe(iload_prefetch);
8156 %}
8157
8158 // ---------------- volatile loads and stores ----------------
8159
8160 // Load Byte (8 bit signed)
8161 instruct loadB_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
8162 %{
8163 match(Set dst (LoadB mem));
8164
8165 ins_cost(VOLATILE_REF_COST);
8166 format %{ "ldarsb $dst, $mem\t# byte" %}
8167
8168 ins_encode(aarch64_enc_ldarsb(dst, mem));
8169
8170 ins_pipe(pipe_serial);
8171 %}
8172
8173 // Load Byte (8 bit signed) into long
8174 instruct loadB2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
8175 %{
8176 match(Set dst (ConvI2L (LoadB mem)));
8177
8178 ins_cost(VOLATILE_REF_COST);
8179 format %{ "ldarsb $dst, $mem\t# byte" %}
8180
8181 ins_encode(aarch64_enc_ldarsb(dst, mem));
8182
8183 ins_pipe(pipe_serial);
8184 %}
8185
8186 // Load Byte (8 bit unsigned)
8187 instruct loadUB_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
8188 %{
8189 match(Set dst (LoadUB mem));
8190
8191 ins_cost(VOLATILE_REF_COST);
8192 format %{ "ldarb $dst, $mem\t# byte" %}
8193
8194 ins_encode(aarch64_enc_ldarb(dst, mem));
8195
8196 ins_pipe(pipe_serial);
8197 %}
8198
8199 // Load Byte (8 bit unsigned) into long
8200 instruct loadUB2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
8201 %{
8202 match(Set dst (ConvI2L (LoadUB mem)));
8203
8204 ins_cost(VOLATILE_REF_COST);
8205 format %{ "ldarb $dst, $mem\t# byte" %}
8206
8207 ins_encode(aarch64_enc_ldarb(dst, mem));
8208
8209 ins_pipe(pipe_serial);
8210 %}
8211
8212 // Load Short (16 bit signed)
8213 instruct loadS_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
8214 %{
8215 match(Set dst (LoadS mem));
8216
8217 ins_cost(VOLATILE_REF_COST);
8218 format %{ "ldarshw $dst, $mem\t# short" %}
8219
8220 ins_encode(aarch64_enc_ldarshw(dst, mem));
8221
8222 ins_pipe(pipe_serial);
8223 %}
8224
8225 instruct loadUS_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
8226 %{
8227 match(Set dst (LoadUS mem));
8228
8229 ins_cost(VOLATILE_REF_COST);
8230 format %{ "ldarhw $dst, $mem\t# short" %}
8231
8232 ins_encode(aarch64_enc_ldarhw(dst, mem));
8233
8234 ins_pipe(pipe_serial);
8235 %}
8236
8237 // Load Short/Char (16 bit unsigned) into long
8238 instruct loadUS2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
8239 %{
8240 match(Set dst (ConvI2L (LoadUS mem)));
8241
8242 ins_cost(VOLATILE_REF_COST);
8243 format %{ "ldarh $dst, $mem\t# short" %}
8244
8245 ins_encode(aarch64_enc_ldarh(dst, mem));
8246
8247 ins_pipe(pipe_serial);
8248 %}
8249
8250 // Load Short/Char (16 bit signed) into long
8251 instruct loadS2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
8252 %{
8253 match(Set dst (ConvI2L (LoadS mem)));
8254
8255 ins_cost(VOLATILE_REF_COST);
8256 format %{ "ldarh $dst, $mem\t# short" %}
8257
8258 ins_encode(aarch64_enc_ldarsh(dst, mem));
8259
8260 ins_pipe(pipe_serial);
8261 %}
8262
8263 // Load Integer (32 bit signed)
8264 instruct loadI_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
8265 %{
8266 match(Set dst (LoadI mem));
8267
8268 ins_cost(VOLATILE_REF_COST);
8269 format %{ "ldarw $dst, $mem\t# int" %}
8270
8271 ins_encode(aarch64_enc_ldarw(dst, mem));
8272
8273 ins_pipe(pipe_serial);
8274 %}
8275
8276 // Load Integer (32 bit unsigned) into long
8277 instruct loadUI2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem, immL_32bits mask)
8278 %{
8279 match(Set dst (AndL (ConvI2L (LoadI mem)) mask));
8280
8281 ins_cost(VOLATILE_REF_COST);
8282 format %{ "ldarw $dst, $mem\t# int" %}
8283
8284 ins_encode(aarch64_enc_ldarw(dst, mem));
8285
8286 ins_pipe(pipe_serial);
8287 %}
8288
8289 // Load Long (64 bit signed)
8290 instruct loadL_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
8291 %{
8292 match(Set dst (LoadL mem));
8293
8294 ins_cost(VOLATILE_REF_COST);
8295 format %{ "ldar $dst, $mem\t# int" %}
8296
8297 ins_encode(aarch64_enc_ldar(dst, mem));
8298
8299 ins_pipe(pipe_serial);
8300 %}
8301
8302 // Load Pointer
8303 instruct loadP_volatile(iRegPNoSp dst, /* sync_memory*/indirect mem)
8304 %{
8305 match(Set dst (LoadP mem));
8306 predicate(n->as_Load()->barrier_data() == 0);
8307
8308 ins_cost(VOLATILE_REF_COST);
8309 format %{ "ldar $dst, $mem\t# ptr" %}
8310
8311 ins_encode(aarch64_enc_ldar(dst, mem));
8312
8313 ins_pipe(pipe_serial);
8314 %}
8315
8316 // Load Compressed Pointer
8317 instruct loadN_volatile(iRegNNoSp dst, /* sync_memory*/indirect mem)
8318 %{
8319 match(Set dst (LoadN mem));
8320
8321 ins_cost(VOLATILE_REF_COST);
8322 format %{ "ldarw $dst, $mem\t# compressed ptr" %}
8323
8324 ins_encode(aarch64_enc_ldarw(dst, mem));
8325
8326 ins_pipe(pipe_serial);
8327 %}
8328
8329 // Load Float
8330 instruct loadF_volatile(vRegF dst, /* sync_memory*/indirect mem)
8331 %{
8332 match(Set dst (LoadF mem));
8333
8334 ins_cost(VOLATILE_REF_COST);
8335 format %{ "ldars $dst, $mem\t# float" %}
8336
8337 ins_encode( aarch64_enc_fldars(dst, mem) );
8338
8339 ins_pipe(pipe_serial);
8340 %}
8341
8342 // Load Double
8343 instruct loadD_volatile(vRegD dst, /* sync_memory*/indirect mem)
8344 %{
8345 match(Set dst (LoadD mem));
8346
8347 ins_cost(VOLATILE_REF_COST);
8348 format %{ "ldard $dst, $mem\t# double" %}
8349
8350 ins_encode( aarch64_enc_fldard(dst, mem) );
8351
8352 ins_pipe(pipe_serial);
8353 %}
8354
8355 // Store Byte
8356 instruct storeB_volatile(iRegIorL2I src, /* sync_memory*/indirect mem)
8357 %{
8358 match(Set mem (StoreB mem src));
8359
8360 ins_cost(VOLATILE_REF_COST);
8361 format %{ "stlrb $src, $mem\t# byte" %}
8362
8363 ins_encode(aarch64_enc_stlrb(src, mem));
8364
8365 ins_pipe(pipe_class_memory);
8366 %}
8367
8368 instruct storeimmB0_volatile(immI0 zero, /* sync_memory*/indirect mem)
8369 %{
8370 match(Set mem (StoreB mem zero));
8371
8372 ins_cost(VOLATILE_REF_COST);
8373 format %{ "stlrb zr, $mem\t# byte" %}
8374
8375 ins_encode(aarch64_enc_stlrb0(mem));
8376
8377 ins_pipe(pipe_class_memory);
8378 %}
8379
8380 // Store Char/Short
8381 instruct storeC_volatile(iRegIorL2I src, /* sync_memory*/indirect mem)
8382 %{
8383 match(Set mem (StoreC mem src));
8384
8385 ins_cost(VOLATILE_REF_COST);
8386 format %{ "stlrh $src, $mem\t# short" %}
8387
8388 ins_encode(aarch64_enc_stlrh(src, mem));
8389
8390 ins_pipe(pipe_class_memory);
8391 %}
8392
8393 instruct storeimmC0_volatile(immI0 zero, /* sync_memory*/indirect mem)
8394 %{
8395 match(Set mem (StoreC mem zero));
8396
8397 ins_cost(VOLATILE_REF_COST);
8398 format %{ "stlrh zr, $mem\t# short" %}
8399
8400 ins_encode(aarch64_enc_stlrh0(mem));
8401
8402 ins_pipe(pipe_class_memory);
8403 %}
8404
8405 // Store Integer
8406
8407 instruct storeI_volatile(iRegIorL2I src, /* sync_memory*/indirect mem)
8408 %{
8409 match(Set mem(StoreI mem src));
8410
8411 ins_cost(VOLATILE_REF_COST);
8412 format %{ "stlrw $src, $mem\t# int" %}
8413
8414 ins_encode(aarch64_enc_stlrw(src, mem));
8415
8416 ins_pipe(pipe_class_memory);
8417 %}
8418
8419 instruct storeimmI0_volatile(immI0 zero, /* sync_memory*/indirect mem)
8420 %{
8421 match(Set mem(StoreI mem zero));
8422
8423 ins_cost(VOLATILE_REF_COST);
8424 format %{ "stlrw zr, $mem\t# int" %}
8425
8426 ins_encode(aarch64_enc_stlrw0(mem));
8427
8428 ins_pipe(pipe_class_memory);
8429 %}
8430
8431 // Store Long (64 bit signed)
8432 instruct storeL_volatile(iRegL src, /* sync_memory*/indirect mem)
8433 %{
8434 match(Set mem (StoreL mem src));
8435
8436 ins_cost(VOLATILE_REF_COST);
8437 format %{ "stlr $src, $mem\t# int" %}
8438
8439 ins_encode(aarch64_enc_stlr(src, mem));
8440
8441 ins_pipe(pipe_class_memory);
8442 %}
8443
8444 instruct storeimmL0_volatile(immL0 zero, /* sync_memory*/indirect mem)
8445 %{
8446 match(Set mem (StoreL mem zero));
8447
8448 ins_cost(VOLATILE_REF_COST);
8449 format %{ "stlr zr, $mem\t# int" %}
8450
8451 ins_encode(aarch64_enc_stlr0(mem));
8452
8453 ins_pipe(pipe_class_memory);
8454 %}
8455
8456 // Store Pointer
8457 instruct storeP_volatile(iRegP src, /* sync_memory*/indirect mem)
8458 %{
8459 match(Set mem (StoreP mem src));
8460
8461 ins_cost(VOLATILE_REF_COST);
8462 format %{ "stlr $src, $mem\t# ptr" %}
8463
8464 ins_encode(aarch64_enc_stlr(src, mem));
8465
8466 ins_pipe(pipe_class_memory);
8467 %}
8468
8469 instruct storeimmP0_volatile(immP0 zero, /* sync_memory*/indirect mem)
8470 %{
8471 match(Set mem (StoreP mem zero));
8472
8473 ins_cost(VOLATILE_REF_COST);
8474 format %{ "stlr zr, $mem\t# ptr" %}
8475
8476 ins_encode(aarch64_enc_stlr0(mem));
8477
8478 ins_pipe(pipe_class_memory);
8479 %}
8480
8481 // Store Compressed Pointer
8482 instruct storeN_volatile(iRegN src, /* sync_memory*/indirect mem)
8483 %{
8484 match(Set mem (StoreN mem src));
8485
8486 ins_cost(VOLATILE_REF_COST);
8487 format %{ "stlrw $src, $mem\t# compressed ptr" %}
8488
8489 ins_encode(aarch64_enc_stlrw(src, mem));
8490
8491 ins_pipe(pipe_class_memory);
8492 %}
8493
8494 instruct storeimmN0_volatile(immN0 zero, /* sync_memory*/indirect mem)
8495 %{
8496 match(Set mem (StoreN mem zero));
8497
8498 ins_cost(VOLATILE_REF_COST);
8499 format %{ "stlrw zr, $mem\t# compressed ptr" %}
8500
8501 ins_encode(aarch64_enc_stlrw0(mem));
8502
8503 ins_pipe(pipe_class_memory);
8504 %}
8505
8506 // Store Float
8507 instruct storeF_volatile(vRegF src, /* sync_memory*/indirect mem)
8508 %{
8509 match(Set mem (StoreF mem src));
8510
8511 ins_cost(VOLATILE_REF_COST);
8512 format %{ "stlrs $src, $mem\t# float" %}
8513
8514 ins_encode( aarch64_enc_fstlrs(src, mem) );
8515
8516 ins_pipe(pipe_class_memory);
8517 %}
8518
8519 // TODO
8520 // implement storeImmF0 and storeFImmPacked
8521
8522 // Store Double
8523 instruct storeD_volatile(vRegD src, /* sync_memory*/indirect mem)
8524 %{
8525 match(Set mem (StoreD mem src));
8526
8527 ins_cost(VOLATILE_REF_COST);
8528 format %{ "stlrd $src, $mem\t# double" %}
8529
8530 ins_encode( aarch64_enc_fstlrd(src, mem) );
8531
8532 ins_pipe(pipe_class_memory);
8533 %}
8534
8535 // ---------------- end of volatile loads and stores ----------------
8536
8537 instruct cacheWB(indirect addr)
8538 %{
8539 predicate(VM_Version::supports_data_cache_line_flush());
8540 match(CacheWB addr);
8541
8542 ins_cost(100);
8543 format %{"cache wb $addr" %}
8544 ins_encode %{
8545 assert($addr->index_position() < 0, "should be");
8546 assert($addr$$disp == 0, "should be");
8547 __ cache_wb(Address($addr$$base$$Register, 0));
8548 %}
8549 ins_pipe(pipe_slow); // XXX
8550 %}
8551
8552 instruct cacheWBPreSync()
8553 %{
8554 predicate(VM_Version::supports_data_cache_line_flush());
8555 match(CacheWBPreSync);
8556
8557 ins_cost(100);
8558 format %{"cache wb presync" %}
8559 ins_encode %{
8560 __ cache_wbsync(true);
8561 %}
8562 ins_pipe(pipe_slow); // XXX
8563 %}
8564
8565 instruct cacheWBPostSync()
8566 %{
8567 predicate(VM_Version::supports_data_cache_line_flush());
8568 match(CacheWBPostSync);
8569
8570 ins_cost(100);
8571 format %{"cache wb postsync" %}
8572 ins_encode %{
8573 __ cache_wbsync(false);
8574 %}
8575 ins_pipe(pipe_slow); // XXX
8576 %}
8577
8578 // ============================================================================
8579 // BSWAP Instructions
8580
8581 instruct bytes_reverse_int(iRegINoSp dst, iRegIorL2I src) %{
8582 match(Set dst (ReverseBytesI src));
8583
8584 ins_cost(INSN_COST);
8585 format %{ "revw $dst, $src" %}
8586
8587 ins_encode %{
8588 __ revw(as_Register($dst$$reg), as_Register($src$$reg));
8589 %}
8590
8591 ins_pipe(ialu_reg);
8592 %}
8593
8594 instruct bytes_reverse_long(iRegLNoSp dst, iRegL src) %{
8595 match(Set dst (ReverseBytesL src));
8596
8597 ins_cost(INSN_COST);
8598 format %{ "rev $dst, $src" %}
8599
8600 ins_encode %{
8601 __ rev(as_Register($dst$$reg), as_Register($src$$reg));
8602 %}
8603
8604 ins_pipe(ialu_reg);
8605 %}
8606
8607 instruct bytes_reverse_unsigned_short(iRegINoSp dst, iRegIorL2I src) %{
8608 match(Set dst (ReverseBytesUS src));
8609
8610 ins_cost(INSN_COST);
8611 format %{ "rev16w $dst, $src" %}
8612
8613 ins_encode %{
8614 __ rev16w(as_Register($dst$$reg), as_Register($src$$reg));
8615 %}
8616
8617 ins_pipe(ialu_reg);
8618 %}
8619
8620 instruct bytes_reverse_short(iRegINoSp dst, iRegIorL2I src) %{
8621 match(Set dst (ReverseBytesS src));
8622
8623 ins_cost(INSN_COST);
8624 format %{ "rev16w $dst, $src\n\t"
8625 "sbfmw $dst, $dst, #0, #15" %}
8626
8627 ins_encode %{
8628 __ rev16w(as_Register($dst$$reg), as_Register($src$$reg));
8629 __ sbfmw(as_Register($dst$$reg), as_Register($dst$$reg), 0U, 15U);
8630 %}
8631
8632 ins_pipe(ialu_reg);
8633 %}
8634
8635 // ============================================================================
8636 // Zero Count Instructions
8637
8638 instruct countLeadingZerosI(iRegINoSp dst, iRegIorL2I src) %{
8639 match(Set dst (CountLeadingZerosI src));
8640
8641 ins_cost(INSN_COST);
8642 format %{ "clzw $dst, $src" %}
8643 ins_encode %{
8644 __ clzw(as_Register($dst$$reg), as_Register($src$$reg));
8645 %}
8646
8647 ins_pipe(ialu_reg);
8648 %}
8649
8650 instruct countLeadingZerosL(iRegINoSp dst, iRegL src) %{
8651 match(Set dst (CountLeadingZerosL src));
8652
8653 ins_cost(INSN_COST);
8654 format %{ "clz $dst, $src" %}
8655 ins_encode %{
8656 __ clz(as_Register($dst$$reg), as_Register($src$$reg));
8657 %}
8658
8659 ins_pipe(ialu_reg);
8660 %}
8661
8662 instruct countTrailingZerosI(iRegINoSp dst, iRegIorL2I src) %{
8663 match(Set dst (CountTrailingZerosI src));
8664
8665 ins_cost(INSN_COST * 2);
8666 format %{ "rbitw $dst, $src\n\t"
8667 "clzw $dst, $dst" %}
8668 ins_encode %{
8669 __ rbitw(as_Register($dst$$reg), as_Register($src$$reg));
8670 __ clzw(as_Register($dst$$reg), as_Register($dst$$reg));
8671 %}
8672
8673 ins_pipe(ialu_reg);
8674 %}
8675
8676 instruct countTrailingZerosL(iRegINoSp dst, iRegL src) %{
8677 match(Set dst (CountTrailingZerosL src));
8678
8679 ins_cost(INSN_COST * 2);
8680 format %{ "rbit $dst, $src\n\t"
8681 "clz $dst, $dst" %}
8682 ins_encode %{
8683 __ rbit(as_Register($dst$$reg), as_Register($src$$reg));
8684 __ clz(as_Register($dst$$reg), as_Register($dst$$reg));
8685 %}
8686
8687 ins_pipe(ialu_reg);
8688 %}
8689
8690 //---------- Population Count Instructions -------------------------------------
8691 //
8692
8693 instruct popCountI(iRegINoSp dst, iRegIorL2I src, vRegF tmp) %{
8694 predicate(UsePopCountInstruction);
8695 match(Set dst (PopCountI src));
8696 effect(TEMP tmp);
8697 ins_cost(INSN_COST * 13);
8698
8699 format %{ "movw $src, $src\n\t"
8700 "mov $tmp, $src\t# vector (1D)\n\t"
8701 "cnt $tmp, $tmp\t# vector (8B)\n\t"
8702 "addv $tmp, $tmp\t# vector (8B)\n\t"
8703 "mov $dst, $tmp\t# vector (1D)" %}
8704 ins_encode %{
8705 __ movw($src$$Register, $src$$Register); // ensure top 32 bits 0
8706 __ mov($tmp$$FloatRegister, __ D, 0, $src$$Register);
8707 __ cnt($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
8708 __ addv($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
8709 __ mov($dst$$Register, $tmp$$FloatRegister, __ D, 0);
8710 %}
8711
8712 ins_pipe(pipe_class_default);
8713 %}
8714
8715 instruct popCountI_mem(iRegINoSp dst, memory4 mem, vRegF tmp) %{
8716 predicate(UsePopCountInstruction);
8717 match(Set dst (PopCountI (LoadI mem)));
8718 effect(TEMP tmp);
8719 ins_cost(INSN_COST * 13);
8720
8721 format %{ "ldrs $tmp, $mem\n\t"
8722 "cnt $tmp, $tmp\t# vector (8B)\n\t"
8723 "addv $tmp, $tmp\t# vector (8B)\n\t"
8724 "mov $dst, $tmp\t# vector (1D)" %}
8725 ins_encode %{
8726 FloatRegister tmp_reg = as_FloatRegister($tmp$$reg);
8727 loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrs, tmp_reg, $mem->opcode(),
8728 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
8729 __ cnt($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
8730 __ addv($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
8731 __ mov($dst$$Register, $tmp$$FloatRegister, __ D, 0);
8732 %}
8733
8734 ins_pipe(pipe_class_default);
8735 %}
8736
8737 // Note: Long.bitCount(long) returns an int.
8738 instruct popCountL(iRegINoSp dst, iRegL src, vRegD tmp) %{
8739 predicate(UsePopCountInstruction);
8740 match(Set dst (PopCountL src));
8741 effect(TEMP tmp);
8742 ins_cost(INSN_COST * 13);
8743
8744 format %{ "mov $tmp, $src\t# vector (1D)\n\t"
8745 "cnt $tmp, $tmp\t# vector (8B)\n\t"
8746 "addv $tmp, $tmp\t# vector (8B)\n\t"
8747 "mov $dst, $tmp\t# vector (1D)" %}
8748 ins_encode %{
8749 __ mov($tmp$$FloatRegister, __ D, 0, $src$$Register);
8750 __ cnt($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
8751 __ addv($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
8752 __ mov($dst$$Register, $tmp$$FloatRegister, __ D, 0);
8753 %}
8754
8755 ins_pipe(pipe_class_default);
8756 %}
8757
8758 instruct popCountL_mem(iRegINoSp dst, memory8 mem, vRegD tmp) %{
8759 predicate(UsePopCountInstruction);
8760 match(Set dst (PopCountL (LoadL mem)));
8761 effect(TEMP tmp);
8762 ins_cost(INSN_COST * 13);
8763
8764 format %{ "ldrd $tmp, $mem\n\t"
8765 "cnt $tmp, $tmp\t# vector (8B)\n\t"
8766 "addv $tmp, $tmp\t# vector (8B)\n\t"
8767 "mov $dst, $tmp\t# vector (1D)" %}
8768 ins_encode %{
8769 FloatRegister tmp_reg = as_FloatRegister($tmp$$reg);
8770 loadStore(C2_MacroAssembler(&cbuf), &MacroAssembler::ldrd, tmp_reg, $mem->opcode(),
8771 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
8772 __ cnt($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
8773 __ addv($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
8774 __ mov($dst$$Register, $tmp$$FloatRegister, __ D, 0);
8775 %}
8776
8777 ins_pipe(pipe_class_default);
8778 %}
8779
8780 // ============================================================================
8781 // MemBar Instruction
8782
8783 instruct load_fence() %{
8784 match(LoadFence);
8785 ins_cost(VOLATILE_REF_COST);
8786
8787 format %{ "load_fence" %}
8788
8789 ins_encode %{
8790 __ membar(Assembler::LoadLoad|Assembler::LoadStore);
8791 %}
8792 ins_pipe(pipe_serial);
8793 %}
8794
8795 instruct unnecessary_membar_acquire() %{
8796 predicate(unnecessary_acquire(n));
8797 match(MemBarAcquire);
8798 ins_cost(0);
8799
8800 format %{ "membar_acquire (elided)" %}
8801
8802 ins_encode %{
8803 __ block_comment("membar_acquire (elided)");
8804 %}
8805
8806 ins_pipe(pipe_class_empty);
8807 %}
8808
8809 instruct membar_acquire() %{
8810 match(MemBarAcquire);
8811 ins_cost(VOLATILE_REF_COST);
8812
8813 format %{ "membar_acquire\n\t"
8814 "dmb ish" %}
8815
8816 ins_encode %{
8817 __ block_comment("membar_acquire");
8818 __ membar(Assembler::LoadLoad|Assembler::LoadStore);
8819 %}
8820
8821 ins_pipe(pipe_serial);
8822 %}
8823
8824
8825 instruct membar_acquire_lock() %{
8826 match(MemBarAcquireLock);
8827 ins_cost(VOLATILE_REF_COST);
8828
8829 format %{ "membar_acquire_lock (elided)" %}
8830
8831 ins_encode %{
8832 __ block_comment("membar_acquire_lock (elided)");
8833 %}
8834
8835 ins_pipe(pipe_serial);
8836 %}
8837
8838 instruct store_fence() %{
8839 match(StoreFence);
8840 ins_cost(VOLATILE_REF_COST);
8841
8842 format %{ "store_fence" %}
8843
8844 ins_encode %{
8845 __ membar(Assembler::LoadStore|Assembler::StoreStore);
8846 %}
8847 ins_pipe(pipe_serial);
8848 %}
8849
8850 instruct unnecessary_membar_release() %{
8851 predicate(unnecessary_release(n));
8852 match(MemBarRelease);
8853 ins_cost(0);
8854
8855 format %{ "membar_release (elided)" %}
8856
8857 ins_encode %{
8858 __ block_comment("membar_release (elided)");
8859 %}
8860 ins_pipe(pipe_serial);
8861 %}
8862
8863 instruct membar_release() %{
8864 match(MemBarRelease);
8865 ins_cost(VOLATILE_REF_COST);
8866
8867 format %{ "membar_release\n\t"
8868 "dmb ish" %}
8869
8870 ins_encode %{
8871 __ block_comment("membar_release");
8872 __ membar(Assembler::LoadStore|Assembler::StoreStore);
8873 %}
8874 ins_pipe(pipe_serial);
8875 %}
8876
8877 instruct membar_storestore() %{
8878 match(MemBarStoreStore);
8879 match(StoreStoreFence);
8880 ins_cost(VOLATILE_REF_COST);
8881
8882 format %{ "MEMBAR-store-store" %}
8883
8884 ins_encode %{
8885 __ membar(Assembler::StoreStore);
8886 %}
8887 ins_pipe(pipe_serial);
8888 %}
8889
8890 instruct membar_release_lock() %{
8891 match(MemBarReleaseLock);
8892 ins_cost(VOLATILE_REF_COST);
8893
8894 format %{ "membar_release_lock (elided)" %}
8895
8896 ins_encode %{
8897 __ block_comment("membar_release_lock (elided)");
8898 %}
8899
8900 ins_pipe(pipe_serial);
8901 %}
8902
8903 instruct unnecessary_membar_volatile() %{
8904 predicate(unnecessary_volatile(n));
8905 match(MemBarVolatile);
8906 ins_cost(0);
8907
8908 format %{ "membar_volatile (elided)" %}
8909
8910 ins_encode %{
8911 __ block_comment("membar_volatile (elided)");
8912 %}
8913
8914 ins_pipe(pipe_serial);
8915 %}
8916
8917 instruct membar_volatile() %{
8918 match(MemBarVolatile);
8919 ins_cost(VOLATILE_REF_COST*100);
8920
8921 format %{ "membar_volatile\n\t"
8922 "dmb ish"%}
8923
8924 ins_encode %{
8925 __ block_comment("membar_volatile");
8926 __ membar(Assembler::StoreLoad);
8927 %}
8928
8929 ins_pipe(pipe_serial);
8930 %}
8931
8932 // ============================================================================
8933 // Cast/Convert Instructions
8934
8935 instruct castX2P(iRegPNoSp dst, iRegL src) %{
8936 match(Set dst (CastX2P src));
8937
8938 ins_cost(INSN_COST);
8939 format %{ "mov $dst, $src\t# long -> ptr" %}
8940
8941 ins_encode %{
8942 if ($dst$$reg != $src$$reg) {
8943 __ mov(as_Register($dst$$reg), as_Register($src$$reg));
8944 }
8945 %}
8946
8947 ins_pipe(ialu_reg);
8948 %}
8949
8950 instruct castN2X(iRegLNoSp dst, iRegN src) %{
8951 match(Set dst (CastP2X src));
8952
8953 ins_cost(INSN_COST);
8954 format %{ "mov $dst, $src\t# ptr -> long" %}
8955
8956 ins_encode %{
8957 if ($dst$$reg != $src$$reg) {
8958 __ mov(as_Register($dst$$reg), as_Register($src$$reg));
8959 }
8960 %}
8961
8962 ins_pipe(ialu_reg);
8963 %}
8964
8965 instruct castP2X(iRegLNoSp dst, iRegP src) %{
8966 match(Set dst (CastP2X src));
8967
8968 ins_cost(INSN_COST);
8969 format %{ "mov $dst, $src\t# ptr -> long" %}
8970
8971 ins_encode %{
8972 if ($dst$$reg != $src$$reg) {
8973 __ mov(as_Register($dst$$reg), as_Register($src$$reg));
8974 }
8975 %}
8976
8977 ins_pipe(ialu_reg);
8978 %}
8979
8980 // Convert oop into int for vectors alignment masking
8981 instruct convP2I(iRegINoSp dst, iRegP src) %{
8982 match(Set dst (ConvL2I (CastP2X src)));
8983
8984 ins_cost(INSN_COST);
8985 format %{ "movw $dst, $src\t# ptr -> int" %}
8986 ins_encode %{
8987 __ movw($dst$$Register, $src$$Register);
8988 %}
8989
8990 ins_pipe(ialu_reg);
8991 %}
8992
8993 // Convert compressed oop into int for vectors alignment masking
8994 // in case of 32bit oops (heap < 4Gb).
8995 instruct convN2I(iRegINoSp dst, iRegN src)
8996 %{
8997 predicate(CompressedOops::shift() == 0);
8998 match(Set dst (ConvL2I (CastP2X (DecodeN src))));
8999
9000 ins_cost(INSN_COST);
9001 format %{ "mov dst, $src\t# compressed ptr -> int" %}
9002 ins_encode %{
9003 __ movw($dst$$Register, $src$$Register);
9004 %}
9005
9006 ins_pipe(ialu_reg);
9007 %}
9008
9009
9010 // Convert oop pointer into compressed form
9011 instruct encodeHeapOop(iRegNNoSp dst, iRegP src, rFlagsReg cr) %{
9012 predicate(n->bottom_type()->make_ptr()->ptr() != TypePtr::NotNull);
9013 match(Set dst (EncodeP src));
9014 effect(KILL cr);
9015 ins_cost(INSN_COST * 3);
9016 format %{ "encode_heap_oop $dst, $src" %}
9017 ins_encode %{
9018 Register s = $src$$Register;
9019 Register d = $dst$$Register;
9020 __ encode_heap_oop(d, s);
9021 %}
9022 ins_pipe(ialu_reg);
9023 %}
9024
9025 instruct encodeHeapOop_not_null(iRegNNoSp dst, iRegP src, rFlagsReg cr) %{
9026 predicate(n->bottom_type()->make_ptr()->ptr() == TypePtr::NotNull);
9027 match(Set dst (EncodeP src));
9028 ins_cost(INSN_COST * 3);
9029 format %{ "encode_heap_oop_not_null $dst, $src" %}
9030 ins_encode %{
9031 __ encode_heap_oop_not_null($dst$$Register, $src$$Register);
9032 %}
9033 ins_pipe(ialu_reg);
9034 %}
9035
9036 instruct decodeHeapOop(iRegPNoSp dst, iRegN src, rFlagsReg cr) %{
9037 predicate(n->bottom_type()->is_ptr()->ptr() != TypePtr::NotNull &&
9038 n->bottom_type()->is_ptr()->ptr() != TypePtr::Constant);
9039 match(Set dst (DecodeN src));
9040 ins_cost(INSN_COST * 3);
9041 format %{ "decode_heap_oop $dst, $src" %}
9042 ins_encode %{
9043 Register s = $src$$Register;
9044 Register d = $dst$$Register;
9045 __ decode_heap_oop(d, s);
9046 %}
9047 ins_pipe(ialu_reg);
9048 %}
9049
9050 instruct decodeHeapOop_not_null(iRegPNoSp dst, iRegN src, rFlagsReg cr) %{
9051 predicate(n->bottom_type()->is_ptr()->ptr() == TypePtr::NotNull ||
9052 n->bottom_type()->is_ptr()->ptr() == TypePtr::Constant);
9053 match(Set dst (DecodeN src));
9054 ins_cost(INSN_COST * 3);
9055 format %{ "decode_heap_oop_not_null $dst, $src" %}
9056 ins_encode %{
9057 Register s = $src$$Register;
9058 Register d = $dst$$Register;
9059 __ decode_heap_oop_not_null(d, s);
9060 %}
9061 ins_pipe(ialu_reg);
9062 %}
9063
9064 // n.b. AArch64 implementations of encode_klass_not_null and
9065 // decode_klass_not_null do not modify the flags register so, unlike
9066 // Intel, we don't kill CR as a side effect here
9067
9068 instruct encodeKlass_not_null(iRegNNoSp dst, iRegP src) %{
9069 match(Set dst (EncodePKlass src));
9070
9071 ins_cost(INSN_COST * 3);
9072 format %{ "encode_klass_not_null $dst,$src" %}
9073
9074 ins_encode %{
9075 Register src_reg = as_Register($src$$reg);
9076 Register dst_reg = as_Register($dst$$reg);
9077 __ encode_klass_not_null(dst_reg, src_reg);
9078 %}
9079
9080 ins_pipe(ialu_reg);
9081 %}
9082
9083 instruct decodeKlass_not_null(iRegPNoSp dst, iRegN src) %{
9084 match(Set dst (DecodeNKlass src));
9085
9086 ins_cost(INSN_COST * 3);
9087 format %{ "decode_klass_not_null $dst,$src" %}
9088
9089 ins_encode %{
9090 Register src_reg = as_Register($src$$reg);
9091 Register dst_reg = as_Register($dst$$reg);
9092 if (dst_reg != src_reg) {
9093 __ decode_klass_not_null(dst_reg, src_reg);
9094 } else {
9095 __ decode_klass_not_null(dst_reg);
9096 }
9097 %}
9098
9099 ins_pipe(ialu_reg);
9100 %}
9101
9102 instruct checkCastPP(iRegPNoSp dst)
9103 %{
9104 match(Set dst (CheckCastPP dst));
9105
9106 size(0);
9107 format %{ "# checkcastPP of $dst" %}
9108 ins_encode(/* empty encoding */);
9109 ins_pipe(pipe_class_empty);
9110 %}
9111
9112 instruct castPP(iRegPNoSp dst)
9113 %{
9114 match(Set dst (CastPP dst));
9115
9116 size(0);
9117 format %{ "# castPP of $dst" %}
9118 ins_encode(/* empty encoding */);
9119 ins_pipe(pipe_class_empty);
9120 %}
9121
9122 instruct castII(iRegI dst)
9123 %{
9124 match(Set dst (CastII dst));
9125
9126 size(0);
9127 format %{ "# castII of $dst" %}
9128 ins_encode(/* empty encoding */);
9129 ins_cost(0);
9130 ins_pipe(pipe_class_empty);
9131 %}
9132
9133 instruct castLL(iRegL dst)
9134 %{
9135 match(Set dst (CastLL dst));
9136
9137 size(0);
9138 format %{ "# castLL of $dst" %}
9139 ins_encode(/* empty encoding */);
9140 ins_cost(0);
9141 ins_pipe(pipe_class_empty);
9142 %}
9143
9144 instruct castFF(vRegF dst)
9145 %{
9146 match(Set dst (CastFF dst));
9147
9148 size(0);
9149 format %{ "# castFF of $dst" %}
9150 ins_encode(/* empty encoding */);
9151 ins_cost(0);
9152 ins_pipe(pipe_class_empty);
9153 %}
9154
9155 instruct castDD(vRegD dst)
9156 %{
9157 match(Set dst (CastDD dst));
9158
9159 size(0);
9160 format %{ "# castDD of $dst" %}
9161 ins_encode(/* empty encoding */);
9162 ins_cost(0);
9163 ins_pipe(pipe_class_empty);
9164 %}
9165
9166 instruct castVVD(vecD dst)
9167 %{
9168 match(Set dst (CastVV dst));
9169
9170 size(0);
9171 format %{ "# castVV of $dst" %}
9172 ins_encode(/* empty encoding */);
9173 ins_cost(0);
9174 ins_pipe(pipe_class_empty);
9175 %}
9176
9177 instruct castVVX(vecX dst)
9178 %{
9179 match(Set dst (CastVV dst));
9180
9181 size(0);
9182 format %{ "# castVV of $dst" %}
9183 ins_encode(/* empty encoding */);
9184 ins_cost(0);
9185 ins_pipe(pipe_class_empty);
9186 %}
9187
9188 instruct castVV(vReg dst)
9189 %{
9190 match(Set dst (CastVV dst));
9191
9192 size(0);
9193 format %{ "# castVV of $dst" %}
9194 ins_encode(/* empty encoding */);
9195 ins_cost(0);
9196 ins_pipe(pipe_class_empty);
9197 %}
9198
9199 instruct castVVMask(pRegGov dst)
9200 %{
9201 match(Set dst (CastVV dst));
9202
9203 size(0);
9204 format %{ "# castVV of $dst" %}
9205 ins_encode(/* empty encoding */);
9206 ins_cost(0);
9207 ins_pipe(pipe_class_empty);
9208 %}
9209
9210 // ============================================================================
9211 // Atomic operation instructions
9212 //
9213 // Intel and SPARC both implement Ideal Node LoadPLocked and
9214 // Store{PIL}Conditional instructions using a normal load for the
9215 // LoadPLocked and a CAS for the Store{PIL}Conditional.
9216 //
9217 // The ideal code appears only to use LoadPLocked/StorePLocked as a
9218 // pair to lock object allocations from Eden space when not using
9219 // TLABs.
9220 //
9221 // There does not appear to be a Load{IL}Locked Ideal Node and the
9222 // Ideal code appears to use Store{IL}Conditional as an alias for CAS
9223 // and to use StoreIConditional only for 32-bit and StoreLConditional
9224 // only for 64-bit.
9225 //
9226 // We implement LoadPLocked and StorePLocked instructions using,
9227 // respectively the AArch64 hw load-exclusive and store-conditional
9228 // instructions. Whereas we must implement each of
9229 // Store{IL}Conditional using a CAS which employs a pair of
9230 // instructions comprising a load-exclusive followed by a
9231 // store-conditional.
9232
9233
9234 // Locked-load (linked load) of the current heap-top
9235 // used when updating the eden heap top
9236 // implemented using ldaxr on AArch64
9237
9238 instruct loadPLocked(iRegPNoSp dst, indirect mem)
9239 %{
9240 match(Set dst (LoadPLocked mem));
9241
9242 ins_cost(VOLATILE_REF_COST);
9243
9244 format %{ "ldaxr $dst, $mem\t# ptr linked acquire" %}
9245
9246 ins_encode(aarch64_enc_ldaxr(dst, mem));
9247
9248 ins_pipe(pipe_serial);
9249 %}
9250
9251 // Conditional-store of the updated heap-top.
9252 // Used during allocation of the shared heap.
9253 // Sets flag (EQ) on success.
9254 // implemented using stlxr on AArch64.
9255
9256 instruct storePConditional(memory8 heap_top_ptr, iRegP oldval, iRegP newval, rFlagsReg cr)
9257 %{
9258 match(Set cr (StorePConditional heap_top_ptr (Binary oldval newval)));
9259
9260 ins_cost(VOLATILE_REF_COST);
9261
9262 // TODO
9263 // do we need to do a store-conditional release or can we just use a
9264 // plain store-conditional?
9265
9266 format %{
9267 "stlxr rscratch1, $newval, $heap_top_ptr\t# ptr cond release"
9268 "cmpw rscratch1, zr\t# EQ on successful write"
9269 %}
9270
9271 ins_encode(aarch64_enc_stlxr(newval, heap_top_ptr));
9272
9273 ins_pipe(pipe_serial);
9274 %}
9275
9276 instruct storeLConditional(indirect mem, iRegLNoSp oldval, iRegLNoSp newval, rFlagsReg cr)
9277 %{
9278 match(Set cr (StoreLConditional mem (Binary oldval newval)));
9279
9280 ins_cost(VOLATILE_REF_COST);
9281
9282 format %{
9283 "cmpxchg rscratch1, $mem, $oldval, $newval, $mem\t# if $mem == $oldval then $mem <-- $newval"
9284 "cmpw rscratch1, zr\t# EQ on successful write"
9285 %}
9286
9287 ins_encode(aarch64_enc_cmpxchg_acq(mem, oldval, newval));
9288
9289 ins_pipe(pipe_slow);
9290 %}
9291
9292 // storeIConditional also has acquire semantics, for no better reason
9293 // than matching storeLConditional. At the time of writing this
9294 // comment storeIConditional was not used anywhere by AArch64.
9295 instruct storeIConditional(indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr)
9296 %{
9297 match(Set cr (StoreIConditional mem (Binary oldval newval)));
9298
9299 ins_cost(VOLATILE_REF_COST);
9300
9301 format %{
9302 "cmpxchgw rscratch1, $mem, $oldval, $newval, $mem\t# if $mem == $oldval then $mem <-- $newval"
9303 "cmpw rscratch1, zr\t# EQ on successful write"
9304 %}
9305
9306 ins_encode(aarch64_enc_cmpxchgw_acq(mem, oldval, newval));
9307
9308 ins_pipe(pipe_slow);
9309 %}
9310
9311 // standard CompareAndSwapX when we are using barriers
9312 // these have higher priority than the rules selected by a predicate
9313
9314 // XXX No flag versions for CompareAndSwap{I,L,P,N} because matcher
9315 // can't match them
9316
9317 instruct compareAndSwapB(iRegINoSp res, indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr) %{
9318
9319 match(Set res (CompareAndSwapB mem (Binary oldval newval)));
9320 ins_cost(2 * VOLATILE_REF_COST);
9321
9322 effect(KILL cr);
9323
9324 format %{
9325 "cmpxchgb $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval"
9326 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
9327 %}
9328
9329 ins_encode(aarch64_enc_cmpxchgb(mem, oldval, newval),
9330 aarch64_enc_cset_eq(res));
9331
9332 ins_pipe(pipe_slow);
9333 %}
9334
9335 instruct compareAndSwapS(iRegINoSp res, indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr) %{
9336
9337 match(Set res (CompareAndSwapS mem (Binary oldval newval)));
9338 ins_cost(2 * VOLATILE_REF_COST);
9339
9340 effect(KILL cr);
9341
9342 format %{
9343 "cmpxchgs $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval"
9344 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
9345 %}
9346
9347 ins_encode(aarch64_enc_cmpxchgs(mem, oldval, newval),
9348 aarch64_enc_cset_eq(res));
9349
9350 ins_pipe(pipe_slow);
9351 %}
9352
9353 instruct compareAndSwapI(iRegINoSp res, indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr) %{
9354
9355 match(Set res (CompareAndSwapI mem (Binary oldval newval)));
9356 ins_cost(2 * VOLATILE_REF_COST);
9357
9358 effect(KILL cr);
9359
9360 format %{
9361 "cmpxchgw $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval"
9362 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
9363 %}
9364
9365 ins_encode(aarch64_enc_cmpxchgw(mem, oldval, newval),
9366 aarch64_enc_cset_eq(res));
9367
9368 ins_pipe(pipe_slow);
9369 %}
9370
9371 instruct compareAndSwapL(iRegINoSp res, indirect mem, iRegLNoSp oldval, iRegLNoSp newval, rFlagsReg cr) %{
9372
9373 match(Set res (CompareAndSwapL mem (Binary oldval newval)));
9374 ins_cost(2 * VOLATILE_REF_COST);
9375
9376 effect(KILL cr);
9377
9378 format %{
9379 "cmpxchg $mem, $oldval, $newval\t# (long) if $mem == $oldval then $mem <-- $newval"
9380 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
9381 %}
9382
9383 ins_encode(aarch64_enc_cmpxchg(mem, oldval, newval),
9384 aarch64_enc_cset_eq(res));
9385
9386 ins_pipe(pipe_slow);
9387 %}
9388
9389 instruct compareAndSwapP(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
9390
9391 match(Set res (CompareAndSwapP mem (Binary oldval newval)));
9392 predicate(n->as_LoadStore()->barrier_data() == 0);
9393 ins_cost(2 * VOLATILE_REF_COST);
9394
9395 effect(KILL cr);
9396
9397 format %{
9398 "cmpxchg $mem, $oldval, $newval\t# (ptr) if $mem == $oldval then $mem <-- $newval"
9399 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
9400 %}
9401
9402 ins_encode(aarch64_enc_cmpxchg(mem, oldval, newval),
9403 aarch64_enc_cset_eq(res));
9404
9405 ins_pipe(pipe_slow);
9406 %}
9407
9408 instruct compareAndSwapN(iRegINoSp res, indirect mem, iRegNNoSp oldval, iRegNNoSp newval, rFlagsReg cr) %{
9409
9410 match(Set res (CompareAndSwapN mem (Binary oldval newval)));
9411 ins_cost(2 * VOLATILE_REF_COST);
9412
9413 effect(KILL cr);
9414
9415 format %{
9416 "cmpxchgw $mem, $oldval, $newval\t# (narrow oop) if $mem == $oldval then $mem <-- $newval"
9417 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
9418 %}
9419
9420 ins_encode(aarch64_enc_cmpxchgw(mem, oldval, newval),
9421 aarch64_enc_cset_eq(res));
9422
9423 ins_pipe(pipe_slow);
9424 %}
9425
9426 // alternative CompareAndSwapX when we are eliding barriers
9427
9428 instruct compareAndSwapBAcq(iRegINoSp res, indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr) %{
9429
9430 predicate(needs_acquiring_load_exclusive(n));
9431 match(Set res (CompareAndSwapB mem (Binary oldval newval)));
9432 ins_cost(VOLATILE_REF_COST);
9433
9434 effect(KILL cr);
9435
9436 format %{
9437 "cmpxchgb_acq $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval"
9438 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
9439 %}
9440
9441 ins_encode(aarch64_enc_cmpxchgb_acq(mem, oldval, newval),
9442 aarch64_enc_cset_eq(res));
9443
9444 ins_pipe(pipe_slow);
9445 %}
9446
9447 instruct compareAndSwapSAcq(iRegINoSp res, indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr) %{
9448
9449 predicate(needs_acquiring_load_exclusive(n));
9450 match(Set res (CompareAndSwapS mem (Binary oldval newval)));
9451 ins_cost(VOLATILE_REF_COST);
9452
9453 effect(KILL cr);
9454
9455 format %{
9456 "cmpxchgs_acq $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval"
9457 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
9458 %}
9459
9460 ins_encode(aarch64_enc_cmpxchgs_acq(mem, oldval, newval),
9461 aarch64_enc_cset_eq(res));
9462
9463 ins_pipe(pipe_slow);
9464 %}
9465
9466 instruct compareAndSwapIAcq(iRegINoSp res, indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr) %{
9467
9468 predicate(needs_acquiring_load_exclusive(n));
9469 match(Set res (CompareAndSwapI mem (Binary oldval newval)));
9470 ins_cost(VOLATILE_REF_COST);
9471
9472 effect(KILL cr);
9473
9474 format %{
9475 "cmpxchgw_acq $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval"
9476 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
9477 %}
9478
9479 ins_encode(aarch64_enc_cmpxchgw_acq(mem, oldval, newval),
9480 aarch64_enc_cset_eq(res));
9481
9482 ins_pipe(pipe_slow);
9483 %}
9484
9485 instruct compareAndSwapLAcq(iRegINoSp res, indirect mem, iRegLNoSp oldval, iRegLNoSp newval, rFlagsReg cr) %{
9486
9487 predicate(needs_acquiring_load_exclusive(n));
9488 match(Set res (CompareAndSwapL mem (Binary oldval newval)));
9489 ins_cost(VOLATILE_REF_COST);
9490
9491 effect(KILL cr);
9492
9493 format %{
9494 "cmpxchg_acq $mem, $oldval, $newval\t# (long) if $mem == $oldval then $mem <-- $newval"
9495 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
9496 %}
9497
9498 ins_encode(aarch64_enc_cmpxchg_acq(mem, oldval, newval),
9499 aarch64_enc_cset_eq(res));
9500
9501 ins_pipe(pipe_slow);
9502 %}
9503
9504 instruct compareAndSwapPAcq(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
9505
9506 predicate(needs_acquiring_load_exclusive(n) && (n->as_LoadStore()->barrier_data() == 0));
9507 match(Set res (CompareAndSwapP mem (Binary oldval newval)));
9508 ins_cost(VOLATILE_REF_COST);
9509
9510 effect(KILL cr);
9511
9512 format %{
9513 "cmpxchg_acq $mem, $oldval, $newval\t# (ptr) if $mem == $oldval then $mem <-- $newval"
9514 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
9515 %}
9516
9517 ins_encode(aarch64_enc_cmpxchg_acq(mem, oldval, newval),
9518 aarch64_enc_cset_eq(res));
9519
9520 ins_pipe(pipe_slow);
9521 %}
9522
9523 instruct compareAndSwapNAcq(iRegINoSp res, indirect mem, iRegNNoSp oldval, iRegNNoSp newval, rFlagsReg cr) %{
9524
9525 predicate(needs_acquiring_load_exclusive(n));
9526 match(Set res (CompareAndSwapN mem (Binary oldval newval)));
9527 ins_cost(VOLATILE_REF_COST);
9528
9529 effect(KILL cr);
9530
9531 format %{
9532 "cmpxchgw_acq $mem, $oldval, $newval\t# (narrow oop) if $mem == $oldval then $mem <-- $newval"
9533 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
9534 %}
9535
9536 ins_encode(aarch64_enc_cmpxchgw_acq(mem, oldval, newval),
9537 aarch64_enc_cset_eq(res));
9538
9539 ins_pipe(pipe_slow);
9540 %}
9541
9542
9543 // ---------------------------------------------------------------------
9544
9545 // BEGIN This section of the file is automatically generated. Do not edit --------------
9546
9547 // Sundry CAS operations. Note that release is always true,
9548 // regardless of the memory ordering of the CAS. This is because we
9549 // need the volatile case to be sequentially consistent but there is
9550 // no trailing StoreLoad barrier emitted by C2. Unfortunately we
9551 // can't check the type of memory ordering here, so we always emit a
9552 // STLXR.
9553
9554 // This section is generated from aarch64_ad_cas.m4
9555
9556
9557
9558 // This pattern is generated automatically from cas.m4.
9559 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9560 instruct compareAndExchangeB(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
9561
9562 match(Set res (CompareAndExchangeB mem (Binary oldval newval)));
9563 ins_cost(2 * VOLATILE_REF_COST);
9564 effect(TEMP_DEF res, KILL cr);
9565 format %{
9566 "cmpxchgb $res = $mem, $oldval, $newval\t# (byte, weak) if $mem == $oldval then $mem <-- $newval"
9567 %}
9568 ins_encode %{
9569 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
9570 Assembler::byte, /*acquire*/ false, /*release*/ true,
9571 /*weak*/ false, $res$$Register);
9572 __ sxtbw($res$$Register, $res$$Register);
9573 %}
9574 ins_pipe(pipe_slow);
9575 %}
9576
9577 // This pattern is generated automatically from cas.m4.
9578 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9579 instruct compareAndExchangeS(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
9580
9581 match(Set res (CompareAndExchangeS mem (Binary oldval newval)));
9582 ins_cost(2 * VOLATILE_REF_COST);
9583 effect(TEMP_DEF res, KILL cr);
9584 format %{
9585 "cmpxchgs $res = $mem, $oldval, $newval\t# (short, weak) if $mem == $oldval then $mem <-- $newval"
9586 %}
9587 ins_encode %{
9588 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
9589 Assembler::halfword, /*acquire*/ false, /*release*/ true,
9590 /*weak*/ false, $res$$Register);
9591 __ sxthw($res$$Register, $res$$Register);
9592 %}
9593 ins_pipe(pipe_slow);
9594 %}
9595
9596 // This pattern is generated automatically from cas.m4.
9597 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9598 instruct compareAndExchangeI(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
9599
9600 match(Set res (CompareAndExchangeI mem (Binary oldval newval)));
9601 ins_cost(2 * VOLATILE_REF_COST);
9602 effect(TEMP_DEF res, KILL cr);
9603 format %{
9604 "cmpxchgw $res = $mem, $oldval, $newval\t# (int, weak) if $mem == $oldval then $mem <-- $newval"
9605 %}
9606 ins_encode %{
9607 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
9608 Assembler::word, /*acquire*/ false, /*release*/ true,
9609 /*weak*/ false, $res$$Register);
9610 %}
9611 ins_pipe(pipe_slow);
9612 %}
9613
9614 // This pattern is generated automatically from cas.m4.
9615 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9616 instruct compareAndExchangeL(iRegLNoSp res, indirect mem, iRegL oldval, iRegL newval, rFlagsReg cr) %{
9617
9618 match(Set res (CompareAndExchangeL mem (Binary oldval newval)));
9619 ins_cost(2 * VOLATILE_REF_COST);
9620 effect(TEMP_DEF res, KILL cr);
9621 format %{
9622 "cmpxchg $res = $mem, $oldval, $newval\t# (long, weak) if $mem == $oldval then $mem <-- $newval"
9623 %}
9624 ins_encode %{
9625 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
9626 Assembler::xword, /*acquire*/ false, /*release*/ true,
9627 /*weak*/ false, $res$$Register);
9628 %}
9629 ins_pipe(pipe_slow);
9630 %}
9631
9632 // This pattern is generated automatically from cas.m4.
9633 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9634 instruct compareAndExchangeN(iRegNNoSp res, indirect mem, iRegN oldval, iRegN newval, rFlagsReg cr) %{
9635
9636 match(Set res (CompareAndExchangeN mem (Binary oldval newval)));
9637 ins_cost(2 * VOLATILE_REF_COST);
9638 effect(TEMP_DEF res, KILL cr);
9639 format %{
9640 "cmpxchgw $res = $mem, $oldval, $newval\t# (narrow oop, weak) if $mem == $oldval then $mem <-- $newval"
9641 %}
9642 ins_encode %{
9643 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
9644 Assembler::word, /*acquire*/ false, /*release*/ true,
9645 /*weak*/ false, $res$$Register);
9646 %}
9647 ins_pipe(pipe_slow);
9648 %}
9649
9650 // This pattern is generated automatically from cas.m4.
9651 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9652 instruct compareAndExchangeP(iRegPNoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
9653 predicate(n->as_LoadStore()->barrier_data() == 0);
9654 match(Set res (CompareAndExchangeP mem (Binary oldval newval)));
9655 ins_cost(2 * VOLATILE_REF_COST);
9656 effect(TEMP_DEF res, KILL cr);
9657 format %{
9658 "cmpxchg $res = $mem, $oldval, $newval\t# (ptr, weak) if $mem == $oldval then $mem <-- $newval"
9659 %}
9660 ins_encode %{
9661 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
9662 Assembler::xword, /*acquire*/ false, /*release*/ true,
9663 /*weak*/ false, $res$$Register);
9664 %}
9665 ins_pipe(pipe_slow);
9666 %}
9667
9668 // This pattern is generated automatically from cas.m4.
9669 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9670 instruct compareAndExchangeBAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
9671 predicate(needs_acquiring_load_exclusive(n));
9672 match(Set res (CompareAndExchangeB mem (Binary oldval newval)));
9673 ins_cost(VOLATILE_REF_COST);
9674 effect(TEMP_DEF res, KILL cr);
9675 format %{
9676 "cmpxchgb_acq $res = $mem, $oldval, $newval\t# (byte, weak) if $mem == $oldval then $mem <-- $newval"
9677 %}
9678 ins_encode %{
9679 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
9680 Assembler::byte, /*acquire*/ true, /*release*/ true,
9681 /*weak*/ false, $res$$Register);
9682 __ sxtbw($res$$Register, $res$$Register);
9683 %}
9684 ins_pipe(pipe_slow);
9685 %}
9686
9687 // This pattern is generated automatically from cas.m4.
9688 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9689 instruct compareAndExchangeSAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
9690 predicate(needs_acquiring_load_exclusive(n));
9691 match(Set res (CompareAndExchangeS mem (Binary oldval newval)));
9692 ins_cost(VOLATILE_REF_COST);
9693 effect(TEMP_DEF res, KILL cr);
9694 format %{
9695 "cmpxchgs_acq $res = $mem, $oldval, $newval\t# (short, weak) if $mem == $oldval then $mem <-- $newval"
9696 %}
9697 ins_encode %{
9698 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
9699 Assembler::halfword, /*acquire*/ true, /*release*/ true,
9700 /*weak*/ false, $res$$Register);
9701 __ sxthw($res$$Register, $res$$Register);
9702 %}
9703 ins_pipe(pipe_slow);
9704 %}
9705
9706 // This pattern is generated automatically from cas.m4.
9707 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9708 instruct compareAndExchangeIAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
9709 predicate(needs_acquiring_load_exclusive(n));
9710 match(Set res (CompareAndExchangeI mem (Binary oldval newval)));
9711 ins_cost(VOLATILE_REF_COST);
9712 effect(TEMP_DEF res, KILL cr);
9713 format %{
9714 "cmpxchgw_acq $res = $mem, $oldval, $newval\t# (int, weak) if $mem == $oldval then $mem <-- $newval"
9715 %}
9716 ins_encode %{
9717 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
9718 Assembler::word, /*acquire*/ true, /*release*/ true,
9719 /*weak*/ false, $res$$Register);
9720 %}
9721 ins_pipe(pipe_slow);
9722 %}
9723
9724 // This pattern is generated automatically from cas.m4.
9725 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9726 instruct compareAndExchangeLAcq(iRegLNoSp res, indirect mem, iRegL oldval, iRegL newval, rFlagsReg cr) %{
9727 predicate(needs_acquiring_load_exclusive(n));
9728 match(Set res (CompareAndExchangeL mem (Binary oldval newval)));
9729 ins_cost(VOLATILE_REF_COST);
9730 effect(TEMP_DEF res, KILL cr);
9731 format %{
9732 "cmpxchg_acq $res = $mem, $oldval, $newval\t# (long, weak) if $mem == $oldval then $mem <-- $newval"
9733 %}
9734 ins_encode %{
9735 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
9736 Assembler::xword, /*acquire*/ true, /*release*/ true,
9737 /*weak*/ false, $res$$Register);
9738 %}
9739 ins_pipe(pipe_slow);
9740 %}
9741
9742 // This pattern is generated automatically from cas.m4.
9743 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9744 instruct compareAndExchangeNAcq(iRegNNoSp res, indirect mem, iRegN oldval, iRegN newval, rFlagsReg cr) %{
9745 predicate(needs_acquiring_load_exclusive(n));
9746 match(Set res (CompareAndExchangeN mem (Binary oldval newval)));
9747 ins_cost(VOLATILE_REF_COST);
9748 effect(TEMP_DEF res, KILL cr);
9749 format %{
9750 "cmpxchgw_acq $res = $mem, $oldval, $newval\t# (narrow oop, weak) if $mem == $oldval then $mem <-- $newval"
9751 %}
9752 ins_encode %{
9753 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
9754 Assembler::word, /*acquire*/ true, /*release*/ true,
9755 /*weak*/ false, $res$$Register);
9756 %}
9757 ins_pipe(pipe_slow);
9758 %}
9759
9760 // This pattern is generated automatically from cas.m4.
9761 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9762 instruct compareAndExchangePAcq(iRegPNoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
9763 predicate(needs_acquiring_load_exclusive(n) && (n->as_LoadStore()->barrier_data() == 0));
9764 match(Set res (CompareAndExchangeP mem (Binary oldval newval)));
9765 ins_cost(VOLATILE_REF_COST);
9766 effect(TEMP_DEF res, KILL cr);
9767 format %{
9768 "cmpxchg_acq $res = $mem, $oldval, $newval\t# (ptr, weak) if $mem == $oldval then $mem <-- $newval"
9769 %}
9770 ins_encode %{
9771 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
9772 Assembler::xword, /*acquire*/ true, /*release*/ true,
9773 /*weak*/ false, $res$$Register);
9774 %}
9775 ins_pipe(pipe_slow);
9776 %}
9777
9778 // This pattern is generated automatically from cas.m4.
9779 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9780 instruct weakCompareAndSwapB(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
9781
9782 match(Set res (WeakCompareAndSwapB mem (Binary oldval newval)));
9783 ins_cost(2 * VOLATILE_REF_COST);
9784 effect(KILL cr);
9785 format %{
9786 "cmpxchgb $res = $mem, $oldval, $newval\t# (byte, weak) if $mem == $oldval then $mem <-- $newval"
9787 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
9788 %}
9789 ins_encode %{
9790 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
9791 Assembler::byte, /*acquire*/ false, /*release*/ true,
9792 /*weak*/ true, noreg);
9793 __ csetw($res$$Register, Assembler::EQ);
9794 %}
9795 ins_pipe(pipe_slow);
9796 %}
9797
9798 // This pattern is generated automatically from cas.m4.
9799 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9800 instruct weakCompareAndSwapS(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
9801
9802 match(Set res (WeakCompareAndSwapS mem (Binary oldval newval)));
9803 ins_cost(2 * VOLATILE_REF_COST);
9804 effect(KILL cr);
9805 format %{
9806 "cmpxchgs $res = $mem, $oldval, $newval\t# (short, weak) if $mem == $oldval then $mem <-- $newval"
9807 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
9808 %}
9809 ins_encode %{
9810 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
9811 Assembler::halfword, /*acquire*/ false, /*release*/ true,
9812 /*weak*/ true, noreg);
9813 __ csetw($res$$Register, Assembler::EQ);
9814 %}
9815 ins_pipe(pipe_slow);
9816 %}
9817
9818 // This pattern is generated automatically from cas.m4.
9819 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9820 instruct weakCompareAndSwapI(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
9821
9822 match(Set res (WeakCompareAndSwapI mem (Binary oldval newval)));
9823 ins_cost(2 * VOLATILE_REF_COST);
9824 effect(KILL cr);
9825 format %{
9826 "cmpxchgw $res = $mem, $oldval, $newval\t# (int, weak) if $mem == $oldval then $mem <-- $newval"
9827 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
9828 %}
9829 ins_encode %{
9830 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
9831 Assembler::word, /*acquire*/ false, /*release*/ true,
9832 /*weak*/ true, noreg);
9833 __ csetw($res$$Register, Assembler::EQ);
9834 %}
9835 ins_pipe(pipe_slow);
9836 %}
9837
9838 // This pattern is generated automatically from cas.m4.
9839 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9840 instruct weakCompareAndSwapL(iRegINoSp res, indirect mem, iRegL oldval, iRegL newval, rFlagsReg cr) %{
9841
9842 match(Set res (WeakCompareAndSwapL mem (Binary oldval newval)));
9843 ins_cost(2 * VOLATILE_REF_COST);
9844 effect(KILL cr);
9845 format %{
9846 "cmpxchg $res = $mem, $oldval, $newval\t# (long, weak) if $mem == $oldval then $mem <-- $newval"
9847 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
9848 %}
9849 ins_encode %{
9850 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
9851 Assembler::xword, /*acquire*/ false, /*release*/ true,
9852 /*weak*/ true, noreg);
9853 __ csetw($res$$Register, Assembler::EQ);
9854 %}
9855 ins_pipe(pipe_slow);
9856 %}
9857
9858 // This pattern is generated automatically from cas.m4.
9859 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9860 instruct weakCompareAndSwapN(iRegINoSp res, indirect mem, iRegN oldval, iRegN newval, rFlagsReg cr) %{
9861
9862 match(Set res (WeakCompareAndSwapN mem (Binary oldval newval)));
9863 ins_cost(2 * VOLATILE_REF_COST);
9864 effect(KILL cr);
9865 format %{
9866 "cmpxchgw $res = $mem, $oldval, $newval\t# (narrow oop, weak) if $mem == $oldval then $mem <-- $newval"
9867 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
9868 %}
9869 ins_encode %{
9870 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
9871 Assembler::word, /*acquire*/ false, /*release*/ true,
9872 /*weak*/ true, noreg);
9873 __ csetw($res$$Register, Assembler::EQ);
9874 %}
9875 ins_pipe(pipe_slow);
9876 %}
9877
9878 // This pattern is generated automatically from cas.m4.
9879 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9880 instruct weakCompareAndSwapP(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
9881 predicate(n->as_LoadStore()->barrier_data() == 0);
9882 match(Set res (WeakCompareAndSwapP mem (Binary oldval newval)));
9883 ins_cost(2 * VOLATILE_REF_COST);
9884 effect(KILL cr);
9885 format %{
9886 "cmpxchg $res = $mem, $oldval, $newval\t# (ptr, weak) if $mem == $oldval then $mem <-- $newval"
9887 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
9888 %}
9889 ins_encode %{
9890 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
9891 Assembler::xword, /*acquire*/ false, /*release*/ true,
9892 /*weak*/ true, noreg);
9893 __ csetw($res$$Register, Assembler::EQ);
9894 %}
9895 ins_pipe(pipe_slow);
9896 %}
9897
9898 // This pattern is generated automatically from cas.m4.
9899 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9900 instruct weakCompareAndSwapBAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
9901 predicate(needs_acquiring_load_exclusive(n));
9902 match(Set res (WeakCompareAndSwapB mem (Binary oldval newval)));
9903 ins_cost(VOLATILE_REF_COST);
9904 effect(KILL cr);
9905 format %{
9906 "cmpxchgb_acq $res = $mem, $oldval, $newval\t# (byte, weak) if $mem == $oldval then $mem <-- $newval"
9907 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
9908 %}
9909 ins_encode %{
9910 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
9911 Assembler::byte, /*acquire*/ true, /*release*/ true,
9912 /*weak*/ true, noreg);
9913 __ csetw($res$$Register, Assembler::EQ);
9914 %}
9915 ins_pipe(pipe_slow);
9916 %}
9917
9918 // This pattern is generated automatically from cas.m4.
9919 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9920 instruct weakCompareAndSwapSAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
9921 predicate(needs_acquiring_load_exclusive(n));
9922 match(Set res (WeakCompareAndSwapS mem (Binary oldval newval)));
9923 ins_cost(VOLATILE_REF_COST);
9924 effect(KILL cr);
9925 format %{
9926 "cmpxchgs_acq $res = $mem, $oldval, $newval\t# (short, weak) if $mem == $oldval then $mem <-- $newval"
9927 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
9928 %}
9929 ins_encode %{
9930 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
9931 Assembler::halfword, /*acquire*/ true, /*release*/ true,
9932 /*weak*/ true, noreg);
9933 __ csetw($res$$Register, Assembler::EQ);
9934 %}
9935 ins_pipe(pipe_slow);
9936 %}
9937
9938 // This pattern is generated automatically from cas.m4.
9939 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9940 instruct weakCompareAndSwapIAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
9941 predicate(needs_acquiring_load_exclusive(n));
9942 match(Set res (WeakCompareAndSwapI mem (Binary oldval newval)));
9943 ins_cost(VOLATILE_REF_COST);
9944 effect(KILL cr);
9945 format %{
9946 "cmpxchgw_acq $res = $mem, $oldval, $newval\t# (int, weak) if $mem == $oldval then $mem <-- $newval"
9947 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
9948 %}
9949 ins_encode %{
9950 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
9951 Assembler::word, /*acquire*/ true, /*release*/ true,
9952 /*weak*/ true, noreg);
9953 __ csetw($res$$Register, Assembler::EQ);
9954 %}
9955 ins_pipe(pipe_slow);
9956 %}
9957
9958 // This pattern is generated automatically from cas.m4.
9959 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9960 instruct weakCompareAndSwapLAcq(iRegINoSp res, indirect mem, iRegL oldval, iRegL newval, rFlagsReg cr) %{
9961 predicate(needs_acquiring_load_exclusive(n));
9962 match(Set res (WeakCompareAndSwapL mem (Binary oldval newval)));
9963 ins_cost(VOLATILE_REF_COST);
9964 effect(KILL cr);
9965 format %{
9966 "cmpxchg_acq $res = $mem, $oldval, $newval\t# (long, weak) if $mem == $oldval then $mem <-- $newval"
9967 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
9968 %}
9969 ins_encode %{
9970 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
9971 Assembler::xword, /*acquire*/ true, /*release*/ true,
9972 /*weak*/ true, noreg);
9973 __ csetw($res$$Register, Assembler::EQ);
9974 %}
9975 ins_pipe(pipe_slow);
9976 %}
9977
9978 // This pattern is generated automatically from cas.m4.
9979 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9980 instruct weakCompareAndSwapNAcq(iRegINoSp res, indirect mem, iRegN oldval, iRegN newval, rFlagsReg cr) %{
9981 predicate(needs_acquiring_load_exclusive(n));
9982 match(Set res (WeakCompareAndSwapN mem (Binary oldval newval)));
9983 ins_cost(VOLATILE_REF_COST);
9984 effect(KILL cr);
9985 format %{
9986 "cmpxchgw_acq $res = $mem, $oldval, $newval\t# (narrow oop, weak) if $mem == $oldval then $mem <-- $newval"
9987 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
9988 %}
9989 ins_encode %{
9990 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
9991 Assembler::word, /*acquire*/ true, /*release*/ true,
9992 /*weak*/ true, noreg);
9993 __ csetw($res$$Register, Assembler::EQ);
9994 %}
9995 ins_pipe(pipe_slow);
9996 %}
9997
9998 // This pattern is generated automatically from cas.m4.
9999 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10000 instruct weakCompareAndSwapPAcq(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
10001 predicate(needs_acquiring_load_exclusive(n) && (n->as_LoadStore()->barrier_data() == 0));
10002 match(Set res (WeakCompareAndSwapP mem (Binary oldval newval)));
10003 ins_cost(VOLATILE_REF_COST);
10004 effect(KILL cr);
10005 format %{
10006 "cmpxchg_acq $res = $mem, $oldval, $newval\t# (ptr, weak) if $mem == $oldval then $mem <-- $newval"
10007 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
10008 %}
10009 ins_encode %{
10010 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
10011 Assembler::xword, /*acquire*/ true, /*release*/ true,
10012 /*weak*/ true, noreg);
10013 __ csetw($res$$Register, Assembler::EQ);
10014 %}
10015 ins_pipe(pipe_slow);
10016 %}
10017
10018 // END This section of the file is automatically generated. Do not edit --------------
10019 // ---------------------------------------------------------------------
10020
10021 instruct get_and_setI(indirect mem, iRegI newv, iRegINoSp prev) %{
10022 match(Set prev (GetAndSetI mem newv));
10023 ins_cost(2 * VOLATILE_REF_COST);
10024 format %{ "atomic_xchgw $prev, $newv, [$mem]" %}
10025 ins_encode %{
10026 __ atomic_xchgw($prev$$Register, $newv$$Register, as_Register($mem$$base));
10027 %}
10028 ins_pipe(pipe_serial);
10029 %}
10030
10031 instruct get_and_setL(indirect mem, iRegL newv, iRegLNoSp prev) %{
10032 match(Set prev (GetAndSetL mem newv));
10033 ins_cost(2 * VOLATILE_REF_COST);
10034 format %{ "atomic_xchg $prev, $newv, [$mem]" %}
10035 ins_encode %{
10036 __ atomic_xchg($prev$$Register, $newv$$Register, as_Register($mem$$base));
10037 %}
10038 ins_pipe(pipe_serial);
10039 %}
10040
10041 instruct get_and_setN(indirect mem, iRegN newv, iRegINoSp prev) %{
10042 match(Set prev (GetAndSetN mem newv));
10043 ins_cost(2 * VOLATILE_REF_COST);
10044 format %{ "atomic_xchgw $prev, $newv, [$mem]" %}
10045 ins_encode %{
10046 __ atomic_xchgw($prev$$Register, $newv$$Register, as_Register($mem$$base));
10047 %}
10048 ins_pipe(pipe_serial);
10049 %}
10050
10051 instruct get_and_setP(indirect mem, iRegP newv, iRegPNoSp prev) %{
10052 predicate(n->as_LoadStore()->barrier_data() == 0);
10053 match(Set prev (GetAndSetP mem newv));
10054 ins_cost(2 * VOLATILE_REF_COST);
10055 format %{ "atomic_xchg $prev, $newv, [$mem]" %}
10056 ins_encode %{
10057 __ atomic_xchg($prev$$Register, $newv$$Register, as_Register($mem$$base));
10058 %}
10059 ins_pipe(pipe_serial);
10060 %}
10061
10062 instruct get_and_setIAcq(indirect mem, iRegI newv, iRegINoSp prev) %{
10063 predicate(needs_acquiring_load_exclusive(n));
10064 match(Set prev (GetAndSetI mem newv));
10065 ins_cost(VOLATILE_REF_COST);
10066 format %{ "atomic_xchgw_acq $prev, $newv, [$mem]" %}
10067 ins_encode %{
10068 __ atomic_xchgalw($prev$$Register, $newv$$Register, as_Register($mem$$base));
10069 %}
10070 ins_pipe(pipe_serial);
10071 %}
10072
10073 instruct get_and_setLAcq(indirect mem, iRegL newv, iRegLNoSp prev) %{
10074 predicate(needs_acquiring_load_exclusive(n));
10075 match(Set prev (GetAndSetL mem newv));
10076 ins_cost(VOLATILE_REF_COST);
10077 format %{ "atomic_xchg_acq $prev, $newv, [$mem]" %}
10078 ins_encode %{
10079 __ atomic_xchgal($prev$$Register, $newv$$Register, as_Register($mem$$base));
10080 %}
10081 ins_pipe(pipe_serial);
10082 %}
10083
10084 instruct get_and_setNAcq(indirect mem, iRegN newv, iRegINoSp prev) %{
10085 predicate(needs_acquiring_load_exclusive(n));
10086 match(Set prev (GetAndSetN mem newv));
10087 ins_cost(VOLATILE_REF_COST);
10088 format %{ "atomic_xchgw_acq $prev, $newv, [$mem]" %}
10089 ins_encode %{
10090 __ atomic_xchgalw($prev$$Register, $newv$$Register, as_Register($mem$$base));
10091 %}
10092 ins_pipe(pipe_serial);
10093 %}
10094
10095 instruct get_and_setPAcq(indirect mem, iRegP newv, iRegPNoSp prev) %{
10096 predicate(needs_acquiring_load_exclusive(n) && (n->as_LoadStore()->barrier_data() == 0));
10097 match(Set prev (GetAndSetP mem newv));
10098 ins_cost(VOLATILE_REF_COST);
10099 format %{ "atomic_xchg_acq $prev, $newv, [$mem]" %}
10100 ins_encode %{
10101 __ atomic_xchgal($prev$$Register, $newv$$Register, as_Register($mem$$base));
10102 %}
10103 ins_pipe(pipe_serial);
10104 %}
10105
10106
10107 instruct get_and_addL(indirect mem, iRegLNoSp newval, iRegL incr) %{
10108 match(Set newval (GetAndAddL mem incr));
10109 ins_cost(2 * VOLATILE_REF_COST + 1);
10110 format %{ "get_and_addL $newval, [$mem], $incr" %}
10111 ins_encode %{
10112 __ atomic_add($newval$$Register, $incr$$Register, as_Register($mem$$base));
10113 %}
10114 ins_pipe(pipe_serial);
10115 %}
10116
10117 instruct get_and_addL_no_res(indirect mem, Universe dummy, iRegL incr) %{
10118 predicate(n->as_LoadStore()->result_not_used());
10119 match(Set dummy (GetAndAddL mem incr));
10120 ins_cost(2 * VOLATILE_REF_COST);
10121 format %{ "get_and_addL [$mem], $incr" %}
10122 ins_encode %{
10123 __ atomic_add(noreg, $incr$$Register, as_Register($mem$$base));
10124 %}
10125 ins_pipe(pipe_serial);
10126 %}
10127
10128 instruct get_and_addLi(indirect mem, iRegLNoSp newval, immLAddSub incr) %{
10129 match(Set newval (GetAndAddL mem incr));
10130 ins_cost(2 * VOLATILE_REF_COST + 1);
10131 format %{ "get_and_addL $newval, [$mem], $incr" %}
10132 ins_encode %{
10133 __ atomic_add($newval$$Register, $incr$$constant, as_Register($mem$$base));
10134 %}
10135 ins_pipe(pipe_serial);
10136 %}
10137
10138 instruct get_and_addLi_no_res(indirect mem, Universe dummy, immLAddSub incr) %{
10139 predicate(n->as_LoadStore()->result_not_used());
10140 match(Set dummy (GetAndAddL mem incr));
10141 ins_cost(2 * VOLATILE_REF_COST);
10142 format %{ "get_and_addL [$mem], $incr" %}
10143 ins_encode %{
10144 __ atomic_add(noreg, $incr$$constant, as_Register($mem$$base));
10145 %}
10146 ins_pipe(pipe_serial);
10147 %}
10148
10149 instruct get_and_addI(indirect mem, iRegINoSp newval, iRegIorL2I incr) %{
10150 match(Set newval (GetAndAddI mem incr));
10151 ins_cost(2 * VOLATILE_REF_COST + 1);
10152 format %{ "get_and_addI $newval, [$mem], $incr" %}
10153 ins_encode %{
10154 __ atomic_addw($newval$$Register, $incr$$Register, as_Register($mem$$base));
10155 %}
10156 ins_pipe(pipe_serial);
10157 %}
10158
10159 instruct get_and_addI_no_res(indirect mem, Universe dummy, iRegIorL2I incr) %{
10160 predicate(n->as_LoadStore()->result_not_used());
10161 match(Set dummy (GetAndAddI mem incr));
10162 ins_cost(2 * VOLATILE_REF_COST);
10163 format %{ "get_and_addI [$mem], $incr" %}
10164 ins_encode %{
10165 __ atomic_addw(noreg, $incr$$Register, as_Register($mem$$base));
10166 %}
10167 ins_pipe(pipe_serial);
10168 %}
10169
10170 instruct get_and_addIi(indirect mem, iRegINoSp newval, immIAddSub incr) %{
10171 match(Set newval (GetAndAddI mem incr));
10172 ins_cost(2 * VOLATILE_REF_COST + 1);
10173 format %{ "get_and_addI $newval, [$mem], $incr" %}
10174 ins_encode %{
10175 __ atomic_addw($newval$$Register, $incr$$constant, as_Register($mem$$base));
10176 %}
10177 ins_pipe(pipe_serial);
10178 %}
10179
10180 instruct get_and_addIi_no_res(indirect mem, Universe dummy, immIAddSub incr) %{
10181 predicate(n->as_LoadStore()->result_not_used());
10182 match(Set dummy (GetAndAddI mem incr));
10183 ins_cost(2 * VOLATILE_REF_COST);
10184 format %{ "get_and_addI [$mem], $incr" %}
10185 ins_encode %{
10186 __ atomic_addw(noreg, $incr$$constant, as_Register($mem$$base));
10187 %}
10188 ins_pipe(pipe_serial);
10189 %}
10190
10191 instruct get_and_addLAcq(indirect mem, iRegLNoSp newval, iRegL incr) %{
10192 predicate(needs_acquiring_load_exclusive(n));
10193 match(Set newval (GetAndAddL mem incr));
10194 ins_cost(VOLATILE_REF_COST + 1);
10195 format %{ "get_and_addL_acq $newval, [$mem], $incr" %}
10196 ins_encode %{
10197 __ atomic_addal($newval$$Register, $incr$$Register, as_Register($mem$$base));
10198 %}
10199 ins_pipe(pipe_serial);
10200 %}
10201
10202 instruct get_and_addL_no_resAcq(indirect mem, Universe dummy, iRegL incr) %{
10203 predicate(n->as_LoadStore()->result_not_used() && needs_acquiring_load_exclusive(n));
10204 match(Set dummy (GetAndAddL mem incr));
10205 ins_cost(VOLATILE_REF_COST);
10206 format %{ "get_and_addL_acq [$mem], $incr" %}
10207 ins_encode %{
10208 __ atomic_addal(noreg, $incr$$Register, as_Register($mem$$base));
10209 %}
10210 ins_pipe(pipe_serial);
10211 %}
10212
10213 instruct get_and_addLiAcq(indirect mem, iRegLNoSp newval, immLAddSub incr) %{
10214 predicate(needs_acquiring_load_exclusive(n));
10215 match(Set newval (GetAndAddL mem incr));
10216 ins_cost(VOLATILE_REF_COST + 1);
10217 format %{ "get_and_addL_acq $newval, [$mem], $incr" %}
10218 ins_encode %{
10219 __ atomic_addal($newval$$Register, $incr$$constant, as_Register($mem$$base));
10220 %}
10221 ins_pipe(pipe_serial);
10222 %}
10223
10224 instruct get_and_addLi_no_resAcq(indirect mem, Universe dummy, immLAddSub incr) %{
10225 predicate(n->as_LoadStore()->result_not_used() && needs_acquiring_load_exclusive(n));
10226 match(Set dummy (GetAndAddL mem incr));
10227 ins_cost(VOLATILE_REF_COST);
10228 format %{ "get_and_addL_acq [$mem], $incr" %}
10229 ins_encode %{
10230 __ atomic_addal(noreg, $incr$$constant, as_Register($mem$$base));
10231 %}
10232 ins_pipe(pipe_serial);
10233 %}
10234
10235 instruct get_and_addIAcq(indirect mem, iRegINoSp newval, iRegIorL2I incr) %{
10236 predicate(needs_acquiring_load_exclusive(n));
10237 match(Set newval (GetAndAddI mem incr));
10238 ins_cost(VOLATILE_REF_COST + 1);
10239 format %{ "get_and_addI_acq $newval, [$mem], $incr" %}
10240 ins_encode %{
10241 __ atomic_addalw($newval$$Register, $incr$$Register, as_Register($mem$$base));
10242 %}
10243 ins_pipe(pipe_serial);
10244 %}
10245
10246 instruct get_and_addI_no_resAcq(indirect mem, Universe dummy, iRegIorL2I incr) %{
10247 predicate(n->as_LoadStore()->result_not_used() && needs_acquiring_load_exclusive(n));
10248 match(Set dummy (GetAndAddI mem incr));
10249 ins_cost(VOLATILE_REF_COST);
10250 format %{ "get_and_addI_acq [$mem], $incr" %}
10251 ins_encode %{
10252 __ atomic_addalw(noreg, $incr$$Register, as_Register($mem$$base));
10253 %}
10254 ins_pipe(pipe_serial);
10255 %}
10256
10257 instruct get_and_addIiAcq(indirect mem, iRegINoSp newval, immIAddSub incr) %{
10258 predicate(needs_acquiring_load_exclusive(n));
10259 match(Set newval (GetAndAddI mem incr));
10260 ins_cost(VOLATILE_REF_COST + 1);
10261 format %{ "get_and_addI_acq $newval, [$mem], $incr" %}
10262 ins_encode %{
10263 __ atomic_addalw($newval$$Register, $incr$$constant, as_Register($mem$$base));
10264 %}
10265 ins_pipe(pipe_serial);
10266 %}
10267
10268 instruct get_and_addIi_no_resAcq(indirect mem, Universe dummy, immIAddSub incr) %{
10269 predicate(n->as_LoadStore()->result_not_used() && needs_acquiring_load_exclusive(n));
10270 match(Set dummy (GetAndAddI mem incr));
10271 ins_cost(VOLATILE_REF_COST);
10272 format %{ "get_and_addI_acq [$mem], $incr" %}
10273 ins_encode %{
10274 __ atomic_addalw(noreg, $incr$$constant, as_Register($mem$$base));
10275 %}
10276 ins_pipe(pipe_serial);
10277 %}
10278
10279 // Manifest a CmpL result in an integer register.
10280 // (src1 < src2) ? -1 : ((src1 > src2) ? 1 : 0)
10281 instruct cmpL3_reg_reg(iRegINoSp dst, iRegL src1, iRegL src2, rFlagsReg flags)
10282 %{
10283 match(Set dst (CmpL3 src1 src2));
10284 effect(KILL flags);
10285
10286 ins_cost(INSN_COST * 6);
10287 format %{
10288 "cmp $src1, $src2"
10289 "csetw $dst, ne"
10290 "cnegw $dst, lt"
10291 %}
10292 // format %{ "CmpL3 $dst, $src1, $src2" %}
10293 ins_encode %{
10294 __ cmp($src1$$Register, $src2$$Register);
10295 __ csetw($dst$$Register, Assembler::NE);
10296 __ cnegw($dst$$Register, $dst$$Register, Assembler::LT);
10297 %}
10298
10299 ins_pipe(pipe_class_default);
10300 %}
10301
10302 instruct cmpL3_reg_imm(iRegINoSp dst, iRegL src1, immLAddSub src2, rFlagsReg flags)
10303 %{
10304 match(Set dst (CmpL3 src1 src2));
10305 effect(KILL flags);
10306
10307 ins_cost(INSN_COST * 6);
10308 format %{
10309 "cmp $src1, $src2"
10310 "csetw $dst, ne"
10311 "cnegw $dst, lt"
10312 %}
10313 ins_encode %{
10314 int32_t con = (int32_t)$src2$$constant;
10315 if (con < 0) {
10316 __ adds(zr, $src1$$Register, -con);
10317 } else {
10318 __ subs(zr, $src1$$Register, con);
10319 }
10320 __ csetw($dst$$Register, Assembler::NE);
10321 __ cnegw($dst$$Register, $dst$$Register, Assembler::LT);
10322 %}
10323
10324 ins_pipe(pipe_class_default);
10325 %}
10326
10327 // ============================================================================
10328 // Conditional Move Instructions
10329
10330 // n.b. we have identical rules for both a signed compare op (cmpOp)
10331 // and an unsigned compare op (cmpOpU). it would be nice if we could
10332 // define an op class which merged both inputs and use it to type the
10333 // argument to a single rule. unfortunatelyt his fails because the
10334 // opclass does not live up to the COND_INTER interface of its
10335 // component operands. When the generic code tries to negate the
10336 // operand it ends up running the generci Machoper::negate method
10337 // which throws a ShouldNotHappen. So, we have to provide two flavours
10338 // of each rule, one for a cmpOp and a second for a cmpOpU (sigh).
10339
10340 instruct cmovI_reg_reg(cmpOp cmp, rFlagsReg cr, iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10341 match(Set dst (CMoveI (Binary cmp cr) (Binary src1 src2)));
10342
10343 ins_cost(INSN_COST * 2);
10344 format %{ "cselw $dst, $src2, $src1 $cmp\t# signed, int" %}
10345
10346 ins_encode %{
10347 __ cselw(as_Register($dst$$reg),
10348 as_Register($src2$$reg),
10349 as_Register($src1$$reg),
10350 (Assembler::Condition)$cmp$$cmpcode);
10351 %}
10352
10353 ins_pipe(icond_reg_reg);
10354 %}
10355
10356 instruct cmovUI_reg_reg(cmpOpU cmp, rFlagsRegU cr, iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10357 match(Set dst (CMoveI (Binary cmp cr) (Binary src1 src2)));
10358
10359 ins_cost(INSN_COST * 2);
10360 format %{ "cselw $dst, $src2, $src1 $cmp\t# unsigned, int" %}
10361
10362 ins_encode %{
10363 __ cselw(as_Register($dst$$reg),
10364 as_Register($src2$$reg),
10365 as_Register($src1$$reg),
10366 (Assembler::Condition)$cmp$$cmpcode);
10367 %}
10368
10369 ins_pipe(icond_reg_reg);
10370 %}
10371
10372 // special cases where one arg is zero
10373
10374 // n.b. this is selected in preference to the rule above because it
10375 // avoids loading constant 0 into a source register
10376
10377 // TODO
10378 // we ought only to be able to cull one of these variants as the ideal
10379 // transforms ought always to order the zero consistently (to left/right?)
10380
10381 instruct cmovI_zero_reg(cmpOp cmp, rFlagsReg cr, iRegINoSp dst, immI0 zero, iRegIorL2I src) %{
10382 match(Set dst (CMoveI (Binary cmp cr) (Binary zero src)));
10383
10384 ins_cost(INSN_COST * 2);
10385 format %{ "cselw $dst, $src, zr $cmp\t# signed, int" %}
10386
10387 ins_encode %{
10388 __ cselw(as_Register($dst$$reg),
10389 as_Register($src$$reg),
10390 zr,
10391 (Assembler::Condition)$cmp$$cmpcode);
10392 %}
10393
10394 ins_pipe(icond_reg);
10395 %}
10396
10397 instruct cmovUI_zero_reg(cmpOpU cmp, rFlagsRegU cr, iRegINoSp dst, immI0 zero, iRegIorL2I src) %{
10398 match(Set dst (CMoveI (Binary cmp cr) (Binary zero src)));
10399
10400 ins_cost(INSN_COST * 2);
10401 format %{ "cselw $dst, $src, zr $cmp\t# unsigned, int" %}
10402
10403 ins_encode %{
10404 __ cselw(as_Register($dst$$reg),
10405 as_Register($src$$reg),
10406 zr,
10407 (Assembler::Condition)$cmp$$cmpcode);
10408 %}
10409
10410 ins_pipe(icond_reg);
10411 %}
10412
10413 instruct cmovI_reg_zero(cmpOp cmp, rFlagsReg cr, iRegINoSp dst, iRegIorL2I src, immI0 zero) %{
10414 match(Set dst (CMoveI (Binary cmp cr) (Binary src zero)));
10415
10416 ins_cost(INSN_COST * 2);
10417 format %{ "cselw $dst, zr, $src $cmp\t# signed, int" %}
10418
10419 ins_encode %{
10420 __ cselw(as_Register($dst$$reg),
10421 zr,
10422 as_Register($src$$reg),
10423 (Assembler::Condition)$cmp$$cmpcode);
10424 %}
10425
10426 ins_pipe(icond_reg);
10427 %}
10428
10429 instruct cmovUI_reg_zero(cmpOpU cmp, rFlagsRegU cr, iRegINoSp dst, iRegIorL2I src, immI0 zero) %{
10430 match(Set dst (CMoveI (Binary cmp cr) (Binary src zero)));
10431
10432 ins_cost(INSN_COST * 2);
10433 format %{ "cselw $dst, zr, $src $cmp\t# unsigned, int" %}
10434
10435 ins_encode %{
10436 __ cselw(as_Register($dst$$reg),
10437 zr,
10438 as_Register($src$$reg),
10439 (Assembler::Condition)$cmp$$cmpcode);
10440 %}
10441
10442 ins_pipe(icond_reg);
10443 %}
10444
10445 // special case for creating a boolean 0 or 1
10446
10447 // n.b. this is selected in preference to the rule above because it
10448 // avoids loading constants 0 and 1 into a source register
10449
10450 instruct cmovI_reg_zero_one(cmpOp cmp, rFlagsReg cr, iRegINoSp dst, immI0 zero, immI_1 one) %{
10451 match(Set dst (CMoveI (Binary cmp cr) (Binary one zero)));
10452
10453 ins_cost(INSN_COST * 2);
10454 format %{ "csincw $dst, zr, zr $cmp\t# signed, int" %}
10455
10456 ins_encode %{
10457 // equivalently
10458 // cset(as_Register($dst$$reg),
10459 // negate_condition((Assembler::Condition)$cmp$$cmpcode));
10460 __ csincw(as_Register($dst$$reg),
10461 zr,
10462 zr,
10463 (Assembler::Condition)$cmp$$cmpcode);
10464 %}
10465
10466 ins_pipe(icond_none);
10467 %}
10468
10469 instruct cmovUI_reg_zero_one(cmpOpU cmp, rFlagsRegU cr, iRegINoSp dst, immI0 zero, immI_1 one) %{
10470 match(Set dst (CMoveI (Binary cmp cr) (Binary one zero)));
10471
10472 ins_cost(INSN_COST * 2);
10473 format %{ "csincw $dst, zr, zr $cmp\t# unsigned, int" %}
10474
10475 ins_encode %{
10476 // equivalently
10477 // cset(as_Register($dst$$reg),
10478 // negate_condition((Assembler::Condition)$cmp$$cmpcode));
10479 __ csincw(as_Register($dst$$reg),
10480 zr,
10481 zr,
10482 (Assembler::Condition)$cmp$$cmpcode);
10483 %}
10484
10485 ins_pipe(icond_none);
10486 %}
10487
10488 instruct cmovL_reg_reg(cmpOp cmp, rFlagsReg cr, iRegLNoSp dst, iRegL src1, iRegL src2) %{
10489 match(Set dst (CMoveL (Binary cmp cr) (Binary src1 src2)));
10490
10491 ins_cost(INSN_COST * 2);
10492 format %{ "csel $dst, $src2, $src1 $cmp\t# signed, long" %}
10493
10494 ins_encode %{
10495 __ csel(as_Register($dst$$reg),
10496 as_Register($src2$$reg),
10497 as_Register($src1$$reg),
10498 (Assembler::Condition)$cmp$$cmpcode);
10499 %}
10500
10501 ins_pipe(icond_reg_reg);
10502 %}
10503
10504 instruct cmovUL_reg_reg(cmpOpU cmp, rFlagsRegU cr, iRegLNoSp dst, iRegL src1, iRegL src2) %{
10505 match(Set dst (CMoveL (Binary cmp cr) (Binary src1 src2)));
10506
10507 ins_cost(INSN_COST * 2);
10508 format %{ "csel $dst, $src2, $src1 $cmp\t# unsigned, long" %}
10509
10510 ins_encode %{
10511 __ csel(as_Register($dst$$reg),
10512 as_Register($src2$$reg),
10513 as_Register($src1$$reg),
10514 (Assembler::Condition)$cmp$$cmpcode);
10515 %}
10516
10517 ins_pipe(icond_reg_reg);
10518 %}
10519
10520 // special cases where one arg is zero
10521
10522 instruct cmovL_reg_zero(cmpOp cmp, rFlagsReg cr, iRegLNoSp dst, iRegL src, immL0 zero) %{
10523 match(Set dst (CMoveL (Binary cmp cr) (Binary src zero)));
10524
10525 ins_cost(INSN_COST * 2);
10526 format %{ "csel $dst, zr, $src $cmp\t# signed, long" %}
10527
10528 ins_encode %{
10529 __ csel(as_Register($dst$$reg),
10530 zr,
10531 as_Register($src$$reg),
10532 (Assembler::Condition)$cmp$$cmpcode);
10533 %}
10534
10535 ins_pipe(icond_reg);
10536 %}
10537
10538 instruct cmovUL_reg_zero(cmpOpU cmp, rFlagsRegU cr, iRegLNoSp dst, iRegL src, immL0 zero) %{
10539 match(Set dst (CMoveL (Binary cmp cr) (Binary src zero)));
10540
10541 ins_cost(INSN_COST * 2);
10542 format %{ "csel $dst, zr, $src $cmp\t# unsigned, long" %}
10543
10544 ins_encode %{
10545 __ csel(as_Register($dst$$reg),
10546 zr,
10547 as_Register($src$$reg),
10548 (Assembler::Condition)$cmp$$cmpcode);
10549 %}
10550
10551 ins_pipe(icond_reg);
10552 %}
10553
10554 instruct cmovL_zero_reg(cmpOp cmp, rFlagsReg cr, iRegLNoSp dst, immL0 zero, iRegL src) %{
10555 match(Set dst (CMoveL (Binary cmp cr) (Binary zero src)));
10556
10557 ins_cost(INSN_COST * 2);
10558 format %{ "csel $dst, $src, zr $cmp\t# signed, long" %}
10559
10560 ins_encode %{
10561 __ csel(as_Register($dst$$reg),
10562 as_Register($src$$reg),
10563 zr,
10564 (Assembler::Condition)$cmp$$cmpcode);
10565 %}
10566
10567 ins_pipe(icond_reg);
10568 %}
10569
10570 instruct cmovUL_zero_reg(cmpOpU cmp, rFlagsRegU cr, iRegLNoSp dst, immL0 zero, iRegL src) %{
10571 match(Set dst (CMoveL (Binary cmp cr) (Binary zero src)));
10572
10573 ins_cost(INSN_COST * 2);
10574 format %{ "csel $dst, $src, zr $cmp\t# unsigned, long" %}
10575
10576 ins_encode %{
10577 __ csel(as_Register($dst$$reg),
10578 as_Register($src$$reg),
10579 zr,
10580 (Assembler::Condition)$cmp$$cmpcode);
10581 %}
10582
10583 ins_pipe(icond_reg);
10584 %}
10585
10586 instruct cmovP_reg_reg(cmpOp cmp, rFlagsReg cr, iRegPNoSp dst, iRegP src1, iRegP src2) %{
10587 match(Set dst (CMoveP (Binary cmp cr) (Binary src1 src2)));
10588
10589 ins_cost(INSN_COST * 2);
10590 format %{ "csel $dst, $src2, $src1 $cmp\t# signed, ptr" %}
10591
10592 ins_encode %{
10593 __ csel(as_Register($dst$$reg),
10594 as_Register($src2$$reg),
10595 as_Register($src1$$reg),
10596 (Assembler::Condition)$cmp$$cmpcode);
10597 %}
10598
10599 ins_pipe(icond_reg_reg);
10600 %}
10601
10602 instruct cmovUP_reg_reg(cmpOpU cmp, rFlagsRegU cr, iRegPNoSp dst, iRegP src1, iRegP src2) %{
10603 match(Set dst (CMoveP (Binary cmp cr) (Binary src1 src2)));
10604
10605 ins_cost(INSN_COST * 2);
10606 format %{ "csel $dst, $src2, $src1 $cmp\t# unsigned, ptr" %}
10607
10608 ins_encode %{
10609 __ csel(as_Register($dst$$reg),
10610 as_Register($src2$$reg),
10611 as_Register($src1$$reg),
10612 (Assembler::Condition)$cmp$$cmpcode);
10613 %}
10614
10615 ins_pipe(icond_reg_reg);
10616 %}
10617
10618 // special cases where one arg is zero
10619
10620 instruct cmovP_reg_zero(cmpOp cmp, rFlagsReg cr, iRegPNoSp dst, iRegP src, immP0 zero) %{
10621 match(Set dst (CMoveP (Binary cmp cr) (Binary src zero)));
10622
10623 ins_cost(INSN_COST * 2);
10624 format %{ "csel $dst, zr, $src $cmp\t# signed, ptr" %}
10625
10626 ins_encode %{
10627 __ csel(as_Register($dst$$reg),
10628 zr,
10629 as_Register($src$$reg),
10630 (Assembler::Condition)$cmp$$cmpcode);
10631 %}
10632
10633 ins_pipe(icond_reg);
10634 %}
10635
10636 instruct cmovUP_reg_zero(cmpOpU cmp, rFlagsRegU cr, iRegPNoSp dst, iRegP src, immP0 zero) %{
10637 match(Set dst (CMoveP (Binary cmp cr) (Binary src zero)));
10638
10639 ins_cost(INSN_COST * 2);
10640 format %{ "csel $dst, zr, $src $cmp\t# unsigned, ptr" %}
10641
10642 ins_encode %{
10643 __ csel(as_Register($dst$$reg),
10644 zr,
10645 as_Register($src$$reg),
10646 (Assembler::Condition)$cmp$$cmpcode);
10647 %}
10648
10649 ins_pipe(icond_reg);
10650 %}
10651
10652 instruct cmovP_zero_reg(cmpOp cmp, rFlagsReg cr, iRegPNoSp dst, immP0 zero, iRegP src) %{
10653 match(Set dst (CMoveP (Binary cmp cr) (Binary zero src)));
10654
10655 ins_cost(INSN_COST * 2);
10656 format %{ "csel $dst, $src, zr $cmp\t# signed, ptr" %}
10657
10658 ins_encode %{
10659 __ csel(as_Register($dst$$reg),
10660 as_Register($src$$reg),
10661 zr,
10662 (Assembler::Condition)$cmp$$cmpcode);
10663 %}
10664
10665 ins_pipe(icond_reg);
10666 %}
10667
10668 instruct cmovUP_zero_reg(cmpOpU cmp, rFlagsRegU cr, iRegPNoSp dst, immP0 zero, iRegP src) %{
10669 match(Set dst (CMoveP (Binary cmp cr) (Binary zero src)));
10670
10671 ins_cost(INSN_COST * 2);
10672 format %{ "csel $dst, $src, zr $cmp\t# unsigned, ptr" %}
10673
10674 ins_encode %{
10675 __ csel(as_Register($dst$$reg),
10676 as_Register($src$$reg),
10677 zr,
10678 (Assembler::Condition)$cmp$$cmpcode);
10679 %}
10680
10681 ins_pipe(icond_reg);
10682 %}
10683
10684 instruct cmovN_reg_reg(cmpOp cmp, rFlagsReg cr, iRegNNoSp dst, iRegN src1, iRegN src2) %{
10685 match(Set dst (CMoveN (Binary cmp cr) (Binary src1 src2)));
10686
10687 ins_cost(INSN_COST * 2);
10688 format %{ "cselw $dst, $src2, $src1 $cmp\t# signed, compressed ptr" %}
10689
10690 ins_encode %{
10691 __ cselw(as_Register($dst$$reg),
10692 as_Register($src2$$reg),
10693 as_Register($src1$$reg),
10694 (Assembler::Condition)$cmp$$cmpcode);
10695 %}
10696
10697 ins_pipe(icond_reg_reg);
10698 %}
10699
10700 instruct cmovUN_reg_reg(cmpOpU cmp, rFlagsRegU cr, iRegNNoSp dst, iRegN src1, iRegN src2) %{
10701 match(Set dst (CMoveN (Binary cmp cr) (Binary src1 src2)));
10702
10703 ins_cost(INSN_COST * 2);
10704 format %{ "cselw $dst, $src2, $src1 $cmp\t# signed, compressed ptr" %}
10705
10706 ins_encode %{
10707 __ cselw(as_Register($dst$$reg),
10708 as_Register($src2$$reg),
10709 as_Register($src1$$reg),
10710 (Assembler::Condition)$cmp$$cmpcode);
10711 %}
10712
10713 ins_pipe(icond_reg_reg);
10714 %}
10715
10716 // special cases where one arg is zero
10717
10718 instruct cmovN_reg_zero(cmpOp cmp, rFlagsReg cr, iRegNNoSp dst, iRegN src, immN0 zero) %{
10719 match(Set dst (CMoveN (Binary cmp cr) (Binary src zero)));
10720
10721 ins_cost(INSN_COST * 2);
10722 format %{ "cselw $dst, zr, $src $cmp\t# signed, compressed ptr" %}
10723
10724 ins_encode %{
10725 __ cselw(as_Register($dst$$reg),
10726 zr,
10727 as_Register($src$$reg),
10728 (Assembler::Condition)$cmp$$cmpcode);
10729 %}
10730
10731 ins_pipe(icond_reg);
10732 %}
10733
10734 instruct cmovUN_reg_zero(cmpOpU cmp, rFlagsRegU cr, iRegNNoSp dst, iRegN src, immN0 zero) %{
10735 match(Set dst (CMoveN (Binary cmp cr) (Binary src zero)));
10736
10737 ins_cost(INSN_COST * 2);
10738 format %{ "cselw $dst, zr, $src $cmp\t# unsigned, compressed ptr" %}
10739
10740 ins_encode %{
10741 __ cselw(as_Register($dst$$reg),
10742 zr,
10743 as_Register($src$$reg),
10744 (Assembler::Condition)$cmp$$cmpcode);
10745 %}
10746
10747 ins_pipe(icond_reg);
10748 %}
10749
10750 instruct cmovN_zero_reg(cmpOp cmp, rFlagsReg cr, iRegNNoSp dst, immN0 zero, iRegN src) %{
10751 match(Set dst (CMoveN (Binary cmp cr) (Binary zero src)));
10752
10753 ins_cost(INSN_COST * 2);
10754 format %{ "cselw $dst, $src, zr $cmp\t# signed, compressed ptr" %}
10755
10756 ins_encode %{
10757 __ cselw(as_Register($dst$$reg),
10758 as_Register($src$$reg),
10759 zr,
10760 (Assembler::Condition)$cmp$$cmpcode);
10761 %}
10762
10763 ins_pipe(icond_reg);
10764 %}
10765
10766 instruct cmovUN_zero_reg(cmpOpU cmp, rFlagsRegU cr, iRegNNoSp dst, immN0 zero, iRegN src) %{
10767 match(Set dst (CMoveN (Binary cmp cr) (Binary zero src)));
10768
10769 ins_cost(INSN_COST * 2);
10770 format %{ "cselw $dst, $src, zr $cmp\t# unsigned, compressed ptr" %}
10771
10772 ins_encode %{
10773 __ cselw(as_Register($dst$$reg),
10774 as_Register($src$$reg),
10775 zr,
10776 (Assembler::Condition)$cmp$$cmpcode);
10777 %}
10778
10779 ins_pipe(icond_reg);
10780 %}
10781
10782 instruct cmovF_reg(cmpOp cmp, rFlagsReg cr, vRegF dst, vRegF src1, vRegF src2)
10783 %{
10784 match(Set dst (CMoveF (Binary cmp cr) (Binary src1 src2)));
10785
10786 ins_cost(INSN_COST * 3);
10787
10788 format %{ "fcsels $dst, $src1, $src2, $cmp\t# signed cmove float\n\t" %}
10789 ins_encode %{
10790 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
10791 __ fcsels(as_FloatRegister($dst$$reg),
10792 as_FloatRegister($src2$$reg),
10793 as_FloatRegister($src1$$reg),
10794 cond);
10795 %}
10796
10797 ins_pipe(fp_cond_reg_reg_s);
10798 %}
10799
10800 instruct cmovUF_reg(cmpOpU cmp, rFlagsRegU cr, vRegF dst, vRegF src1, vRegF src2)
10801 %{
10802 match(Set dst (CMoveF (Binary cmp cr) (Binary src1 src2)));
10803
10804 ins_cost(INSN_COST * 3);
10805
10806 format %{ "fcsels $dst, $src1, $src2, $cmp\t# unsigned cmove float\n\t" %}
10807 ins_encode %{
10808 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
10809 __ fcsels(as_FloatRegister($dst$$reg),
10810 as_FloatRegister($src2$$reg),
10811 as_FloatRegister($src1$$reg),
10812 cond);
10813 %}
10814
10815 ins_pipe(fp_cond_reg_reg_s);
10816 %}
10817
10818 instruct cmovD_reg(cmpOp cmp, rFlagsReg cr, vRegD dst, vRegD src1, vRegD src2)
10819 %{
10820 match(Set dst (CMoveD (Binary cmp cr) (Binary src1 src2)));
10821
10822 ins_cost(INSN_COST * 3);
10823
10824 format %{ "fcseld $dst, $src1, $src2, $cmp\t# signed cmove float\n\t" %}
10825 ins_encode %{
10826 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
10827 __ fcseld(as_FloatRegister($dst$$reg),
10828 as_FloatRegister($src2$$reg),
10829 as_FloatRegister($src1$$reg),
10830 cond);
10831 %}
10832
10833 ins_pipe(fp_cond_reg_reg_d);
10834 %}
10835
10836 instruct cmovUD_reg(cmpOpU cmp, rFlagsRegU cr, vRegD dst, vRegD src1, vRegD src2)
10837 %{
10838 match(Set dst (CMoveD (Binary cmp cr) (Binary src1 src2)));
10839
10840 ins_cost(INSN_COST * 3);
10841
10842 format %{ "fcseld $dst, $src1, $src2, $cmp\t# unsigned cmove float\n\t" %}
10843 ins_encode %{
10844 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
10845 __ fcseld(as_FloatRegister($dst$$reg),
10846 as_FloatRegister($src2$$reg),
10847 as_FloatRegister($src1$$reg),
10848 cond);
10849 %}
10850
10851 ins_pipe(fp_cond_reg_reg_d);
10852 %}
10853
10854 // ============================================================================
10855 // Arithmetic Instructions
10856 //
10857
10858 // Integer Addition
10859
10860 // TODO
10861 // these currently employ operations which do not set CR and hence are
10862 // not flagged as killing CR but we would like to isolate the cases
10863 // where we want to set flags from those where we don't. need to work
10864 // out how to do that.
10865
10866 instruct addI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10867 match(Set dst (AddI src1 src2));
10868
10869 ins_cost(INSN_COST);
10870 format %{ "addw $dst, $src1, $src2" %}
10871
10872 ins_encode %{
10873 __ addw(as_Register($dst$$reg),
10874 as_Register($src1$$reg),
10875 as_Register($src2$$reg));
10876 %}
10877
10878 ins_pipe(ialu_reg_reg);
10879 %}
10880
10881 instruct addI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immIAddSub src2) %{
10882 match(Set dst (AddI src1 src2));
10883
10884 ins_cost(INSN_COST);
10885 format %{ "addw $dst, $src1, $src2" %}
10886
10887 // use opcode to indicate that this is an add not a sub
10888 opcode(0x0);
10889
10890 ins_encode(aarch64_enc_addsubw_imm(dst, src1, src2));
10891
10892 ins_pipe(ialu_reg_imm);
10893 %}
10894
10895 instruct addI_reg_imm_i2l(iRegINoSp dst, iRegL src1, immIAddSub src2) %{
10896 match(Set dst (AddI (ConvL2I src1) src2));
10897
10898 ins_cost(INSN_COST);
10899 format %{ "addw $dst, $src1, $src2" %}
10900
10901 // use opcode to indicate that this is an add not a sub
10902 opcode(0x0);
10903
10904 ins_encode(aarch64_enc_addsubw_imm(dst, src1, src2));
10905
10906 ins_pipe(ialu_reg_imm);
10907 %}
10908
10909 // Pointer Addition
10910 instruct addP_reg_reg(iRegPNoSp dst, iRegP src1, iRegL src2) %{
10911 match(Set dst (AddP src1 src2));
10912
10913 ins_cost(INSN_COST);
10914 format %{ "add $dst, $src1, $src2\t# ptr" %}
10915
10916 ins_encode %{
10917 __ add(as_Register($dst$$reg),
10918 as_Register($src1$$reg),
10919 as_Register($src2$$reg));
10920 %}
10921
10922 ins_pipe(ialu_reg_reg);
10923 %}
10924
10925 instruct addP_reg_reg_ext(iRegPNoSp dst, iRegP src1, iRegIorL2I src2) %{
10926 match(Set dst (AddP src1 (ConvI2L src2)));
10927
10928 ins_cost(1.9 * INSN_COST);
10929 format %{ "add $dst, $src1, $src2, sxtw\t# ptr" %}
10930
10931 ins_encode %{
10932 __ add(as_Register($dst$$reg),
10933 as_Register($src1$$reg),
10934 as_Register($src2$$reg), ext::sxtw);
10935 %}
10936
10937 ins_pipe(ialu_reg_reg);
10938 %}
10939
10940 instruct addP_reg_reg_lsl(iRegPNoSp dst, iRegP src1, iRegL src2, immIScale scale) %{
10941 match(Set dst (AddP src1 (LShiftL src2 scale)));
10942
10943 ins_cost(1.9 * INSN_COST);
10944 format %{ "add $dst, $src1, $src2, LShiftL $scale\t# ptr" %}
10945
10946 ins_encode %{
10947 __ lea(as_Register($dst$$reg),
10948 Address(as_Register($src1$$reg), as_Register($src2$$reg),
10949 Address::lsl($scale$$constant)));
10950 %}
10951
10952 ins_pipe(ialu_reg_reg_shift);
10953 %}
10954
10955 instruct addP_reg_reg_ext_shift(iRegPNoSp dst, iRegP src1, iRegIorL2I src2, immIScale scale) %{
10956 match(Set dst (AddP src1 (LShiftL (ConvI2L src2) scale)));
10957
10958 ins_cost(1.9 * INSN_COST);
10959 format %{ "add $dst, $src1, $src2, I2L $scale\t# ptr" %}
10960
10961 ins_encode %{
10962 __ lea(as_Register($dst$$reg),
10963 Address(as_Register($src1$$reg), as_Register($src2$$reg),
10964 Address::sxtw($scale$$constant)));
10965 %}
10966
10967 ins_pipe(ialu_reg_reg_shift);
10968 %}
10969
10970 instruct lshift_ext(iRegLNoSp dst, iRegIorL2I src, immI scale, rFlagsReg cr) %{
10971 match(Set dst (LShiftL (ConvI2L src) scale));
10972
10973 ins_cost(INSN_COST);
10974 format %{ "sbfiz $dst, $src, $scale & 63, -$scale & 63\t" %}
10975
10976 ins_encode %{
10977 __ sbfiz(as_Register($dst$$reg),
10978 as_Register($src$$reg),
10979 $scale$$constant & 63, MIN2(32, (int)((-$scale$$constant) & 63)));
10980 %}
10981
10982 ins_pipe(ialu_reg_shift);
10983 %}
10984
10985 // Pointer Immediate Addition
10986 // n.b. this needs to be more expensive than using an indirect memory
10987 // operand
10988 instruct addP_reg_imm(iRegPNoSp dst, iRegP src1, immLAddSub src2) %{
10989 match(Set dst (AddP src1 src2));
10990
10991 ins_cost(INSN_COST);
10992 format %{ "add $dst, $src1, $src2\t# ptr" %}
10993
10994 // use opcode to indicate that this is an add not a sub
10995 opcode(0x0);
10996
10997 ins_encode( aarch64_enc_addsub_imm(dst, src1, src2) );
10998
10999 ins_pipe(ialu_reg_imm);
11000 %}
11001
11002 // Long Addition
11003 instruct addL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
11004
11005 match(Set dst (AddL src1 src2));
11006
11007 ins_cost(INSN_COST);
11008 format %{ "add $dst, $src1, $src2" %}
11009
11010 ins_encode %{
11011 __ add(as_Register($dst$$reg),
11012 as_Register($src1$$reg),
11013 as_Register($src2$$reg));
11014 %}
11015
11016 ins_pipe(ialu_reg_reg);
11017 %}
11018
11019 // No constant pool entries requiredLong Immediate Addition.
11020 instruct addL_reg_imm(iRegLNoSp dst, iRegL src1, immLAddSub src2) %{
11021 match(Set dst (AddL src1 src2));
11022
11023 ins_cost(INSN_COST);
11024 format %{ "add $dst, $src1, $src2" %}
11025
11026 // use opcode to indicate that this is an add not a sub
11027 opcode(0x0);
11028
11029 ins_encode( aarch64_enc_addsub_imm(dst, src1, src2) );
11030
11031 ins_pipe(ialu_reg_imm);
11032 %}
11033
11034 // Integer Subtraction
11035 instruct subI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
11036 match(Set dst (SubI src1 src2));
11037
11038 ins_cost(INSN_COST);
11039 format %{ "subw $dst, $src1, $src2" %}
11040
11041 ins_encode %{
11042 __ subw(as_Register($dst$$reg),
11043 as_Register($src1$$reg),
11044 as_Register($src2$$reg));
11045 %}
11046
11047 ins_pipe(ialu_reg_reg);
11048 %}
11049
11050 // Immediate Subtraction
11051 instruct subI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immIAddSub src2) %{
11052 match(Set dst (SubI src1 src2));
11053
11054 ins_cost(INSN_COST);
11055 format %{ "subw $dst, $src1, $src2" %}
11056
11057 // use opcode to indicate that this is a sub not an add
11058 opcode(0x1);
11059
11060 ins_encode(aarch64_enc_addsubw_imm(dst, src1, src2));
11061
11062 ins_pipe(ialu_reg_imm);
11063 %}
11064
11065 // Long Subtraction
11066 instruct subL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
11067
11068 match(Set dst (SubL src1 src2));
11069
11070 ins_cost(INSN_COST);
11071 format %{ "sub $dst, $src1, $src2" %}
11072
11073 ins_encode %{
11074 __ sub(as_Register($dst$$reg),
11075 as_Register($src1$$reg),
11076 as_Register($src2$$reg));
11077 %}
11078
11079 ins_pipe(ialu_reg_reg);
11080 %}
11081
11082 // No constant pool entries requiredLong Immediate Subtraction.
11083 instruct subL_reg_imm(iRegLNoSp dst, iRegL src1, immLAddSub src2) %{
11084 match(Set dst (SubL src1 src2));
11085
11086 ins_cost(INSN_COST);
11087 format %{ "sub$dst, $src1, $src2" %}
11088
11089 // use opcode to indicate that this is a sub not an add
11090 opcode(0x1);
11091
11092 ins_encode( aarch64_enc_addsub_imm(dst, src1, src2) );
11093
11094 ins_pipe(ialu_reg_imm);
11095 %}
11096
11097 // Integer Negation (special case for sub)
11098
11099 instruct negI_reg(iRegINoSp dst, iRegIorL2I src, immI0 zero, rFlagsReg cr) %{
11100 match(Set dst (SubI zero src));
11101
11102 ins_cost(INSN_COST);
11103 format %{ "negw $dst, $src\t# int" %}
11104
11105 ins_encode %{
11106 __ negw(as_Register($dst$$reg),
11107 as_Register($src$$reg));
11108 %}
11109
11110 ins_pipe(ialu_reg);
11111 %}
11112
11113 // Long Negation
11114
11115 instruct negL_reg(iRegLNoSp dst, iRegL src, immL0 zero, rFlagsReg cr) %{
11116 match(Set dst (SubL zero src));
11117
11118 ins_cost(INSN_COST);
11119 format %{ "neg $dst, $src\t# long" %}
11120
11121 ins_encode %{
11122 __ neg(as_Register($dst$$reg),
11123 as_Register($src$$reg));
11124 %}
11125
11126 ins_pipe(ialu_reg);
11127 %}
11128
11129 // Integer Multiply
11130
11131 instruct mulI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
11132 match(Set dst (MulI src1 src2));
11133
11134 ins_cost(INSN_COST * 3);
11135 format %{ "mulw $dst, $src1, $src2" %}
11136
11137 ins_encode %{
11138 __ mulw(as_Register($dst$$reg),
11139 as_Register($src1$$reg),
11140 as_Register($src2$$reg));
11141 %}
11142
11143 ins_pipe(imul_reg_reg);
11144 %}
11145
11146 instruct smulI(iRegLNoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
11147 match(Set dst (MulL (ConvI2L src1) (ConvI2L src2)));
11148
11149 ins_cost(INSN_COST * 3);
11150 format %{ "smull $dst, $src1, $src2" %}
11151
11152 ins_encode %{
11153 __ smull(as_Register($dst$$reg),
11154 as_Register($src1$$reg),
11155 as_Register($src2$$reg));
11156 %}
11157
11158 ins_pipe(imul_reg_reg);
11159 %}
11160
11161 // Long Multiply
11162
11163 instruct mulL(iRegLNoSp dst, iRegL src1, iRegL src2) %{
11164 match(Set dst (MulL src1 src2));
11165
11166 ins_cost(INSN_COST * 5);
11167 format %{ "mul $dst, $src1, $src2" %}
11168
11169 ins_encode %{
11170 __ mul(as_Register($dst$$reg),
11171 as_Register($src1$$reg),
11172 as_Register($src2$$reg));
11173 %}
11174
11175 ins_pipe(lmul_reg_reg);
11176 %}
11177
11178 instruct mulHiL_rReg(iRegLNoSp dst, iRegL src1, iRegL src2, rFlagsReg cr)
11179 %{
11180 match(Set dst (MulHiL src1 src2));
11181
11182 ins_cost(INSN_COST * 7);
11183 format %{ "smulh $dst, $src1, $src2, \t# mulhi" %}
11184
11185 ins_encode %{
11186 __ smulh(as_Register($dst$$reg),
11187 as_Register($src1$$reg),
11188 as_Register($src2$$reg));
11189 %}
11190
11191 ins_pipe(lmul_reg_reg);
11192 %}
11193
11194 // Combined Integer Multiply & Add/Sub
11195
11196 instruct maddI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegIorL2I src3) %{
11197 match(Set dst (AddI src3 (MulI src1 src2)));
11198
11199 ins_cost(INSN_COST * 3);
11200 format %{ "madd $dst, $src1, $src2, $src3" %}
11201
11202 ins_encode %{
11203 __ maddw(as_Register($dst$$reg),
11204 as_Register($src1$$reg),
11205 as_Register($src2$$reg),
11206 as_Register($src3$$reg));
11207 %}
11208
11209 ins_pipe(imac_reg_reg);
11210 %}
11211
11212 instruct msubI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegIorL2I src3) %{
11213 match(Set dst (SubI src3 (MulI src1 src2)));
11214
11215 ins_cost(INSN_COST * 3);
11216 format %{ "msub $dst, $src1, $src2, $src3" %}
11217
11218 ins_encode %{
11219 __ msubw(as_Register($dst$$reg),
11220 as_Register($src1$$reg),
11221 as_Register($src2$$reg),
11222 as_Register($src3$$reg));
11223 %}
11224
11225 ins_pipe(imac_reg_reg);
11226 %}
11227
11228 // Combined Integer Multiply & Neg
11229
11230 instruct mnegI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI0 zero) %{
11231 match(Set dst (MulI (SubI zero src1) src2));
11232
11233 ins_cost(INSN_COST * 3);
11234 format %{ "mneg $dst, $src1, $src2" %}
11235
11236 ins_encode %{
11237 __ mnegw(as_Register($dst$$reg),
11238 as_Register($src1$$reg),
11239 as_Register($src2$$reg));
11240 %}
11241
11242 ins_pipe(imac_reg_reg);
11243 %}
11244
11245 // Combined Long Multiply & Add/Sub
11246
11247 instruct maddL(iRegLNoSp dst, iRegL src1, iRegL src2, iRegL src3) %{
11248 match(Set dst (AddL src3 (MulL src1 src2)));
11249
11250 ins_cost(INSN_COST * 5);
11251 format %{ "madd $dst, $src1, $src2, $src3" %}
11252
11253 ins_encode %{
11254 __ madd(as_Register($dst$$reg),
11255 as_Register($src1$$reg),
11256 as_Register($src2$$reg),
11257 as_Register($src3$$reg));
11258 %}
11259
11260 ins_pipe(lmac_reg_reg);
11261 %}
11262
11263 instruct msubL(iRegLNoSp dst, iRegL src1, iRegL src2, iRegL src3) %{
11264 match(Set dst (SubL src3 (MulL src1 src2)));
11265
11266 ins_cost(INSN_COST * 5);
11267 format %{ "msub $dst, $src1, $src2, $src3" %}
11268
11269 ins_encode %{
11270 __ msub(as_Register($dst$$reg),
11271 as_Register($src1$$reg),
11272 as_Register($src2$$reg),
11273 as_Register($src3$$reg));
11274 %}
11275
11276 ins_pipe(lmac_reg_reg);
11277 %}
11278
11279 // Combined Long Multiply & Neg
11280
11281 instruct mnegL(iRegLNoSp dst, iRegL src1, iRegL src2, immL0 zero) %{
11282 match(Set dst (MulL (SubL zero src1) src2));
11283
11284 ins_cost(INSN_COST * 5);
11285 format %{ "mneg $dst, $src1, $src2" %}
11286
11287 ins_encode %{
11288 __ mneg(as_Register($dst$$reg),
11289 as_Register($src1$$reg),
11290 as_Register($src2$$reg));
11291 %}
11292
11293 ins_pipe(lmac_reg_reg);
11294 %}
11295
11296 // Combine Integer Signed Multiply & Add/Sub/Neg Long
11297
11298 instruct smaddL(iRegLNoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegLNoSp src3) %{
11299 match(Set dst (AddL src3 (MulL (ConvI2L src1) (ConvI2L src2))));
11300
11301 ins_cost(INSN_COST * 3);
11302 format %{ "smaddl $dst, $src1, $src2, $src3" %}
11303
11304 ins_encode %{
11305 __ smaddl(as_Register($dst$$reg),
11306 as_Register($src1$$reg),
11307 as_Register($src2$$reg),
11308 as_Register($src3$$reg));
11309 %}
11310
11311 ins_pipe(imac_reg_reg);
11312 %}
11313
11314 instruct smsubL(iRegLNoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegLNoSp src3) %{
11315 match(Set dst (SubL src3 (MulL (ConvI2L src1) (ConvI2L src2))));
11316
11317 ins_cost(INSN_COST * 3);
11318 format %{ "smsubl $dst, $src1, $src2, $src3" %}
11319
11320 ins_encode %{
11321 __ smsubl(as_Register($dst$$reg),
11322 as_Register($src1$$reg),
11323 as_Register($src2$$reg),
11324 as_Register($src3$$reg));
11325 %}
11326
11327 ins_pipe(imac_reg_reg);
11328 %}
11329
11330 instruct smnegL(iRegLNoSp dst, iRegIorL2I src1, iRegIorL2I src2, immL0 zero) %{
11331 match(Set dst (MulL (SubL zero (ConvI2L src1)) (ConvI2L src2)));
11332
11333 ins_cost(INSN_COST * 3);
11334 format %{ "smnegl $dst, $src1, $src2" %}
11335
11336 ins_encode %{
11337 __ smnegl(as_Register($dst$$reg),
11338 as_Register($src1$$reg),
11339 as_Register($src2$$reg));
11340 %}
11341
11342 ins_pipe(imac_reg_reg);
11343 %}
11344
11345 // Combined Multiply-Add Shorts into Integer (dst = src1 * src2 + src3 * src4)
11346
11347 instruct muladdS2I(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegIorL2I src3, iRegIorL2I src4) %{
11348 match(Set dst (MulAddS2I (Binary src1 src2) (Binary src3 src4)));
11349
11350 ins_cost(INSN_COST * 5);
11351 format %{ "mulw rscratch1, $src1, $src2\n\t"
11352 "maddw $dst, $src3, $src4, rscratch1" %}
11353
11354 ins_encode %{
11355 __ mulw(rscratch1, as_Register($src1$$reg), as_Register($src2$$reg));
11356 __ maddw(as_Register($dst$$reg), as_Register($src3$$reg), as_Register($src4$$reg), rscratch1); %}
11357
11358 ins_pipe(imac_reg_reg);
11359 %}
11360
11361 // Integer Divide
11362
11363 instruct divI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
11364 match(Set dst (DivI src1 src2));
11365
11366 ins_cost(INSN_COST * 19);
11367 format %{ "sdivw $dst, $src1, $src2" %}
11368
11369 ins_encode(aarch64_enc_divw(dst, src1, src2));
11370 ins_pipe(idiv_reg_reg);
11371 %}
11372
11373 // Long Divide
11374
11375 instruct divL(iRegLNoSp dst, iRegL src1, iRegL src2) %{
11376 match(Set dst (DivL src1 src2));
11377
11378 ins_cost(INSN_COST * 35);
11379 format %{ "sdiv $dst, $src1, $src2" %}
11380
11381 ins_encode(aarch64_enc_div(dst, src1, src2));
11382 ins_pipe(ldiv_reg_reg);
11383 %}
11384
11385 // Integer Remainder
11386
11387 instruct modI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
11388 match(Set dst (ModI src1 src2));
11389
11390 ins_cost(INSN_COST * 22);
11391 format %{ "sdivw rscratch1, $src1, $src2\n\t"
11392 "msubw($dst, rscratch1, $src2, $src1" %}
11393
11394 ins_encode(aarch64_enc_modw(dst, src1, src2));
11395 ins_pipe(idiv_reg_reg);
11396 %}
11397
11398 // Long Remainder
11399
11400 instruct modL(iRegLNoSp dst, iRegL src1, iRegL src2) %{
11401 match(Set dst (ModL src1 src2));
11402
11403 ins_cost(INSN_COST * 38);
11404 format %{ "sdiv rscratch1, $src1, $src2\n"
11405 "msub($dst, rscratch1, $src2, $src1" %}
11406
11407 ins_encode(aarch64_enc_mod(dst, src1, src2));
11408 ins_pipe(ldiv_reg_reg);
11409 %}
11410
11411 // Integer Shifts
11412
11413 // Shift Left Register
11414 instruct lShiftI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
11415 match(Set dst (LShiftI src1 src2));
11416
11417 ins_cost(INSN_COST * 2);
11418 format %{ "lslvw $dst, $src1, $src2" %}
11419
11420 ins_encode %{
11421 __ lslvw(as_Register($dst$$reg),
11422 as_Register($src1$$reg),
11423 as_Register($src2$$reg));
11424 %}
11425
11426 ins_pipe(ialu_reg_reg_vshift);
11427 %}
11428
11429 // Shift Left Immediate
11430 instruct lShiftI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immI src2) %{
11431 match(Set dst (LShiftI src1 src2));
11432
11433 ins_cost(INSN_COST);
11434 format %{ "lslw $dst, $src1, ($src2 & 0x1f)" %}
11435
11436 ins_encode %{
11437 __ lslw(as_Register($dst$$reg),
11438 as_Register($src1$$reg),
11439 $src2$$constant & 0x1f);
11440 %}
11441
11442 ins_pipe(ialu_reg_shift);
11443 %}
11444
11445 // Shift Right Logical Register
11446 instruct urShiftI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
11447 match(Set dst (URShiftI src1 src2));
11448
11449 ins_cost(INSN_COST * 2);
11450 format %{ "lsrvw $dst, $src1, $src2" %}
11451
11452 ins_encode %{
11453 __ lsrvw(as_Register($dst$$reg),
11454 as_Register($src1$$reg),
11455 as_Register($src2$$reg));
11456 %}
11457
11458 ins_pipe(ialu_reg_reg_vshift);
11459 %}
11460
11461 // Shift Right Logical Immediate
11462 instruct urShiftI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immI src2) %{
11463 match(Set dst (URShiftI src1 src2));
11464
11465 ins_cost(INSN_COST);
11466 format %{ "lsrw $dst, $src1, ($src2 & 0x1f)" %}
11467
11468 ins_encode %{
11469 __ lsrw(as_Register($dst$$reg),
11470 as_Register($src1$$reg),
11471 $src2$$constant & 0x1f);
11472 %}
11473
11474 ins_pipe(ialu_reg_shift);
11475 %}
11476
11477 // Shift Right Arithmetic Register
11478 instruct rShiftI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
11479 match(Set dst (RShiftI src1 src2));
11480
11481 ins_cost(INSN_COST * 2);
11482 format %{ "asrvw $dst, $src1, $src2" %}
11483
11484 ins_encode %{
11485 __ asrvw(as_Register($dst$$reg),
11486 as_Register($src1$$reg),
11487 as_Register($src2$$reg));
11488 %}
11489
11490 ins_pipe(ialu_reg_reg_vshift);
11491 %}
11492
11493 // Shift Right Arithmetic Immediate
11494 instruct rShiftI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immI src2) %{
11495 match(Set dst (RShiftI src1 src2));
11496
11497 ins_cost(INSN_COST);
11498 format %{ "asrw $dst, $src1, ($src2 & 0x1f)" %}
11499
11500 ins_encode %{
11501 __ asrw(as_Register($dst$$reg),
11502 as_Register($src1$$reg),
11503 $src2$$constant & 0x1f);
11504 %}
11505
11506 ins_pipe(ialu_reg_shift);
11507 %}
11508
11509 // Combined Int Mask and Right Shift (using UBFM)
11510 // TODO
11511
11512 // Long Shifts
11513
11514 // Shift Left Register
11515 instruct lShiftL_reg_reg(iRegLNoSp dst, iRegL src1, iRegIorL2I src2) %{
11516 match(Set dst (LShiftL src1 src2));
11517
11518 ins_cost(INSN_COST * 2);
11519 format %{ "lslv $dst, $src1, $src2" %}
11520
11521 ins_encode %{
11522 __ lslv(as_Register($dst$$reg),
11523 as_Register($src1$$reg),
11524 as_Register($src2$$reg));
11525 %}
11526
11527 ins_pipe(ialu_reg_reg_vshift);
11528 %}
11529
11530 // Shift Left Immediate
11531 instruct lShiftL_reg_imm(iRegLNoSp dst, iRegL src1, immI src2) %{
11532 match(Set dst (LShiftL src1 src2));
11533
11534 ins_cost(INSN_COST);
11535 format %{ "lsl $dst, $src1, ($src2 & 0x3f)" %}
11536
11537 ins_encode %{
11538 __ lsl(as_Register($dst$$reg),
11539 as_Register($src1$$reg),
11540 $src2$$constant & 0x3f);
11541 %}
11542
11543 ins_pipe(ialu_reg_shift);
11544 %}
11545
11546 // Shift Right Logical Register
11547 instruct urShiftL_reg_reg(iRegLNoSp dst, iRegL src1, iRegIorL2I src2) %{
11548 match(Set dst (URShiftL src1 src2));
11549
11550 ins_cost(INSN_COST * 2);
11551 format %{ "lsrv $dst, $src1, $src2" %}
11552
11553 ins_encode %{
11554 __ lsrv(as_Register($dst$$reg),
11555 as_Register($src1$$reg),
11556 as_Register($src2$$reg));
11557 %}
11558
11559 ins_pipe(ialu_reg_reg_vshift);
11560 %}
11561
11562 // Shift Right Logical Immediate
11563 instruct urShiftL_reg_imm(iRegLNoSp dst, iRegL src1, immI src2) %{
11564 match(Set dst (URShiftL src1 src2));
11565
11566 ins_cost(INSN_COST);
11567 format %{ "lsr $dst, $src1, ($src2 & 0x3f)" %}
11568
11569 ins_encode %{
11570 __ lsr(as_Register($dst$$reg),
11571 as_Register($src1$$reg),
11572 $src2$$constant & 0x3f);
11573 %}
11574
11575 ins_pipe(ialu_reg_shift);
11576 %}
11577
11578 // A special-case pattern for card table stores.
11579 instruct urShiftP_reg_imm(iRegLNoSp dst, iRegP src1, immI src2) %{
11580 match(Set dst (URShiftL (CastP2X src1) src2));
11581
11582 ins_cost(INSN_COST);
11583 format %{ "lsr $dst, p2x($src1), ($src2 & 0x3f)" %}
11584
11585 ins_encode %{
11586 __ lsr(as_Register($dst$$reg),
11587 as_Register($src1$$reg),
11588 $src2$$constant & 0x3f);
11589 %}
11590
11591 ins_pipe(ialu_reg_shift);
11592 %}
11593
11594 // Shift Right Arithmetic Register
11595 instruct rShiftL_reg_reg(iRegLNoSp dst, iRegL src1, iRegIorL2I src2) %{
11596 match(Set dst (RShiftL src1 src2));
11597
11598 ins_cost(INSN_COST * 2);
11599 format %{ "asrv $dst, $src1, $src2" %}
11600
11601 ins_encode %{
11602 __ asrv(as_Register($dst$$reg),
11603 as_Register($src1$$reg),
11604 as_Register($src2$$reg));
11605 %}
11606
11607 ins_pipe(ialu_reg_reg_vshift);
11608 %}
11609
11610 // Shift Right Arithmetic Immediate
11611 instruct rShiftL_reg_imm(iRegLNoSp dst, iRegL src1, immI src2) %{
11612 match(Set dst (RShiftL src1 src2));
11613
11614 ins_cost(INSN_COST);
11615 format %{ "asr $dst, $src1, ($src2 & 0x3f)" %}
11616
11617 ins_encode %{
11618 __ asr(as_Register($dst$$reg),
11619 as_Register($src1$$reg),
11620 $src2$$constant & 0x3f);
11621 %}
11622
11623 ins_pipe(ialu_reg_shift);
11624 %}
11625
11626 // BEGIN This section of the file is automatically generated. Do not edit --------------
11627 // This section is generated from aarch64_ad.m4
11628
11629
11630 // This pattern is automatically generated from aarch64_ad.m4
11631 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11632 instruct regL_not_reg(iRegLNoSp dst,
11633 iRegL src1, immL_M1 m1,
11634 rFlagsReg cr) %{
11635 match(Set dst (XorL src1 m1));
11636 ins_cost(INSN_COST);
11637 format %{ "eon $dst, $src1, zr" %}
11638
11639 ins_encode %{
11640 __ eon(as_Register($dst$$reg),
11641 as_Register($src1$$reg),
11642 zr,
11643 Assembler::LSL, 0);
11644 %}
11645
11646 ins_pipe(ialu_reg);
11647 %}
11648
11649 // This pattern is automatically generated from aarch64_ad.m4
11650 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11651 instruct regI_not_reg(iRegINoSp dst,
11652 iRegIorL2I src1, immI_M1 m1,
11653 rFlagsReg cr) %{
11654 match(Set dst (XorI src1 m1));
11655 ins_cost(INSN_COST);
11656 format %{ "eonw $dst, $src1, zr" %}
11657
11658 ins_encode %{
11659 __ eonw(as_Register($dst$$reg),
11660 as_Register($src1$$reg),
11661 zr,
11662 Assembler::LSL, 0);
11663 %}
11664
11665 ins_pipe(ialu_reg);
11666 %}
11667
11668 // This pattern is automatically generated from aarch64_ad.m4
11669 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11670 instruct AndI_reg_not_reg(iRegINoSp dst,
11671 iRegIorL2I src1, iRegIorL2I src2, immI_M1 m1) %{
11672 match(Set dst (AndI src1 (XorI src2 m1)));
11673 ins_cost(INSN_COST);
11674 format %{ "bicw $dst, $src1, $src2" %}
11675
11676 ins_encode %{
11677 __ bicw(as_Register($dst$$reg),
11678 as_Register($src1$$reg),
11679 as_Register($src2$$reg),
11680 Assembler::LSL, 0);
11681 %}
11682
11683 ins_pipe(ialu_reg_reg);
11684 %}
11685
11686 // This pattern is automatically generated from aarch64_ad.m4
11687 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11688 instruct AndL_reg_not_reg(iRegLNoSp dst,
11689 iRegL src1, iRegL src2, immL_M1 m1) %{
11690 match(Set dst (AndL src1 (XorL src2 m1)));
11691 ins_cost(INSN_COST);
11692 format %{ "bic $dst, $src1, $src2" %}
11693
11694 ins_encode %{
11695 __ bic(as_Register($dst$$reg),
11696 as_Register($src1$$reg),
11697 as_Register($src2$$reg),
11698 Assembler::LSL, 0);
11699 %}
11700
11701 ins_pipe(ialu_reg_reg);
11702 %}
11703
11704 // This pattern is automatically generated from aarch64_ad.m4
11705 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11706 instruct OrI_reg_not_reg(iRegINoSp dst,
11707 iRegIorL2I src1, iRegIorL2I src2, immI_M1 m1) %{
11708 match(Set dst (OrI src1 (XorI src2 m1)));
11709 ins_cost(INSN_COST);
11710 format %{ "ornw $dst, $src1, $src2" %}
11711
11712 ins_encode %{
11713 __ ornw(as_Register($dst$$reg),
11714 as_Register($src1$$reg),
11715 as_Register($src2$$reg),
11716 Assembler::LSL, 0);
11717 %}
11718
11719 ins_pipe(ialu_reg_reg);
11720 %}
11721
11722 // This pattern is automatically generated from aarch64_ad.m4
11723 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11724 instruct OrL_reg_not_reg(iRegLNoSp dst,
11725 iRegL src1, iRegL src2, immL_M1 m1) %{
11726 match(Set dst (OrL src1 (XorL src2 m1)));
11727 ins_cost(INSN_COST);
11728 format %{ "orn $dst, $src1, $src2" %}
11729
11730 ins_encode %{
11731 __ orn(as_Register($dst$$reg),
11732 as_Register($src1$$reg),
11733 as_Register($src2$$reg),
11734 Assembler::LSL, 0);
11735 %}
11736
11737 ins_pipe(ialu_reg_reg);
11738 %}
11739
11740 // This pattern is automatically generated from aarch64_ad.m4
11741 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11742 instruct XorI_reg_not_reg(iRegINoSp dst,
11743 iRegIorL2I src1, iRegIorL2I src2, immI_M1 m1) %{
11744 match(Set dst (XorI m1 (XorI src2 src1)));
11745 ins_cost(INSN_COST);
11746 format %{ "eonw $dst, $src1, $src2" %}
11747
11748 ins_encode %{
11749 __ eonw(as_Register($dst$$reg),
11750 as_Register($src1$$reg),
11751 as_Register($src2$$reg),
11752 Assembler::LSL, 0);
11753 %}
11754
11755 ins_pipe(ialu_reg_reg);
11756 %}
11757
11758 // This pattern is automatically generated from aarch64_ad.m4
11759 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11760 instruct XorL_reg_not_reg(iRegLNoSp dst,
11761 iRegL src1, iRegL src2, immL_M1 m1) %{
11762 match(Set dst (XorL m1 (XorL src2 src1)));
11763 ins_cost(INSN_COST);
11764 format %{ "eon $dst, $src1, $src2" %}
11765
11766 ins_encode %{
11767 __ eon(as_Register($dst$$reg),
11768 as_Register($src1$$reg),
11769 as_Register($src2$$reg),
11770 Assembler::LSL, 0);
11771 %}
11772
11773 ins_pipe(ialu_reg_reg);
11774 %}
11775
11776 // This pattern is automatically generated from aarch64_ad.m4
11777 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11778 // val & (-1 ^ (val >>> shift)) ==> bicw
11779 instruct AndI_reg_URShift_not_reg(iRegINoSp dst,
11780 iRegIorL2I src1, iRegIorL2I src2,
11781 immI src3, immI_M1 src4) %{
11782 match(Set dst (AndI src1 (XorI(URShiftI src2 src3) src4)));
11783 ins_cost(1.9 * INSN_COST);
11784 format %{ "bicw $dst, $src1, $src2, LSR $src3" %}
11785
11786 ins_encode %{
11787 __ bicw(as_Register($dst$$reg),
11788 as_Register($src1$$reg),
11789 as_Register($src2$$reg),
11790 Assembler::LSR,
11791 $src3$$constant & 0x1f);
11792 %}
11793
11794 ins_pipe(ialu_reg_reg_shift);
11795 %}
11796
11797 // This pattern is automatically generated from aarch64_ad.m4
11798 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11799 // val & (-1 ^ (val >>> shift)) ==> bic
11800 instruct AndL_reg_URShift_not_reg(iRegLNoSp dst,
11801 iRegL src1, iRegL src2,
11802 immI src3, immL_M1 src4) %{
11803 match(Set dst (AndL src1 (XorL(URShiftL src2 src3) src4)));
11804 ins_cost(1.9 * INSN_COST);
11805 format %{ "bic $dst, $src1, $src2, LSR $src3" %}
11806
11807 ins_encode %{
11808 __ bic(as_Register($dst$$reg),
11809 as_Register($src1$$reg),
11810 as_Register($src2$$reg),
11811 Assembler::LSR,
11812 $src3$$constant & 0x3f);
11813 %}
11814
11815 ins_pipe(ialu_reg_reg_shift);
11816 %}
11817
11818 // This pattern is automatically generated from aarch64_ad.m4
11819 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11820 // val & (-1 ^ (val >> shift)) ==> bicw
11821 instruct AndI_reg_RShift_not_reg(iRegINoSp dst,
11822 iRegIorL2I src1, iRegIorL2I src2,
11823 immI src3, immI_M1 src4) %{
11824 match(Set dst (AndI src1 (XorI(RShiftI src2 src3) src4)));
11825 ins_cost(1.9 * INSN_COST);
11826 format %{ "bicw $dst, $src1, $src2, ASR $src3" %}
11827
11828 ins_encode %{
11829 __ bicw(as_Register($dst$$reg),
11830 as_Register($src1$$reg),
11831 as_Register($src2$$reg),
11832 Assembler::ASR,
11833 $src3$$constant & 0x1f);
11834 %}
11835
11836 ins_pipe(ialu_reg_reg_shift);
11837 %}
11838
11839 // This pattern is automatically generated from aarch64_ad.m4
11840 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11841 // val & (-1 ^ (val >> shift)) ==> bic
11842 instruct AndL_reg_RShift_not_reg(iRegLNoSp dst,
11843 iRegL src1, iRegL src2,
11844 immI src3, immL_M1 src4) %{
11845 match(Set dst (AndL src1 (XorL(RShiftL src2 src3) src4)));
11846 ins_cost(1.9 * INSN_COST);
11847 format %{ "bic $dst, $src1, $src2, ASR $src3" %}
11848
11849 ins_encode %{
11850 __ bic(as_Register($dst$$reg),
11851 as_Register($src1$$reg),
11852 as_Register($src2$$reg),
11853 Assembler::ASR,
11854 $src3$$constant & 0x3f);
11855 %}
11856
11857 ins_pipe(ialu_reg_reg_shift);
11858 %}
11859
11860 // This pattern is automatically generated from aarch64_ad.m4
11861 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11862 // val & (-1 ^ (val ror shift)) ==> bicw
11863 instruct AndI_reg_RotateRight_not_reg(iRegINoSp dst,
11864 iRegIorL2I src1, iRegIorL2I src2,
11865 immI src3, immI_M1 src4) %{
11866 match(Set dst (AndI src1 (XorI(RotateRight src2 src3) src4)));
11867 ins_cost(1.9 * INSN_COST);
11868 format %{ "bicw $dst, $src1, $src2, ROR $src3" %}
11869
11870 ins_encode %{
11871 __ bicw(as_Register($dst$$reg),
11872 as_Register($src1$$reg),
11873 as_Register($src2$$reg),
11874 Assembler::ROR,
11875 $src3$$constant & 0x1f);
11876 %}
11877
11878 ins_pipe(ialu_reg_reg_shift);
11879 %}
11880
11881 // This pattern is automatically generated from aarch64_ad.m4
11882 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11883 // val & (-1 ^ (val ror shift)) ==> bic
11884 instruct AndL_reg_RotateRight_not_reg(iRegLNoSp dst,
11885 iRegL src1, iRegL src2,
11886 immI src3, immL_M1 src4) %{
11887 match(Set dst (AndL src1 (XorL(RotateRight src2 src3) src4)));
11888 ins_cost(1.9 * INSN_COST);
11889 format %{ "bic $dst, $src1, $src2, ROR $src3" %}
11890
11891 ins_encode %{
11892 __ bic(as_Register($dst$$reg),
11893 as_Register($src1$$reg),
11894 as_Register($src2$$reg),
11895 Assembler::ROR,
11896 $src3$$constant & 0x3f);
11897 %}
11898
11899 ins_pipe(ialu_reg_reg_shift);
11900 %}
11901
11902 // This pattern is automatically generated from aarch64_ad.m4
11903 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11904 // val & (-1 ^ (val << shift)) ==> bicw
11905 instruct AndI_reg_LShift_not_reg(iRegINoSp dst,
11906 iRegIorL2I src1, iRegIorL2I src2,
11907 immI src3, immI_M1 src4) %{
11908 match(Set dst (AndI src1 (XorI(LShiftI src2 src3) src4)));
11909 ins_cost(1.9 * INSN_COST);
11910 format %{ "bicw $dst, $src1, $src2, LSL $src3" %}
11911
11912 ins_encode %{
11913 __ bicw(as_Register($dst$$reg),
11914 as_Register($src1$$reg),
11915 as_Register($src2$$reg),
11916 Assembler::LSL,
11917 $src3$$constant & 0x1f);
11918 %}
11919
11920 ins_pipe(ialu_reg_reg_shift);
11921 %}
11922
11923 // This pattern is automatically generated from aarch64_ad.m4
11924 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11925 // val & (-1 ^ (val << shift)) ==> bic
11926 instruct AndL_reg_LShift_not_reg(iRegLNoSp dst,
11927 iRegL src1, iRegL src2,
11928 immI src3, immL_M1 src4) %{
11929 match(Set dst (AndL src1 (XorL(LShiftL src2 src3) src4)));
11930 ins_cost(1.9 * INSN_COST);
11931 format %{ "bic $dst, $src1, $src2, LSL $src3" %}
11932
11933 ins_encode %{
11934 __ bic(as_Register($dst$$reg),
11935 as_Register($src1$$reg),
11936 as_Register($src2$$reg),
11937 Assembler::LSL,
11938 $src3$$constant & 0x3f);
11939 %}
11940
11941 ins_pipe(ialu_reg_reg_shift);
11942 %}
11943
11944 // This pattern is automatically generated from aarch64_ad.m4
11945 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11946 // val ^ (-1 ^ (val >>> shift)) ==> eonw
11947 instruct XorI_reg_URShift_not_reg(iRegINoSp dst,
11948 iRegIorL2I src1, iRegIorL2I src2,
11949 immI src3, immI_M1 src4) %{
11950 match(Set dst (XorI src4 (XorI(URShiftI src2 src3) src1)));
11951 ins_cost(1.9 * INSN_COST);
11952 format %{ "eonw $dst, $src1, $src2, LSR $src3" %}
11953
11954 ins_encode %{
11955 __ eonw(as_Register($dst$$reg),
11956 as_Register($src1$$reg),
11957 as_Register($src2$$reg),
11958 Assembler::LSR,
11959 $src3$$constant & 0x1f);
11960 %}
11961
11962 ins_pipe(ialu_reg_reg_shift);
11963 %}
11964
11965 // This pattern is automatically generated from aarch64_ad.m4
11966 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11967 // val ^ (-1 ^ (val >>> shift)) ==> eon
11968 instruct XorL_reg_URShift_not_reg(iRegLNoSp dst,
11969 iRegL src1, iRegL src2,
11970 immI src3, immL_M1 src4) %{
11971 match(Set dst (XorL src4 (XorL(URShiftL src2 src3) src1)));
11972 ins_cost(1.9 * INSN_COST);
11973 format %{ "eon $dst, $src1, $src2, LSR $src3" %}
11974
11975 ins_encode %{
11976 __ eon(as_Register($dst$$reg),
11977 as_Register($src1$$reg),
11978 as_Register($src2$$reg),
11979 Assembler::LSR,
11980 $src3$$constant & 0x3f);
11981 %}
11982
11983 ins_pipe(ialu_reg_reg_shift);
11984 %}
11985
11986 // This pattern is automatically generated from aarch64_ad.m4
11987 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11988 // val ^ (-1 ^ (val >> shift)) ==> eonw
11989 instruct XorI_reg_RShift_not_reg(iRegINoSp dst,
11990 iRegIorL2I src1, iRegIorL2I src2,
11991 immI src3, immI_M1 src4) %{
11992 match(Set dst (XorI src4 (XorI(RShiftI src2 src3) src1)));
11993 ins_cost(1.9 * INSN_COST);
11994 format %{ "eonw $dst, $src1, $src2, ASR $src3" %}
11995
11996 ins_encode %{
11997 __ eonw(as_Register($dst$$reg),
11998 as_Register($src1$$reg),
11999 as_Register($src2$$reg),
12000 Assembler::ASR,
12001 $src3$$constant & 0x1f);
12002 %}
12003
12004 ins_pipe(ialu_reg_reg_shift);
12005 %}
12006
12007 // This pattern is automatically generated from aarch64_ad.m4
12008 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12009 // val ^ (-1 ^ (val >> shift)) ==> eon
12010 instruct XorL_reg_RShift_not_reg(iRegLNoSp dst,
12011 iRegL src1, iRegL src2,
12012 immI src3, immL_M1 src4) %{
12013 match(Set dst (XorL src4 (XorL(RShiftL src2 src3) src1)));
12014 ins_cost(1.9 * INSN_COST);
12015 format %{ "eon $dst, $src1, $src2, ASR $src3" %}
12016
12017 ins_encode %{
12018 __ eon(as_Register($dst$$reg),
12019 as_Register($src1$$reg),
12020 as_Register($src2$$reg),
12021 Assembler::ASR,
12022 $src3$$constant & 0x3f);
12023 %}
12024
12025 ins_pipe(ialu_reg_reg_shift);
12026 %}
12027
12028 // This pattern is automatically generated from aarch64_ad.m4
12029 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12030 // val ^ (-1 ^ (val ror shift)) ==> eonw
12031 instruct XorI_reg_RotateRight_not_reg(iRegINoSp dst,
12032 iRegIorL2I src1, iRegIorL2I src2,
12033 immI src3, immI_M1 src4) %{
12034 match(Set dst (XorI src4 (XorI(RotateRight src2 src3) src1)));
12035 ins_cost(1.9 * INSN_COST);
12036 format %{ "eonw $dst, $src1, $src2, ROR $src3" %}
12037
12038 ins_encode %{
12039 __ eonw(as_Register($dst$$reg),
12040 as_Register($src1$$reg),
12041 as_Register($src2$$reg),
12042 Assembler::ROR,
12043 $src3$$constant & 0x1f);
12044 %}
12045
12046 ins_pipe(ialu_reg_reg_shift);
12047 %}
12048
12049 // This pattern is automatically generated from aarch64_ad.m4
12050 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12051 // val ^ (-1 ^ (val ror shift)) ==> eon
12052 instruct XorL_reg_RotateRight_not_reg(iRegLNoSp dst,
12053 iRegL src1, iRegL src2,
12054 immI src3, immL_M1 src4) %{
12055 match(Set dst (XorL src4 (XorL(RotateRight src2 src3) src1)));
12056 ins_cost(1.9 * INSN_COST);
12057 format %{ "eon $dst, $src1, $src2, ROR $src3" %}
12058
12059 ins_encode %{
12060 __ eon(as_Register($dst$$reg),
12061 as_Register($src1$$reg),
12062 as_Register($src2$$reg),
12063 Assembler::ROR,
12064 $src3$$constant & 0x3f);
12065 %}
12066
12067 ins_pipe(ialu_reg_reg_shift);
12068 %}
12069
12070 // This pattern is automatically generated from aarch64_ad.m4
12071 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12072 // val ^ (-1 ^ (val << shift)) ==> eonw
12073 instruct XorI_reg_LShift_not_reg(iRegINoSp dst,
12074 iRegIorL2I src1, iRegIorL2I src2,
12075 immI src3, immI_M1 src4) %{
12076 match(Set dst (XorI src4 (XorI(LShiftI src2 src3) src1)));
12077 ins_cost(1.9 * INSN_COST);
12078 format %{ "eonw $dst, $src1, $src2, LSL $src3" %}
12079
12080 ins_encode %{
12081 __ eonw(as_Register($dst$$reg),
12082 as_Register($src1$$reg),
12083 as_Register($src2$$reg),
12084 Assembler::LSL,
12085 $src3$$constant & 0x1f);
12086 %}
12087
12088 ins_pipe(ialu_reg_reg_shift);
12089 %}
12090
12091 // This pattern is automatically generated from aarch64_ad.m4
12092 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12093 // val ^ (-1 ^ (val << shift)) ==> eon
12094 instruct XorL_reg_LShift_not_reg(iRegLNoSp dst,
12095 iRegL src1, iRegL src2,
12096 immI src3, immL_M1 src4) %{
12097 match(Set dst (XorL src4 (XorL(LShiftL src2 src3) src1)));
12098 ins_cost(1.9 * INSN_COST);
12099 format %{ "eon $dst, $src1, $src2, LSL $src3" %}
12100
12101 ins_encode %{
12102 __ eon(as_Register($dst$$reg),
12103 as_Register($src1$$reg),
12104 as_Register($src2$$reg),
12105 Assembler::LSL,
12106 $src3$$constant & 0x3f);
12107 %}
12108
12109 ins_pipe(ialu_reg_reg_shift);
12110 %}
12111
12112 // This pattern is automatically generated from aarch64_ad.m4
12113 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12114 // val | (-1 ^ (val >>> shift)) ==> ornw
12115 instruct OrI_reg_URShift_not_reg(iRegINoSp dst,
12116 iRegIorL2I src1, iRegIorL2I src2,
12117 immI src3, immI_M1 src4) %{
12118 match(Set dst (OrI src1 (XorI(URShiftI src2 src3) src4)));
12119 ins_cost(1.9 * INSN_COST);
12120 format %{ "ornw $dst, $src1, $src2, LSR $src3" %}
12121
12122 ins_encode %{
12123 __ ornw(as_Register($dst$$reg),
12124 as_Register($src1$$reg),
12125 as_Register($src2$$reg),
12126 Assembler::LSR,
12127 $src3$$constant & 0x1f);
12128 %}
12129
12130 ins_pipe(ialu_reg_reg_shift);
12131 %}
12132
12133 // This pattern is automatically generated from aarch64_ad.m4
12134 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12135 // val | (-1 ^ (val >>> shift)) ==> orn
12136 instruct OrL_reg_URShift_not_reg(iRegLNoSp dst,
12137 iRegL src1, iRegL src2,
12138 immI src3, immL_M1 src4) %{
12139 match(Set dst (OrL src1 (XorL(URShiftL src2 src3) src4)));
12140 ins_cost(1.9 * INSN_COST);
12141 format %{ "orn $dst, $src1, $src2, LSR $src3" %}
12142
12143 ins_encode %{
12144 __ orn(as_Register($dst$$reg),
12145 as_Register($src1$$reg),
12146 as_Register($src2$$reg),
12147 Assembler::LSR,
12148 $src3$$constant & 0x3f);
12149 %}
12150
12151 ins_pipe(ialu_reg_reg_shift);
12152 %}
12153
12154 // This pattern is automatically generated from aarch64_ad.m4
12155 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12156 // val | (-1 ^ (val >> shift)) ==> ornw
12157 instruct OrI_reg_RShift_not_reg(iRegINoSp dst,
12158 iRegIorL2I src1, iRegIorL2I src2,
12159 immI src3, immI_M1 src4) %{
12160 match(Set dst (OrI src1 (XorI(RShiftI src2 src3) src4)));
12161 ins_cost(1.9 * INSN_COST);
12162 format %{ "ornw $dst, $src1, $src2, ASR $src3" %}
12163
12164 ins_encode %{
12165 __ ornw(as_Register($dst$$reg),
12166 as_Register($src1$$reg),
12167 as_Register($src2$$reg),
12168 Assembler::ASR,
12169 $src3$$constant & 0x1f);
12170 %}
12171
12172 ins_pipe(ialu_reg_reg_shift);
12173 %}
12174
12175 // This pattern is automatically generated from aarch64_ad.m4
12176 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12177 // val | (-1 ^ (val >> shift)) ==> orn
12178 instruct OrL_reg_RShift_not_reg(iRegLNoSp dst,
12179 iRegL src1, iRegL src2,
12180 immI src3, immL_M1 src4) %{
12181 match(Set dst (OrL src1 (XorL(RShiftL src2 src3) src4)));
12182 ins_cost(1.9 * INSN_COST);
12183 format %{ "orn $dst, $src1, $src2, ASR $src3" %}
12184
12185 ins_encode %{
12186 __ orn(as_Register($dst$$reg),
12187 as_Register($src1$$reg),
12188 as_Register($src2$$reg),
12189 Assembler::ASR,
12190 $src3$$constant & 0x3f);
12191 %}
12192
12193 ins_pipe(ialu_reg_reg_shift);
12194 %}
12195
12196 // This pattern is automatically generated from aarch64_ad.m4
12197 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12198 // val | (-1 ^ (val ror shift)) ==> ornw
12199 instruct OrI_reg_RotateRight_not_reg(iRegINoSp dst,
12200 iRegIorL2I src1, iRegIorL2I src2,
12201 immI src3, immI_M1 src4) %{
12202 match(Set dst (OrI src1 (XorI(RotateRight src2 src3) src4)));
12203 ins_cost(1.9 * INSN_COST);
12204 format %{ "ornw $dst, $src1, $src2, ROR $src3" %}
12205
12206 ins_encode %{
12207 __ ornw(as_Register($dst$$reg),
12208 as_Register($src1$$reg),
12209 as_Register($src2$$reg),
12210 Assembler::ROR,
12211 $src3$$constant & 0x1f);
12212 %}
12213
12214 ins_pipe(ialu_reg_reg_shift);
12215 %}
12216
12217 // This pattern is automatically generated from aarch64_ad.m4
12218 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12219 // val | (-1 ^ (val ror shift)) ==> orn
12220 instruct OrL_reg_RotateRight_not_reg(iRegLNoSp dst,
12221 iRegL src1, iRegL src2,
12222 immI src3, immL_M1 src4) %{
12223 match(Set dst (OrL src1 (XorL(RotateRight src2 src3) src4)));
12224 ins_cost(1.9 * INSN_COST);
12225 format %{ "orn $dst, $src1, $src2, ROR $src3" %}
12226
12227 ins_encode %{
12228 __ orn(as_Register($dst$$reg),
12229 as_Register($src1$$reg),
12230 as_Register($src2$$reg),
12231 Assembler::ROR,
12232 $src3$$constant & 0x3f);
12233 %}
12234
12235 ins_pipe(ialu_reg_reg_shift);
12236 %}
12237
12238 // This pattern is automatically generated from aarch64_ad.m4
12239 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12240 // val | (-1 ^ (val << shift)) ==> ornw
12241 instruct OrI_reg_LShift_not_reg(iRegINoSp dst,
12242 iRegIorL2I src1, iRegIorL2I src2,
12243 immI src3, immI_M1 src4) %{
12244 match(Set dst (OrI src1 (XorI(LShiftI src2 src3) src4)));
12245 ins_cost(1.9 * INSN_COST);
12246 format %{ "ornw $dst, $src1, $src2, LSL $src3" %}
12247
12248 ins_encode %{
12249 __ ornw(as_Register($dst$$reg),
12250 as_Register($src1$$reg),
12251 as_Register($src2$$reg),
12252 Assembler::LSL,
12253 $src3$$constant & 0x1f);
12254 %}
12255
12256 ins_pipe(ialu_reg_reg_shift);
12257 %}
12258
12259 // This pattern is automatically generated from aarch64_ad.m4
12260 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12261 // val | (-1 ^ (val << shift)) ==> orn
12262 instruct OrL_reg_LShift_not_reg(iRegLNoSp dst,
12263 iRegL src1, iRegL src2,
12264 immI src3, immL_M1 src4) %{
12265 match(Set dst (OrL src1 (XorL(LShiftL src2 src3) src4)));
12266 ins_cost(1.9 * INSN_COST);
12267 format %{ "orn $dst, $src1, $src2, LSL $src3" %}
12268
12269 ins_encode %{
12270 __ orn(as_Register($dst$$reg),
12271 as_Register($src1$$reg),
12272 as_Register($src2$$reg),
12273 Assembler::LSL,
12274 $src3$$constant & 0x3f);
12275 %}
12276
12277 ins_pipe(ialu_reg_reg_shift);
12278 %}
12279
12280 // This pattern is automatically generated from aarch64_ad.m4
12281 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12282 instruct AndI_reg_URShift_reg(iRegINoSp dst,
12283 iRegIorL2I src1, iRegIorL2I src2,
12284 immI src3) %{
12285 match(Set dst (AndI src1 (URShiftI src2 src3)));
12286
12287 ins_cost(1.9 * INSN_COST);
12288 format %{ "andw $dst, $src1, $src2, LSR $src3" %}
12289
12290 ins_encode %{
12291 __ andw(as_Register($dst$$reg),
12292 as_Register($src1$$reg),
12293 as_Register($src2$$reg),
12294 Assembler::LSR,
12295 $src3$$constant & 0x1f);
12296 %}
12297
12298 ins_pipe(ialu_reg_reg_shift);
12299 %}
12300
12301 // This pattern is automatically generated from aarch64_ad.m4
12302 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12303 instruct AndL_reg_URShift_reg(iRegLNoSp dst,
12304 iRegL src1, iRegL src2,
12305 immI src3) %{
12306 match(Set dst (AndL src1 (URShiftL src2 src3)));
12307
12308 ins_cost(1.9 * INSN_COST);
12309 format %{ "andr $dst, $src1, $src2, LSR $src3" %}
12310
12311 ins_encode %{
12312 __ andr(as_Register($dst$$reg),
12313 as_Register($src1$$reg),
12314 as_Register($src2$$reg),
12315 Assembler::LSR,
12316 $src3$$constant & 0x3f);
12317 %}
12318
12319 ins_pipe(ialu_reg_reg_shift);
12320 %}
12321
12322 // This pattern is automatically generated from aarch64_ad.m4
12323 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12324 instruct AndI_reg_RShift_reg(iRegINoSp dst,
12325 iRegIorL2I src1, iRegIorL2I src2,
12326 immI src3) %{
12327 match(Set dst (AndI src1 (RShiftI src2 src3)));
12328
12329 ins_cost(1.9 * INSN_COST);
12330 format %{ "andw $dst, $src1, $src2, ASR $src3" %}
12331
12332 ins_encode %{
12333 __ andw(as_Register($dst$$reg),
12334 as_Register($src1$$reg),
12335 as_Register($src2$$reg),
12336 Assembler::ASR,
12337 $src3$$constant & 0x1f);
12338 %}
12339
12340 ins_pipe(ialu_reg_reg_shift);
12341 %}
12342
12343 // This pattern is automatically generated from aarch64_ad.m4
12344 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12345 instruct AndL_reg_RShift_reg(iRegLNoSp dst,
12346 iRegL src1, iRegL src2,
12347 immI src3) %{
12348 match(Set dst (AndL src1 (RShiftL src2 src3)));
12349
12350 ins_cost(1.9 * INSN_COST);
12351 format %{ "andr $dst, $src1, $src2, ASR $src3" %}
12352
12353 ins_encode %{
12354 __ andr(as_Register($dst$$reg),
12355 as_Register($src1$$reg),
12356 as_Register($src2$$reg),
12357 Assembler::ASR,
12358 $src3$$constant & 0x3f);
12359 %}
12360
12361 ins_pipe(ialu_reg_reg_shift);
12362 %}
12363
12364 // This pattern is automatically generated from aarch64_ad.m4
12365 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12366 instruct AndI_reg_LShift_reg(iRegINoSp dst,
12367 iRegIorL2I src1, iRegIorL2I src2,
12368 immI src3) %{
12369 match(Set dst (AndI src1 (LShiftI src2 src3)));
12370
12371 ins_cost(1.9 * INSN_COST);
12372 format %{ "andw $dst, $src1, $src2, LSL $src3" %}
12373
12374 ins_encode %{
12375 __ andw(as_Register($dst$$reg),
12376 as_Register($src1$$reg),
12377 as_Register($src2$$reg),
12378 Assembler::LSL,
12379 $src3$$constant & 0x1f);
12380 %}
12381
12382 ins_pipe(ialu_reg_reg_shift);
12383 %}
12384
12385 // This pattern is automatically generated from aarch64_ad.m4
12386 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12387 instruct AndL_reg_LShift_reg(iRegLNoSp dst,
12388 iRegL src1, iRegL src2,
12389 immI src3) %{
12390 match(Set dst (AndL src1 (LShiftL src2 src3)));
12391
12392 ins_cost(1.9 * INSN_COST);
12393 format %{ "andr $dst, $src1, $src2, LSL $src3" %}
12394
12395 ins_encode %{
12396 __ andr(as_Register($dst$$reg),
12397 as_Register($src1$$reg),
12398 as_Register($src2$$reg),
12399 Assembler::LSL,
12400 $src3$$constant & 0x3f);
12401 %}
12402
12403 ins_pipe(ialu_reg_reg_shift);
12404 %}
12405
12406 // This pattern is automatically generated from aarch64_ad.m4
12407 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12408 instruct AndI_reg_RotateRight_reg(iRegINoSp dst,
12409 iRegIorL2I src1, iRegIorL2I src2,
12410 immI src3) %{
12411 match(Set dst (AndI src1 (RotateRight src2 src3)));
12412
12413 ins_cost(1.9 * INSN_COST);
12414 format %{ "andw $dst, $src1, $src2, ROR $src3" %}
12415
12416 ins_encode %{
12417 __ andw(as_Register($dst$$reg),
12418 as_Register($src1$$reg),
12419 as_Register($src2$$reg),
12420 Assembler::ROR,
12421 $src3$$constant & 0x1f);
12422 %}
12423
12424 ins_pipe(ialu_reg_reg_shift);
12425 %}
12426
12427 // This pattern is automatically generated from aarch64_ad.m4
12428 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12429 instruct AndL_reg_RotateRight_reg(iRegLNoSp dst,
12430 iRegL src1, iRegL src2,
12431 immI src3) %{
12432 match(Set dst (AndL src1 (RotateRight src2 src3)));
12433
12434 ins_cost(1.9 * INSN_COST);
12435 format %{ "andr $dst, $src1, $src2, ROR $src3" %}
12436
12437 ins_encode %{
12438 __ andr(as_Register($dst$$reg),
12439 as_Register($src1$$reg),
12440 as_Register($src2$$reg),
12441 Assembler::ROR,
12442 $src3$$constant & 0x3f);
12443 %}
12444
12445 ins_pipe(ialu_reg_reg_shift);
12446 %}
12447
12448 // This pattern is automatically generated from aarch64_ad.m4
12449 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12450 instruct XorI_reg_URShift_reg(iRegINoSp dst,
12451 iRegIorL2I src1, iRegIorL2I src2,
12452 immI src3) %{
12453 match(Set dst (XorI src1 (URShiftI src2 src3)));
12454
12455 ins_cost(1.9 * INSN_COST);
12456 format %{ "eorw $dst, $src1, $src2, LSR $src3" %}
12457
12458 ins_encode %{
12459 __ eorw(as_Register($dst$$reg),
12460 as_Register($src1$$reg),
12461 as_Register($src2$$reg),
12462 Assembler::LSR,
12463 $src3$$constant & 0x1f);
12464 %}
12465
12466 ins_pipe(ialu_reg_reg_shift);
12467 %}
12468
12469 // This pattern is automatically generated from aarch64_ad.m4
12470 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12471 instruct XorL_reg_URShift_reg(iRegLNoSp dst,
12472 iRegL src1, iRegL src2,
12473 immI src3) %{
12474 match(Set dst (XorL src1 (URShiftL src2 src3)));
12475
12476 ins_cost(1.9 * INSN_COST);
12477 format %{ "eor $dst, $src1, $src2, LSR $src3" %}
12478
12479 ins_encode %{
12480 __ eor(as_Register($dst$$reg),
12481 as_Register($src1$$reg),
12482 as_Register($src2$$reg),
12483 Assembler::LSR,
12484 $src3$$constant & 0x3f);
12485 %}
12486
12487 ins_pipe(ialu_reg_reg_shift);
12488 %}
12489
12490 // This pattern is automatically generated from aarch64_ad.m4
12491 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12492 instruct XorI_reg_RShift_reg(iRegINoSp dst,
12493 iRegIorL2I src1, iRegIorL2I src2,
12494 immI src3) %{
12495 match(Set dst (XorI src1 (RShiftI src2 src3)));
12496
12497 ins_cost(1.9 * INSN_COST);
12498 format %{ "eorw $dst, $src1, $src2, ASR $src3" %}
12499
12500 ins_encode %{
12501 __ eorw(as_Register($dst$$reg),
12502 as_Register($src1$$reg),
12503 as_Register($src2$$reg),
12504 Assembler::ASR,
12505 $src3$$constant & 0x1f);
12506 %}
12507
12508 ins_pipe(ialu_reg_reg_shift);
12509 %}
12510
12511 // This pattern is automatically generated from aarch64_ad.m4
12512 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12513 instruct XorL_reg_RShift_reg(iRegLNoSp dst,
12514 iRegL src1, iRegL src2,
12515 immI src3) %{
12516 match(Set dst (XorL src1 (RShiftL src2 src3)));
12517
12518 ins_cost(1.9 * INSN_COST);
12519 format %{ "eor $dst, $src1, $src2, ASR $src3" %}
12520
12521 ins_encode %{
12522 __ eor(as_Register($dst$$reg),
12523 as_Register($src1$$reg),
12524 as_Register($src2$$reg),
12525 Assembler::ASR,
12526 $src3$$constant & 0x3f);
12527 %}
12528
12529 ins_pipe(ialu_reg_reg_shift);
12530 %}
12531
12532 // This pattern is automatically generated from aarch64_ad.m4
12533 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12534 instruct XorI_reg_LShift_reg(iRegINoSp dst,
12535 iRegIorL2I src1, iRegIorL2I src2,
12536 immI src3) %{
12537 match(Set dst (XorI src1 (LShiftI src2 src3)));
12538
12539 ins_cost(1.9 * INSN_COST);
12540 format %{ "eorw $dst, $src1, $src2, LSL $src3" %}
12541
12542 ins_encode %{
12543 __ eorw(as_Register($dst$$reg),
12544 as_Register($src1$$reg),
12545 as_Register($src2$$reg),
12546 Assembler::LSL,
12547 $src3$$constant & 0x1f);
12548 %}
12549
12550 ins_pipe(ialu_reg_reg_shift);
12551 %}
12552
12553 // This pattern is automatically generated from aarch64_ad.m4
12554 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12555 instruct XorL_reg_LShift_reg(iRegLNoSp dst,
12556 iRegL src1, iRegL src2,
12557 immI src3) %{
12558 match(Set dst (XorL src1 (LShiftL src2 src3)));
12559
12560 ins_cost(1.9 * INSN_COST);
12561 format %{ "eor $dst, $src1, $src2, LSL $src3" %}
12562
12563 ins_encode %{
12564 __ eor(as_Register($dst$$reg),
12565 as_Register($src1$$reg),
12566 as_Register($src2$$reg),
12567 Assembler::LSL,
12568 $src3$$constant & 0x3f);
12569 %}
12570
12571 ins_pipe(ialu_reg_reg_shift);
12572 %}
12573
12574 // This pattern is automatically generated from aarch64_ad.m4
12575 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12576 instruct XorI_reg_RotateRight_reg(iRegINoSp dst,
12577 iRegIorL2I src1, iRegIorL2I src2,
12578 immI src3) %{
12579 match(Set dst (XorI src1 (RotateRight src2 src3)));
12580
12581 ins_cost(1.9 * INSN_COST);
12582 format %{ "eorw $dst, $src1, $src2, ROR $src3" %}
12583
12584 ins_encode %{
12585 __ eorw(as_Register($dst$$reg),
12586 as_Register($src1$$reg),
12587 as_Register($src2$$reg),
12588 Assembler::ROR,
12589 $src3$$constant & 0x1f);
12590 %}
12591
12592 ins_pipe(ialu_reg_reg_shift);
12593 %}
12594
12595 // This pattern is automatically generated from aarch64_ad.m4
12596 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12597 instruct XorL_reg_RotateRight_reg(iRegLNoSp dst,
12598 iRegL src1, iRegL src2,
12599 immI src3) %{
12600 match(Set dst (XorL src1 (RotateRight src2 src3)));
12601
12602 ins_cost(1.9 * INSN_COST);
12603 format %{ "eor $dst, $src1, $src2, ROR $src3" %}
12604
12605 ins_encode %{
12606 __ eor(as_Register($dst$$reg),
12607 as_Register($src1$$reg),
12608 as_Register($src2$$reg),
12609 Assembler::ROR,
12610 $src3$$constant & 0x3f);
12611 %}
12612
12613 ins_pipe(ialu_reg_reg_shift);
12614 %}
12615
12616 // This pattern is automatically generated from aarch64_ad.m4
12617 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12618 instruct OrI_reg_URShift_reg(iRegINoSp dst,
12619 iRegIorL2I src1, iRegIorL2I src2,
12620 immI src3) %{
12621 match(Set dst (OrI src1 (URShiftI src2 src3)));
12622
12623 ins_cost(1.9 * INSN_COST);
12624 format %{ "orrw $dst, $src1, $src2, LSR $src3" %}
12625
12626 ins_encode %{
12627 __ orrw(as_Register($dst$$reg),
12628 as_Register($src1$$reg),
12629 as_Register($src2$$reg),
12630 Assembler::LSR,
12631 $src3$$constant & 0x1f);
12632 %}
12633
12634 ins_pipe(ialu_reg_reg_shift);
12635 %}
12636
12637 // This pattern is automatically generated from aarch64_ad.m4
12638 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12639 instruct OrL_reg_URShift_reg(iRegLNoSp dst,
12640 iRegL src1, iRegL src2,
12641 immI src3) %{
12642 match(Set dst (OrL src1 (URShiftL src2 src3)));
12643
12644 ins_cost(1.9 * INSN_COST);
12645 format %{ "orr $dst, $src1, $src2, LSR $src3" %}
12646
12647 ins_encode %{
12648 __ orr(as_Register($dst$$reg),
12649 as_Register($src1$$reg),
12650 as_Register($src2$$reg),
12651 Assembler::LSR,
12652 $src3$$constant & 0x3f);
12653 %}
12654
12655 ins_pipe(ialu_reg_reg_shift);
12656 %}
12657
12658 // This pattern is automatically generated from aarch64_ad.m4
12659 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12660 instruct OrI_reg_RShift_reg(iRegINoSp dst,
12661 iRegIorL2I src1, iRegIorL2I src2,
12662 immI src3) %{
12663 match(Set dst (OrI src1 (RShiftI src2 src3)));
12664
12665 ins_cost(1.9 * INSN_COST);
12666 format %{ "orrw $dst, $src1, $src2, ASR $src3" %}
12667
12668 ins_encode %{
12669 __ orrw(as_Register($dst$$reg),
12670 as_Register($src1$$reg),
12671 as_Register($src2$$reg),
12672 Assembler::ASR,
12673 $src3$$constant & 0x1f);
12674 %}
12675
12676 ins_pipe(ialu_reg_reg_shift);
12677 %}
12678
12679 // This pattern is automatically generated from aarch64_ad.m4
12680 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12681 instruct OrL_reg_RShift_reg(iRegLNoSp dst,
12682 iRegL src1, iRegL src2,
12683 immI src3) %{
12684 match(Set dst (OrL src1 (RShiftL src2 src3)));
12685
12686 ins_cost(1.9 * INSN_COST);
12687 format %{ "orr $dst, $src1, $src2, ASR $src3" %}
12688
12689 ins_encode %{
12690 __ orr(as_Register($dst$$reg),
12691 as_Register($src1$$reg),
12692 as_Register($src2$$reg),
12693 Assembler::ASR,
12694 $src3$$constant & 0x3f);
12695 %}
12696
12697 ins_pipe(ialu_reg_reg_shift);
12698 %}
12699
12700 // This pattern is automatically generated from aarch64_ad.m4
12701 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12702 instruct OrI_reg_LShift_reg(iRegINoSp dst,
12703 iRegIorL2I src1, iRegIorL2I src2,
12704 immI src3) %{
12705 match(Set dst (OrI src1 (LShiftI src2 src3)));
12706
12707 ins_cost(1.9 * INSN_COST);
12708 format %{ "orrw $dst, $src1, $src2, LSL $src3" %}
12709
12710 ins_encode %{
12711 __ orrw(as_Register($dst$$reg),
12712 as_Register($src1$$reg),
12713 as_Register($src2$$reg),
12714 Assembler::LSL,
12715 $src3$$constant & 0x1f);
12716 %}
12717
12718 ins_pipe(ialu_reg_reg_shift);
12719 %}
12720
12721 // This pattern is automatically generated from aarch64_ad.m4
12722 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12723 instruct OrL_reg_LShift_reg(iRegLNoSp dst,
12724 iRegL src1, iRegL src2,
12725 immI src3) %{
12726 match(Set dst (OrL src1 (LShiftL src2 src3)));
12727
12728 ins_cost(1.9 * INSN_COST);
12729 format %{ "orr $dst, $src1, $src2, LSL $src3" %}
12730
12731 ins_encode %{
12732 __ orr(as_Register($dst$$reg),
12733 as_Register($src1$$reg),
12734 as_Register($src2$$reg),
12735 Assembler::LSL,
12736 $src3$$constant & 0x3f);
12737 %}
12738
12739 ins_pipe(ialu_reg_reg_shift);
12740 %}
12741
12742 // This pattern is automatically generated from aarch64_ad.m4
12743 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12744 instruct OrI_reg_RotateRight_reg(iRegINoSp dst,
12745 iRegIorL2I src1, iRegIorL2I src2,
12746 immI src3) %{
12747 match(Set dst (OrI src1 (RotateRight src2 src3)));
12748
12749 ins_cost(1.9 * INSN_COST);
12750 format %{ "orrw $dst, $src1, $src2, ROR $src3" %}
12751
12752 ins_encode %{
12753 __ orrw(as_Register($dst$$reg),
12754 as_Register($src1$$reg),
12755 as_Register($src2$$reg),
12756 Assembler::ROR,
12757 $src3$$constant & 0x1f);
12758 %}
12759
12760 ins_pipe(ialu_reg_reg_shift);
12761 %}
12762
12763 // This pattern is automatically generated from aarch64_ad.m4
12764 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12765 instruct OrL_reg_RotateRight_reg(iRegLNoSp dst,
12766 iRegL src1, iRegL src2,
12767 immI src3) %{
12768 match(Set dst (OrL src1 (RotateRight src2 src3)));
12769
12770 ins_cost(1.9 * INSN_COST);
12771 format %{ "orr $dst, $src1, $src2, ROR $src3" %}
12772
12773 ins_encode %{
12774 __ orr(as_Register($dst$$reg),
12775 as_Register($src1$$reg),
12776 as_Register($src2$$reg),
12777 Assembler::ROR,
12778 $src3$$constant & 0x3f);
12779 %}
12780
12781 ins_pipe(ialu_reg_reg_shift);
12782 %}
12783
12784 // This pattern is automatically generated from aarch64_ad.m4
12785 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12786 instruct AddI_reg_URShift_reg(iRegINoSp dst,
12787 iRegIorL2I src1, iRegIorL2I src2,
12788 immI src3) %{
12789 match(Set dst (AddI src1 (URShiftI src2 src3)));
12790
12791 ins_cost(1.9 * INSN_COST);
12792 format %{ "addw $dst, $src1, $src2, LSR $src3" %}
12793
12794 ins_encode %{
12795 __ addw(as_Register($dst$$reg),
12796 as_Register($src1$$reg),
12797 as_Register($src2$$reg),
12798 Assembler::LSR,
12799 $src3$$constant & 0x1f);
12800 %}
12801
12802 ins_pipe(ialu_reg_reg_shift);
12803 %}
12804
12805 // This pattern is automatically generated from aarch64_ad.m4
12806 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12807 instruct AddL_reg_URShift_reg(iRegLNoSp dst,
12808 iRegL src1, iRegL src2,
12809 immI src3) %{
12810 match(Set dst (AddL src1 (URShiftL src2 src3)));
12811
12812 ins_cost(1.9 * INSN_COST);
12813 format %{ "add $dst, $src1, $src2, LSR $src3" %}
12814
12815 ins_encode %{
12816 __ add(as_Register($dst$$reg),
12817 as_Register($src1$$reg),
12818 as_Register($src2$$reg),
12819 Assembler::LSR,
12820 $src3$$constant & 0x3f);
12821 %}
12822
12823 ins_pipe(ialu_reg_reg_shift);
12824 %}
12825
12826 // This pattern is automatically generated from aarch64_ad.m4
12827 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12828 instruct AddI_reg_RShift_reg(iRegINoSp dst,
12829 iRegIorL2I src1, iRegIorL2I src2,
12830 immI src3) %{
12831 match(Set dst (AddI src1 (RShiftI src2 src3)));
12832
12833 ins_cost(1.9 * INSN_COST);
12834 format %{ "addw $dst, $src1, $src2, ASR $src3" %}
12835
12836 ins_encode %{
12837 __ addw(as_Register($dst$$reg),
12838 as_Register($src1$$reg),
12839 as_Register($src2$$reg),
12840 Assembler::ASR,
12841 $src3$$constant & 0x1f);
12842 %}
12843
12844 ins_pipe(ialu_reg_reg_shift);
12845 %}
12846
12847 // This pattern is automatically generated from aarch64_ad.m4
12848 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12849 instruct AddL_reg_RShift_reg(iRegLNoSp dst,
12850 iRegL src1, iRegL src2,
12851 immI src3) %{
12852 match(Set dst (AddL src1 (RShiftL src2 src3)));
12853
12854 ins_cost(1.9 * INSN_COST);
12855 format %{ "add $dst, $src1, $src2, ASR $src3" %}
12856
12857 ins_encode %{
12858 __ add(as_Register($dst$$reg),
12859 as_Register($src1$$reg),
12860 as_Register($src2$$reg),
12861 Assembler::ASR,
12862 $src3$$constant & 0x3f);
12863 %}
12864
12865 ins_pipe(ialu_reg_reg_shift);
12866 %}
12867
12868 // This pattern is automatically generated from aarch64_ad.m4
12869 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12870 instruct AddI_reg_LShift_reg(iRegINoSp dst,
12871 iRegIorL2I src1, iRegIorL2I src2,
12872 immI src3) %{
12873 match(Set dst (AddI src1 (LShiftI src2 src3)));
12874
12875 ins_cost(1.9 * INSN_COST);
12876 format %{ "addw $dst, $src1, $src2, LSL $src3" %}
12877
12878 ins_encode %{
12879 __ addw(as_Register($dst$$reg),
12880 as_Register($src1$$reg),
12881 as_Register($src2$$reg),
12882 Assembler::LSL,
12883 $src3$$constant & 0x1f);
12884 %}
12885
12886 ins_pipe(ialu_reg_reg_shift);
12887 %}
12888
12889 // This pattern is automatically generated from aarch64_ad.m4
12890 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12891 instruct AddL_reg_LShift_reg(iRegLNoSp dst,
12892 iRegL src1, iRegL src2,
12893 immI src3) %{
12894 match(Set dst (AddL src1 (LShiftL src2 src3)));
12895
12896 ins_cost(1.9 * INSN_COST);
12897 format %{ "add $dst, $src1, $src2, LSL $src3" %}
12898
12899 ins_encode %{
12900 __ add(as_Register($dst$$reg),
12901 as_Register($src1$$reg),
12902 as_Register($src2$$reg),
12903 Assembler::LSL,
12904 $src3$$constant & 0x3f);
12905 %}
12906
12907 ins_pipe(ialu_reg_reg_shift);
12908 %}
12909
12910 // This pattern is automatically generated from aarch64_ad.m4
12911 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12912 instruct SubI_reg_URShift_reg(iRegINoSp dst,
12913 iRegIorL2I src1, iRegIorL2I src2,
12914 immI src3) %{
12915 match(Set dst (SubI src1 (URShiftI src2 src3)));
12916
12917 ins_cost(1.9 * INSN_COST);
12918 format %{ "subw $dst, $src1, $src2, LSR $src3" %}
12919
12920 ins_encode %{
12921 __ subw(as_Register($dst$$reg),
12922 as_Register($src1$$reg),
12923 as_Register($src2$$reg),
12924 Assembler::LSR,
12925 $src3$$constant & 0x1f);
12926 %}
12927
12928 ins_pipe(ialu_reg_reg_shift);
12929 %}
12930
12931 // This pattern is automatically generated from aarch64_ad.m4
12932 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12933 instruct SubL_reg_URShift_reg(iRegLNoSp dst,
12934 iRegL src1, iRegL src2,
12935 immI src3) %{
12936 match(Set dst (SubL src1 (URShiftL src2 src3)));
12937
12938 ins_cost(1.9 * INSN_COST);
12939 format %{ "sub $dst, $src1, $src2, LSR $src3" %}
12940
12941 ins_encode %{
12942 __ sub(as_Register($dst$$reg),
12943 as_Register($src1$$reg),
12944 as_Register($src2$$reg),
12945 Assembler::LSR,
12946 $src3$$constant & 0x3f);
12947 %}
12948
12949 ins_pipe(ialu_reg_reg_shift);
12950 %}
12951
12952 // This pattern is automatically generated from aarch64_ad.m4
12953 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12954 instruct SubI_reg_RShift_reg(iRegINoSp dst,
12955 iRegIorL2I src1, iRegIorL2I src2,
12956 immI src3) %{
12957 match(Set dst (SubI src1 (RShiftI src2 src3)));
12958
12959 ins_cost(1.9 * INSN_COST);
12960 format %{ "subw $dst, $src1, $src2, ASR $src3" %}
12961
12962 ins_encode %{
12963 __ subw(as_Register($dst$$reg),
12964 as_Register($src1$$reg),
12965 as_Register($src2$$reg),
12966 Assembler::ASR,
12967 $src3$$constant & 0x1f);
12968 %}
12969
12970 ins_pipe(ialu_reg_reg_shift);
12971 %}
12972
12973 // This pattern is automatically generated from aarch64_ad.m4
12974 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12975 instruct SubL_reg_RShift_reg(iRegLNoSp dst,
12976 iRegL src1, iRegL src2,
12977 immI src3) %{
12978 match(Set dst (SubL src1 (RShiftL src2 src3)));
12979
12980 ins_cost(1.9 * INSN_COST);
12981 format %{ "sub $dst, $src1, $src2, ASR $src3" %}
12982
12983 ins_encode %{
12984 __ sub(as_Register($dst$$reg),
12985 as_Register($src1$$reg),
12986 as_Register($src2$$reg),
12987 Assembler::ASR,
12988 $src3$$constant & 0x3f);
12989 %}
12990
12991 ins_pipe(ialu_reg_reg_shift);
12992 %}
12993
12994 // This pattern is automatically generated from aarch64_ad.m4
12995 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12996 instruct SubI_reg_LShift_reg(iRegINoSp dst,
12997 iRegIorL2I src1, iRegIorL2I src2,
12998 immI src3) %{
12999 match(Set dst (SubI src1 (LShiftI src2 src3)));
13000
13001 ins_cost(1.9 * INSN_COST);
13002 format %{ "subw $dst, $src1, $src2, LSL $src3" %}
13003
13004 ins_encode %{
13005 __ subw(as_Register($dst$$reg),
13006 as_Register($src1$$reg),
13007 as_Register($src2$$reg),
13008 Assembler::LSL,
13009 $src3$$constant & 0x1f);
13010 %}
13011
13012 ins_pipe(ialu_reg_reg_shift);
13013 %}
13014
13015 // This pattern is automatically generated from aarch64_ad.m4
13016 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13017 instruct SubL_reg_LShift_reg(iRegLNoSp dst,
13018 iRegL src1, iRegL src2,
13019 immI src3) %{
13020 match(Set dst (SubL src1 (LShiftL src2 src3)));
13021
13022 ins_cost(1.9 * INSN_COST);
13023 format %{ "sub $dst, $src1, $src2, LSL $src3" %}
13024
13025 ins_encode %{
13026 __ sub(as_Register($dst$$reg),
13027 as_Register($src1$$reg),
13028 as_Register($src2$$reg),
13029 Assembler::LSL,
13030 $src3$$constant & 0x3f);
13031 %}
13032
13033 ins_pipe(ialu_reg_reg_shift);
13034 %}
13035
13036 // This pattern is automatically generated from aarch64_ad.m4
13037 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13038
13039 // Shift Left followed by Shift Right.
13040 // This idiom is used by the compiler for the i2b bytecode etc.
13041 instruct sbfmL(iRegLNoSp dst, iRegL src, immI lshift_count, immI rshift_count)
13042 %{
13043 match(Set dst (RShiftL (LShiftL src lshift_count) rshift_count));
13044 ins_cost(INSN_COST * 2);
13045 format %{ "sbfm $dst, $src, $rshift_count - $lshift_count, #63 - $lshift_count" %}
13046 ins_encode %{
13047 int lshift = $lshift_count$$constant & 63;
13048 int rshift = $rshift_count$$constant & 63;
13049 int s = 63 - lshift;
13050 int r = (rshift - lshift) & 63;
13051 __ sbfm(as_Register($dst$$reg),
13052 as_Register($src$$reg),
13053 r, s);
13054 %}
13055
13056 ins_pipe(ialu_reg_shift);
13057 %}
13058
13059 // This pattern is automatically generated from aarch64_ad.m4
13060 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13061
13062 // Shift Left followed by Shift Right.
13063 // This idiom is used by the compiler for the i2b bytecode etc.
13064 instruct sbfmwI(iRegINoSp dst, iRegIorL2I src, immI lshift_count, immI rshift_count)
13065 %{
13066 match(Set dst (RShiftI (LShiftI src lshift_count) rshift_count));
13067 ins_cost(INSN_COST * 2);
13068 format %{ "sbfmw $dst, $src, $rshift_count - $lshift_count, #31 - $lshift_count" %}
13069 ins_encode %{
13070 int lshift = $lshift_count$$constant & 31;
13071 int rshift = $rshift_count$$constant & 31;
13072 int s = 31 - lshift;
13073 int r = (rshift - lshift) & 31;
13074 __ sbfmw(as_Register($dst$$reg),
13075 as_Register($src$$reg),
13076 r, s);
13077 %}
13078
13079 ins_pipe(ialu_reg_shift);
13080 %}
13081
13082 // This pattern is automatically generated from aarch64_ad.m4
13083 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13084
13085 // Shift Left followed by Shift Right.
13086 // This idiom is used by the compiler for the i2b bytecode etc.
13087 instruct ubfmL(iRegLNoSp dst, iRegL src, immI lshift_count, immI rshift_count)
13088 %{
13089 match(Set dst (URShiftL (LShiftL src lshift_count) rshift_count));
13090 ins_cost(INSN_COST * 2);
13091 format %{ "ubfm $dst, $src, $rshift_count - $lshift_count, #63 - $lshift_count" %}
13092 ins_encode %{
13093 int lshift = $lshift_count$$constant & 63;
13094 int rshift = $rshift_count$$constant & 63;
13095 int s = 63 - lshift;
13096 int r = (rshift - lshift) & 63;
13097 __ ubfm(as_Register($dst$$reg),
13098 as_Register($src$$reg),
13099 r, s);
13100 %}
13101
13102 ins_pipe(ialu_reg_shift);
13103 %}
13104
13105 // This pattern is automatically generated from aarch64_ad.m4
13106 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13107
13108 // Shift Left followed by Shift Right.
13109 // This idiom is used by the compiler for the i2b bytecode etc.
13110 instruct ubfmwI(iRegINoSp dst, iRegIorL2I src, immI lshift_count, immI rshift_count)
13111 %{
13112 match(Set dst (URShiftI (LShiftI src lshift_count) rshift_count));
13113 ins_cost(INSN_COST * 2);
13114 format %{ "ubfmw $dst, $src, $rshift_count - $lshift_count, #31 - $lshift_count" %}
13115 ins_encode %{
13116 int lshift = $lshift_count$$constant & 31;
13117 int rshift = $rshift_count$$constant & 31;
13118 int s = 31 - lshift;
13119 int r = (rshift - lshift) & 31;
13120 __ ubfmw(as_Register($dst$$reg),
13121 as_Register($src$$reg),
13122 r, s);
13123 %}
13124
13125 ins_pipe(ialu_reg_shift);
13126 %}
13127
13128 // Bitfield extract with shift & mask
13129
13130 // This pattern is automatically generated from aarch64_ad.m4
13131 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13132 instruct ubfxwI(iRegINoSp dst, iRegIorL2I src, immI rshift, immI_bitmask mask)
13133 %{
13134 match(Set dst (AndI (URShiftI src rshift) mask));
13135 // Make sure we are not going to exceed what ubfxw can do.
13136 predicate((exact_log2(n->in(2)->get_int() + 1) + (n->in(1)->in(2)->get_int() & 31)) <= (31 + 1));
13137
13138 ins_cost(INSN_COST);
13139 format %{ "ubfxw $dst, $src, $rshift, $mask" %}
13140 ins_encode %{
13141 int rshift = $rshift$$constant & 31;
13142 intptr_t mask = $mask$$constant;
13143 int width = exact_log2(mask+1);
13144 __ ubfxw(as_Register($dst$$reg),
13145 as_Register($src$$reg), rshift, width);
13146 %}
13147 ins_pipe(ialu_reg_shift);
13148 %}
13149
13150 // This pattern is automatically generated from aarch64_ad.m4
13151 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13152 instruct ubfxL(iRegLNoSp dst, iRegL src, immI rshift, immL_bitmask mask)
13153 %{
13154 match(Set dst (AndL (URShiftL src rshift) mask));
13155 // Make sure we are not going to exceed what ubfx can do.
13156 predicate((exact_log2_long(n->in(2)->get_long() + 1) + (n->in(1)->in(2)->get_int() & 63)) <= (63 + 1));
13157
13158 ins_cost(INSN_COST);
13159 format %{ "ubfx $dst, $src, $rshift, $mask" %}
13160 ins_encode %{
13161 int rshift = $rshift$$constant & 63;
13162 intptr_t mask = $mask$$constant;
13163 int width = exact_log2_long(mask+1);
13164 __ ubfx(as_Register($dst$$reg),
13165 as_Register($src$$reg), rshift, width);
13166 %}
13167 ins_pipe(ialu_reg_shift);
13168 %}
13169
13170
13171 // This pattern is automatically generated from aarch64_ad.m4
13172 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13173
13174 // We can use ubfx when extending an And with a mask when we know mask
13175 // is positive. We know that because immI_bitmask guarantees it.
13176 instruct ubfxIConvI2L(iRegLNoSp dst, iRegIorL2I src, immI rshift, immI_bitmask mask)
13177 %{
13178 match(Set dst (ConvI2L (AndI (URShiftI src rshift) mask)));
13179 // Make sure we are not going to exceed what ubfxw can do.
13180 predicate((exact_log2(n->in(1)->in(2)->get_int() + 1) + (n->in(1)->in(1)->in(2)->get_int() & 31)) <= (31 + 1));
13181
13182 ins_cost(INSN_COST * 2);
13183 format %{ "ubfx $dst, $src, $rshift, $mask" %}
13184 ins_encode %{
13185 int rshift = $rshift$$constant & 31;
13186 intptr_t mask = $mask$$constant;
13187 int width = exact_log2(mask+1);
13188 __ ubfx(as_Register($dst$$reg),
13189 as_Register($src$$reg), rshift, width);
13190 %}
13191 ins_pipe(ialu_reg_shift);
13192 %}
13193
13194
13195 // This pattern is automatically generated from aarch64_ad.m4
13196 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13197
13198 // We can use ubfiz when masking by a positive number and then left shifting the result.
13199 // We know that the mask is positive because immI_bitmask guarantees it.
13200 instruct ubfizwI(iRegINoSp dst, iRegIorL2I src, immI lshift, immI_bitmask mask)
13201 %{
13202 match(Set dst (LShiftI (AndI src mask) lshift));
13203 predicate((exact_log2(n->in(1)->in(2)->get_int() + 1) + (n->in(2)->get_int() & 31)) <= (31 + 1));
13204
13205 ins_cost(INSN_COST);
13206 format %{ "ubfizw $dst, $src, $lshift, $mask" %}
13207 ins_encode %{
13208 int lshift = $lshift$$constant & 31;
13209 intptr_t mask = $mask$$constant;
13210 int width = exact_log2(mask+1);
13211 __ ubfizw(as_Register($dst$$reg),
13212 as_Register($src$$reg), lshift, width);
13213 %}
13214 ins_pipe(ialu_reg_shift);
13215 %}
13216
13217 // This pattern is automatically generated from aarch64_ad.m4
13218 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13219
13220 // We can use ubfiz when masking by a positive number and then left shifting the result.
13221 // We know that the mask is positive because immL_bitmask guarantees it.
13222 instruct ubfizL(iRegLNoSp dst, iRegL src, immI lshift, immL_bitmask mask)
13223 %{
13224 match(Set dst (LShiftL (AndL src mask) lshift));
13225 predicate((exact_log2_long(n->in(1)->in(2)->get_long() + 1) + (n->in(2)->get_int() & 63)) <= (63 + 1));
13226
13227 ins_cost(INSN_COST);
13228 format %{ "ubfiz $dst, $src, $lshift, $mask" %}
13229 ins_encode %{
13230 int lshift = $lshift$$constant & 63;
13231 intptr_t mask = $mask$$constant;
13232 int width = exact_log2_long(mask+1);
13233 __ ubfiz(as_Register($dst$$reg),
13234 as_Register($src$$reg), lshift, width);
13235 %}
13236 ins_pipe(ialu_reg_shift);
13237 %}
13238
13239 // This pattern is automatically generated from aarch64_ad.m4
13240 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13241
13242 // We can use ubfiz when masking by a positive number and then left shifting the result.
13243 // We know that the mask is positive because immI_bitmask guarantees it.
13244 instruct ubfizwIConvI2L(iRegLNoSp dst, iRegIorL2I src, immI lshift, immI_bitmask mask)
13245 %{
13246 match(Set dst (ConvI2L (LShiftI (AndI src mask) lshift)));
13247 predicate((exact_log2(n->in(1)->in(1)->in(2)->get_int() + 1) + (n->in(1)->in(2)->get_int() & 31)) <= 31);
13248
13249 ins_cost(INSN_COST);
13250 format %{ "ubfizw $dst, $src, $lshift, $mask" %}
13251 ins_encode %{
13252 int lshift = $lshift$$constant & 31;
13253 intptr_t mask = $mask$$constant;
13254 int width = exact_log2(mask+1);
13255 __ ubfizw(as_Register($dst$$reg),
13256 as_Register($src$$reg), lshift, width);
13257 %}
13258 ins_pipe(ialu_reg_shift);
13259 %}
13260
13261 // This pattern is automatically generated from aarch64_ad.m4
13262 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13263
13264 // We can use ubfiz when masking by a positive number and then left shifting the result.
13265 // We know that the mask is positive because immL_bitmask guarantees it.
13266 instruct ubfizLConvL2I(iRegINoSp dst, iRegL src, immI lshift, immL_positive_bitmaskI mask)
13267 %{
13268 match(Set dst (ConvL2I (LShiftL (AndL src mask) lshift)));
13269 predicate((exact_log2_long(n->in(1)->in(1)->in(2)->get_long() + 1) + (n->in(1)->in(2)->get_int() & 63)) <= 31);
13270
13271 ins_cost(INSN_COST);
13272 format %{ "ubfiz $dst, $src, $lshift, $mask" %}
13273 ins_encode %{
13274 int lshift = $lshift$$constant & 63;
13275 intptr_t mask = $mask$$constant;
13276 int width = exact_log2_long(mask+1);
13277 __ ubfiz(as_Register($dst$$reg),
13278 as_Register($src$$reg), lshift, width);
13279 %}
13280 ins_pipe(ialu_reg_shift);
13281 %}
13282
13283
13284 // This pattern is automatically generated from aarch64_ad.m4
13285 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13286
13287 // If there is a convert I to L block between and AndI and a LShiftL, we can also match ubfiz
13288 instruct ubfizIConvI2L(iRegLNoSp dst, iRegIorL2I src, immI lshift, immI_bitmask mask)
13289 %{
13290 match(Set dst (LShiftL (ConvI2L (AndI src mask)) lshift));
13291 predicate((exact_log2(n->in(1)->in(1)->in(2)->get_int() + 1) + (n->in(2)->get_int() & 63)) <= (63 + 1));
13292
13293 ins_cost(INSN_COST);
13294 format %{ "ubfiz $dst, $src, $lshift, $mask" %}
13295 ins_encode %{
13296 int lshift = $lshift$$constant & 63;
13297 intptr_t mask = $mask$$constant;
13298 int width = exact_log2(mask+1);
13299 __ ubfiz(as_Register($dst$$reg),
13300 as_Register($src$$reg), lshift, width);
13301 %}
13302 ins_pipe(ialu_reg_shift);
13303 %}
13304
13305 // This pattern is automatically generated from aarch64_ad.m4
13306 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13307
13308 // If there is a convert L to I block between and AndL and a LShiftI, we can also match ubfiz
13309 instruct ubfizLConvL2Ix(iRegINoSp dst, iRegL src, immI lshift, immL_positive_bitmaskI mask)
13310 %{
13311 match(Set dst (LShiftI (ConvL2I (AndL src mask)) lshift));
13312 predicate((exact_log2_long(n->in(1)->in(1)->in(2)->get_long() + 1) + (n->in(2)->get_int() & 31)) <= 31);
13313
13314 ins_cost(INSN_COST);
13315 format %{ "ubfiz $dst, $src, $lshift, $mask" %}
13316 ins_encode %{
13317 int lshift = $lshift$$constant & 31;
13318 intptr_t mask = $mask$$constant;
13319 int width = exact_log2(mask+1);
13320 __ ubfiz(as_Register($dst$$reg),
13321 as_Register($src$$reg), lshift, width);
13322 %}
13323 ins_pipe(ialu_reg_shift);
13324 %}
13325
13326 // This pattern is automatically generated from aarch64_ad.m4
13327 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13328
13329 // Can skip int2long conversions after AND with small bitmask
13330 instruct ubfizIConvI2LAndI(iRegLNoSp dst, iRegI src, immI_bitmask msk)
13331 %{
13332 match(Set dst (ConvI2L (AndI src msk)));
13333 ins_cost(INSN_COST);
13334 format %{ "ubfiz $dst, $src, 0, exact_log2($msk + 1) " %}
13335 ins_encode %{
13336 __ ubfiz(as_Register($dst$$reg), as_Register($src$$reg), 0, exact_log2($msk$$constant + 1));
13337 %}
13338 ins_pipe(ialu_reg_shift);
13339 %}
13340
13341
13342 // Rotations
13343 // This pattern is automatically generated from aarch64_ad.m4
13344 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13345 instruct extrOrL(iRegLNoSp dst, iRegL src1, iRegL src2, immI lshift, immI rshift, rFlagsReg cr)
13346 %{
13347 match(Set dst (OrL (LShiftL src1 lshift) (URShiftL src2 rshift)));
13348 predicate(0 == (((n->in(1)->in(2)->get_int() & 63) + (n->in(2)->in(2)->get_int() & 63)) & 63));
13349
13350 ins_cost(INSN_COST);
13351 format %{ "extr $dst, $src1, $src2, #$rshift" %}
13352
13353 ins_encode %{
13354 __ extr(as_Register($dst$$reg), as_Register($src1$$reg), as_Register($src2$$reg),
13355 $rshift$$constant & 63);
13356 %}
13357 ins_pipe(ialu_reg_reg_extr);
13358 %}
13359
13360
13361 // This pattern is automatically generated from aarch64_ad.m4
13362 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13363 instruct extrOrI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI lshift, immI rshift, rFlagsReg cr)
13364 %{
13365 match(Set dst (OrI (LShiftI src1 lshift) (URShiftI src2 rshift)));
13366 predicate(0 == (((n->in(1)->in(2)->get_int() & 31) + (n->in(2)->in(2)->get_int() & 31)) & 31));
13367
13368 ins_cost(INSN_COST);
13369 format %{ "extr $dst, $src1, $src2, #$rshift" %}
13370
13371 ins_encode %{
13372 __ extrw(as_Register($dst$$reg), as_Register($src1$$reg), as_Register($src2$$reg),
13373 $rshift$$constant & 31);
13374 %}
13375 ins_pipe(ialu_reg_reg_extr);
13376 %}
13377
13378
13379 // This pattern is automatically generated from aarch64_ad.m4
13380 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13381 instruct extrAddL(iRegLNoSp dst, iRegL src1, iRegL src2, immI lshift, immI rshift, rFlagsReg cr)
13382 %{
13383 match(Set dst (AddL (LShiftL src1 lshift) (URShiftL src2 rshift)));
13384 predicate(0 == (((n->in(1)->in(2)->get_int() & 63) + (n->in(2)->in(2)->get_int() & 63)) & 63));
13385
13386 ins_cost(INSN_COST);
13387 format %{ "extr $dst, $src1, $src2, #$rshift" %}
13388
13389 ins_encode %{
13390 __ extr(as_Register($dst$$reg), as_Register($src1$$reg), as_Register($src2$$reg),
13391 $rshift$$constant & 63);
13392 %}
13393 ins_pipe(ialu_reg_reg_extr);
13394 %}
13395
13396
13397 // This pattern is automatically generated from aarch64_ad.m4
13398 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13399 instruct extrAddI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI lshift, immI rshift, rFlagsReg cr)
13400 %{
13401 match(Set dst (AddI (LShiftI src1 lshift) (URShiftI src2 rshift)));
13402 predicate(0 == (((n->in(1)->in(2)->get_int() & 31) + (n->in(2)->in(2)->get_int() & 31)) & 31));
13403
13404 ins_cost(INSN_COST);
13405 format %{ "extr $dst, $src1, $src2, #$rshift" %}
13406
13407 ins_encode %{
13408 __ extrw(as_Register($dst$$reg), as_Register($src1$$reg), as_Register($src2$$reg),
13409 $rshift$$constant & 31);
13410 %}
13411 ins_pipe(ialu_reg_reg_extr);
13412 %}
13413
13414
13415 // This pattern is automatically generated from aarch64_ad.m4
13416 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13417 instruct rorI_imm(iRegINoSp dst, iRegI src, immI shift)
13418 %{
13419 match(Set dst (RotateRight src shift));
13420
13421 ins_cost(INSN_COST);
13422 format %{ "ror $dst, $src, $shift" %}
13423
13424 ins_encode %{
13425 __ extrw(as_Register($dst$$reg), as_Register($src$$reg), as_Register($src$$reg),
13426 $shift$$constant & 0x1f);
13427 %}
13428 ins_pipe(ialu_reg_reg_vshift);
13429 %}
13430
13431 // This pattern is automatically generated from aarch64_ad.m4
13432 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13433 instruct rorL_imm(iRegLNoSp dst, iRegL src, immI shift)
13434 %{
13435 match(Set dst (RotateRight src shift));
13436
13437 ins_cost(INSN_COST);
13438 format %{ "ror $dst, $src, $shift" %}
13439
13440 ins_encode %{
13441 __ extr(as_Register($dst$$reg), as_Register($src$$reg), as_Register($src$$reg),
13442 $shift$$constant & 0x3f);
13443 %}
13444 ins_pipe(ialu_reg_reg_vshift);
13445 %}
13446
13447 // This pattern is automatically generated from aarch64_ad.m4
13448 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13449 instruct rorI_reg(iRegINoSp dst, iRegI src, iRegI shift)
13450 %{
13451 match(Set dst (RotateRight src shift));
13452
13453 ins_cost(INSN_COST);
13454 format %{ "ror $dst, $src, $shift" %}
13455
13456 ins_encode %{
13457 __ rorvw(as_Register($dst$$reg), as_Register($src$$reg), as_Register($shift$$reg));
13458 %}
13459 ins_pipe(ialu_reg_reg_vshift);
13460 %}
13461
13462 // This pattern is automatically generated from aarch64_ad.m4
13463 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13464 instruct rorL_reg(iRegLNoSp dst, iRegL src, iRegI shift)
13465 %{
13466 match(Set dst (RotateRight src shift));
13467
13468 ins_cost(INSN_COST);
13469 format %{ "ror $dst, $src, $shift" %}
13470
13471 ins_encode %{
13472 __ rorv(as_Register($dst$$reg), as_Register($src$$reg), as_Register($shift$$reg));
13473 %}
13474 ins_pipe(ialu_reg_reg_vshift);
13475 %}
13476
13477 // This pattern is automatically generated from aarch64_ad.m4
13478 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13479 instruct rolI_reg(iRegINoSp dst, iRegI src, iRegI shift)
13480 %{
13481 match(Set dst (RotateLeft src shift));
13482
13483 ins_cost(INSN_COST);
13484 format %{ "rol $dst, $src, $shift" %}
13485
13486 ins_encode %{
13487 __ subw(rscratch1, zr, as_Register($shift$$reg));
13488 __ rorvw(as_Register($dst$$reg), as_Register($src$$reg), rscratch1);
13489 %}
13490 ins_pipe(ialu_reg_reg_vshift);
13491 %}
13492
13493 // This pattern is automatically generated from aarch64_ad.m4
13494 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13495 instruct rolL_reg(iRegLNoSp dst, iRegL src, iRegI shift)
13496 %{
13497 match(Set dst (RotateLeft src shift));
13498
13499 ins_cost(INSN_COST);
13500 format %{ "rol $dst, $src, $shift" %}
13501
13502 ins_encode %{
13503 __ subw(rscratch1, zr, as_Register($shift$$reg));
13504 __ rorv(as_Register($dst$$reg), as_Register($src$$reg), rscratch1);
13505 %}
13506 ins_pipe(ialu_reg_reg_vshift);
13507 %}
13508
13509
13510 // Add/subtract (extended)
13511
13512 // This pattern is automatically generated from aarch64_ad.m4
13513 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13514 instruct AddExtI(iRegLNoSp dst, iRegL src1, iRegIorL2I src2, rFlagsReg cr)
13515 %{
13516 match(Set dst (AddL src1 (ConvI2L src2)));
13517 ins_cost(INSN_COST);
13518 format %{ "add $dst, $src1, $src2, sxtw" %}
13519
13520 ins_encode %{
13521 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13522 as_Register($src2$$reg), ext::sxtw);
13523 %}
13524 ins_pipe(ialu_reg_reg);
13525 %}
13526
13527 // This pattern is automatically generated from aarch64_ad.m4
13528 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13529 instruct SubExtI(iRegLNoSp dst, iRegL src1, iRegIorL2I src2, rFlagsReg cr)
13530 %{
13531 match(Set dst (SubL src1 (ConvI2L src2)));
13532 ins_cost(INSN_COST);
13533 format %{ "sub $dst, $src1, $src2, sxtw" %}
13534
13535 ins_encode %{
13536 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13537 as_Register($src2$$reg), ext::sxtw);
13538 %}
13539 ins_pipe(ialu_reg_reg);
13540 %}
13541
13542 // This pattern is automatically generated from aarch64_ad.m4
13543 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13544 instruct AddExtI_sxth(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_16 lshift, immI_16 rshift, rFlagsReg cr)
13545 %{
13546 match(Set dst (AddI src1 (RShiftI (LShiftI src2 lshift) rshift)));
13547 ins_cost(INSN_COST);
13548 format %{ "add $dst, $src1, $src2, sxth" %}
13549
13550 ins_encode %{
13551 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13552 as_Register($src2$$reg), ext::sxth);
13553 %}
13554 ins_pipe(ialu_reg_reg);
13555 %}
13556
13557 // This pattern is automatically generated from aarch64_ad.m4
13558 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13559 instruct AddExtI_sxtb(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_24 lshift, immI_24 rshift, rFlagsReg cr)
13560 %{
13561 match(Set dst (AddI src1 (RShiftI (LShiftI src2 lshift) rshift)));
13562 ins_cost(INSN_COST);
13563 format %{ "add $dst, $src1, $src2, sxtb" %}
13564
13565 ins_encode %{
13566 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13567 as_Register($src2$$reg), ext::sxtb);
13568 %}
13569 ins_pipe(ialu_reg_reg);
13570 %}
13571
13572 // This pattern is automatically generated from aarch64_ad.m4
13573 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13574 instruct AddExtI_uxtb(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_24 lshift, immI_24 rshift, rFlagsReg cr)
13575 %{
13576 match(Set dst (AddI src1 (URShiftI (LShiftI src2 lshift) rshift)));
13577 ins_cost(INSN_COST);
13578 format %{ "add $dst, $src1, $src2, uxtb" %}
13579
13580 ins_encode %{
13581 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13582 as_Register($src2$$reg), ext::uxtb);
13583 %}
13584 ins_pipe(ialu_reg_reg);
13585 %}
13586
13587 // This pattern is automatically generated from aarch64_ad.m4
13588 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13589 instruct AddExtL_sxth(iRegLNoSp dst, iRegL src1, iRegL src2, immI_48 lshift, immI_48 rshift, rFlagsReg cr)
13590 %{
13591 match(Set dst (AddL src1 (RShiftL (LShiftL src2 lshift) rshift)));
13592 ins_cost(INSN_COST);
13593 format %{ "add $dst, $src1, $src2, sxth" %}
13594
13595 ins_encode %{
13596 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13597 as_Register($src2$$reg), ext::sxth);
13598 %}
13599 ins_pipe(ialu_reg_reg);
13600 %}
13601
13602 // This pattern is automatically generated from aarch64_ad.m4
13603 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13604 instruct AddExtL_sxtw(iRegLNoSp dst, iRegL src1, iRegL src2, immI_32 lshift, immI_32 rshift, rFlagsReg cr)
13605 %{
13606 match(Set dst (AddL src1 (RShiftL (LShiftL src2 lshift) rshift)));
13607 ins_cost(INSN_COST);
13608 format %{ "add $dst, $src1, $src2, sxtw" %}
13609
13610 ins_encode %{
13611 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13612 as_Register($src2$$reg), ext::sxtw);
13613 %}
13614 ins_pipe(ialu_reg_reg);
13615 %}
13616
13617 // This pattern is automatically generated from aarch64_ad.m4
13618 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13619 instruct AddExtL_sxtb(iRegLNoSp dst, iRegL src1, iRegL src2, immI_56 lshift, immI_56 rshift, rFlagsReg cr)
13620 %{
13621 match(Set dst (AddL src1 (RShiftL (LShiftL src2 lshift) rshift)));
13622 ins_cost(INSN_COST);
13623 format %{ "add $dst, $src1, $src2, sxtb" %}
13624
13625 ins_encode %{
13626 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13627 as_Register($src2$$reg), ext::sxtb);
13628 %}
13629 ins_pipe(ialu_reg_reg);
13630 %}
13631
13632 // This pattern is automatically generated from aarch64_ad.m4
13633 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13634 instruct AddExtL_uxtb(iRegLNoSp dst, iRegL src1, iRegL src2, immI_56 lshift, immI_56 rshift, rFlagsReg cr)
13635 %{
13636 match(Set dst (AddL src1 (URShiftL (LShiftL src2 lshift) rshift)));
13637 ins_cost(INSN_COST);
13638 format %{ "add $dst, $src1, $src2, uxtb" %}
13639
13640 ins_encode %{
13641 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13642 as_Register($src2$$reg), ext::uxtb);
13643 %}
13644 ins_pipe(ialu_reg_reg);
13645 %}
13646
13647 // This pattern is automatically generated from aarch64_ad.m4
13648 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13649 instruct AddExtI_uxtb_and(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_255 mask, rFlagsReg cr)
13650 %{
13651 match(Set dst (AddI src1 (AndI src2 mask)));
13652 ins_cost(INSN_COST);
13653 format %{ "addw $dst, $src1, $src2, uxtb" %}
13654
13655 ins_encode %{
13656 __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
13657 as_Register($src2$$reg), ext::uxtb);
13658 %}
13659 ins_pipe(ialu_reg_reg);
13660 %}
13661
13662 // This pattern is automatically generated from aarch64_ad.m4
13663 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13664 instruct AddExtI_uxth_and(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_65535 mask, rFlagsReg cr)
13665 %{
13666 match(Set dst (AddI src1 (AndI src2 mask)));
13667 ins_cost(INSN_COST);
13668 format %{ "addw $dst, $src1, $src2, uxth" %}
13669
13670 ins_encode %{
13671 __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
13672 as_Register($src2$$reg), ext::uxth);
13673 %}
13674 ins_pipe(ialu_reg_reg);
13675 %}
13676
13677 // This pattern is automatically generated from aarch64_ad.m4
13678 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13679 instruct AddExtL_uxtb_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_255 mask, rFlagsReg cr)
13680 %{
13681 match(Set dst (AddL src1 (AndL src2 mask)));
13682 ins_cost(INSN_COST);
13683 format %{ "add $dst, $src1, $src2, uxtb" %}
13684
13685 ins_encode %{
13686 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13687 as_Register($src2$$reg), ext::uxtb);
13688 %}
13689 ins_pipe(ialu_reg_reg);
13690 %}
13691
13692 // This pattern is automatically generated from aarch64_ad.m4
13693 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13694 instruct AddExtL_uxth_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_65535 mask, rFlagsReg cr)
13695 %{
13696 match(Set dst (AddL src1 (AndL src2 mask)));
13697 ins_cost(INSN_COST);
13698 format %{ "add $dst, $src1, $src2, uxth" %}
13699
13700 ins_encode %{
13701 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13702 as_Register($src2$$reg), ext::uxth);
13703 %}
13704 ins_pipe(ialu_reg_reg);
13705 %}
13706
13707 // This pattern is automatically generated from aarch64_ad.m4
13708 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13709 instruct AddExtL_uxtw_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_4294967295 mask, rFlagsReg cr)
13710 %{
13711 match(Set dst (AddL src1 (AndL src2 mask)));
13712 ins_cost(INSN_COST);
13713 format %{ "add $dst, $src1, $src2, uxtw" %}
13714
13715 ins_encode %{
13716 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13717 as_Register($src2$$reg), ext::uxtw);
13718 %}
13719 ins_pipe(ialu_reg_reg);
13720 %}
13721
13722 // This pattern is automatically generated from aarch64_ad.m4
13723 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13724 instruct SubExtI_uxtb_and(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_255 mask, rFlagsReg cr)
13725 %{
13726 match(Set dst (SubI src1 (AndI src2 mask)));
13727 ins_cost(INSN_COST);
13728 format %{ "subw $dst, $src1, $src2, uxtb" %}
13729
13730 ins_encode %{
13731 __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
13732 as_Register($src2$$reg), ext::uxtb);
13733 %}
13734 ins_pipe(ialu_reg_reg);
13735 %}
13736
13737 // This pattern is automatically generated from aarch64_ad.m4
13738 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13739 instruct SubExtI_uxth_and(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_65535 mask, rFlagsReg cr)
13740 %{
13741 match(Set dst (SubI src1 (AndI src2 mask)));
13742 ins_cost(INSN_COST);
13743 format %{ "subw $dst, $src1, $src2, uxth" %}
13744
13745 ins_encode %{
13746 __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
13747 as_Register($src2$$reg), ext::uxth);
13748 %}
13749 ins_pipe(ialu_reg_reg);
13750 %}
13751
13752 // This pattern is automatically generated from aarch64_ad.m4
13753 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13754 instruct SubExtL_uxtb_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_255 mask, rFlagsReg cr)
13755 %{
13756 match(Set dst (SubL src1 (AndL src2 mask)));
13757 ins_cost(INSN_COST);
13758 format %{ "sub $dst, $src1, $src2, uxtb" %}
13759
13760 ins_encode %{
13761 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13762 as_Register($src2$$reg), ext::uxtb);
13763 %}
13764 ins_pipe(ialu_reg_reg);
13765 %}
13766
13767 // This pattern is automatically generated from aarch64_ad.m4
13768 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13769 instruct SubExtL_uxth_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_65535 mask, rFlagsReg cr)
13770 %{
13771 match(Set dst (SubL src1 (AndL src2 mask)));
13772 ins_cost(INSN_COST);
13773 format %{ "sub $dst, $src1, $src2, uxth" %}
13774
13775 ins_encode %{
13776 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13777 as_Register($src2$$reg), ext::uxth);
13778 %}
13779 ins_pipe(ialu_reg_reg);
13780 %}
13781
13782 // This pattern is automatically generated from aarch64_ad.m4
13783 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13784 instruct SubExtL_uxtw_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_4294967295 mask, rFlagsReg cr)
13785 %{
13786 match(Set dst (SubL src1 (AndL src2 mask)));
13787 ins_cost(INSN_COST);
13788 format %{ "sub $dst, $src1, $src2, uxtw" %}
13789
13790 ins_encode %{
13791 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13792 as_Register($src2$$reg), ext::uxtw);
13793 %}
13794 ins_pipe(ialu_reg_reg);
13795 %}
13796
13797
13798 // This pattern is automatically generated from aarch64_ad.m4
13799 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13800 instruct AddExtL_sxtb_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_56 lshift1, immI_56 rshift1, rFlagsReg cr)
13801 %{
13802 match(Set dst (AddL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
13803 ins_cost(1.9 * INSN_COST);
13804 format %{ "add $dst, $src1, $src2, sxtb #lshift2" %}
13805
13806 ins_encode %{
13807 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13808 as_Register($src2$$reg), ext::sxtb, ($lshift2$$constant));
13809 %}
13810 ins_pipe(ialu_reg_reg_shift);
13811 %}
13812
13813 // This pattern is automatically generated from aarch64_ad.m4
13814 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13815 instruct AddExtL_sxth_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_48 lshift1, immI_48 rshift1, rFlagsReg cr)
13816 %{
13817 match(Set dst (AddL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
13818 ins_cost(1.9 * INSN_COST);
13819 format %{ "add $dst, $src1, $src2, sxth #lshift2" %}
13820
13821 ins_encode %{
13822 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13823 as_Register($src2$$reg), ext::sxth, ($lshift2$$constant));
13824 %}
13825 ins_pipe(ialu_reg_reg_shift);
13826 %}
13827
13828 // This pattern is automatically generated from aarch64_ad.m4
13829 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13830 instruct AddExtL_sxtw_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_32 lshift1, immI_32 rshift1, rFlagsReg cr)
13831 %{
13832 match(Set dst (AddL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
13833 ins_cost(1.9 * INSN_COST);
13834 format %{ "add $dst, $src1, $src2, sxtw #lshift2" %}
13835
13836 ins_encode %{
13837 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13838 as_Register($src2$$reg), ext::sxtw, ($lshift2$$constant));
13839 %}
13840 ins_pipe(ialu_reg_reg_shift);
13841 %}
13842
13843 // This pattern is automatically generated from aarch64_ad.m4
13844 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13845 instruct SubExtL_sxtb_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_56 lshift1, immI_56 rshift1, rFlagsReg cr)
13846 %{
13847 match(Set dst (SubL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
13848 ins_cost(1.9 * INSN_COST);
13849 format %{ "sub $dst, $src1, $src2, sxtb #lshift2" %}
13850
13851 ins_encode %{
13852 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13853 as_Register($src2$$reg), ext::sxtb, ($lshift2$$constant));
13854 %}
13855 ins_pipe(ialu_reg_reg_shift);
13856 %}
13857
13858 // This pattern is automatically generated from aarch64_ad.m4
13859 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13860 instruct SubExtL_sxth_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_48 lshift1, immI_48 rshift1, rFlagsReg cr)
13861 %{
13862 match(Set dst (SubL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
13863 ins_cost(1.9 * INSN_COST);
13864 format %{ "sub $dst, $src1, $src2, sxth #lshift2" %}
13865
13866 ins_encode %{
13867 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13868 as_Register($src2$$reg), ext::sxth, ($lshift2$$constant));
13869 %}
13870 ins_pipe(ialu_reg_reg_shift);
13871 %}
13872
13873 // This pattern is automatically generated from aarch64_ad.m4
13874 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13875 instruct SubExtL_sxtw_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_32 lshift1, immI_32 rshift1, rFlagsReg cr)
13876 %{
13877 match(Set dst (SubL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
13878 ins_cost(1.9 * INSN_COST);
13879 format %{ "sub $dst, $src1, $src2, sxtw #lshift2" %}
13880
13881 ins_encode %{
13882 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13883 as_Register($src2$$reg), ext::sxtw, ($lshift2$$constant));
13884 %}
13885 ins_pipe(ialu_reg_reg_shift);
13886 %}
13887
13888 // This pattern is automatically generated from aarch64_ad.m4
13889 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13890 instruct AddExtI_sxtb_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immIExt lshift2, immI_24 lshift1, immI_24 rshift1, rFlagsReg cr)
13891 %{
13892 match(Set dst (AddI src1 (LShiftI (RShiftI (LShiftI src2 lshift1) rshift1) lshift2)));
13893 ins_cost(1.9 * INSN_COST);
13894 format %{ "addw $dst, $src1, $src2, sxtb #lshift2" %}
13895
13896 ins_encode %{
13897 __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
13898 as_Register($src2$$reg), ext::sxtb, ($lshift2$$constant));
13899 %}
13900 ins_pipe(ialu_reg_reg_shift);
13901 %}
13902
13903 // This pattern is automatically generated from aarch64_ad.m4
13904 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13905 instruct AddExtI_sxth_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immIExt lshift2, immI_16 lshift1, immI_16 rshift1, rFlagsReg cr)
13906 %{
13907 match(Set dst (AddI src1 (LShiftI (RShiftI (LShiftI src2 lshift1) rshift1) lshift2)));
13908 ins_cost(1.9 * INSN_COST);
13909 format %{ "addw $dst, $src1, $src2, sxth #lshift2" %}
13910
13911 ins_encode %{
13912 __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
13913 as_Register($src2$$reg), ext::sxth, ($lshift2$$constant));
13914 %}
13915 ins_pipe(ialu_reg_reg_shift);
13916 %}
13917
13918 // This pattern is automatically generated from aarch64_ad.m4
13919 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13920 instruct SubExtI_sxtb_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immIExt lshift2, immI_24 lshift1, immI_24 rshift1, rFlagsReg cr)
13921 %{
13922 match(Set dst (SubI src1 (LShiftI (RShiftI (LShiftI src2 lshift1) rshift1) lshift2)));
13923 ins_cost(1.9 * INSN_COST);
13924 format %{ "subw $dst, $src1, $src2, sxtb #lshift2" %}
13925
13926 ins_encode %{
13927 __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
13928 as_Register($src2$$reg), ext::sxtb, ($lshift2$$constant));
13929 %}
13930 ins_pipe(ialu_reg_reg_shift);
13931 %}
13932
13933 // This pattern is automatically generated from aarch64_ad.m4
13934 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13935 instruct SubExtI_sxth_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immIExt lshift2, immI_16 lshift1, immI_16 rshift1, rFlagsReg cr)
13936 %{
13937 match(Set dst (SubI src1 (LShiftI (RShiftI (LShiftI src2 lshift1) rshift1) lshift2)));
13938 ins_cost(1.9 * INSN_COST);
13939 format %{ "subw $dst, $src1, $src2, sxth #lshift2" %}
13940
13941 ins_encode %{
13942 __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
13943 as_Register($src2$$reg), ext::sxth, ($lshift2$$constant));
13944 %}
13945 ins_pipe(ialu_reg_reg_shift);
13946 %}
13947
13948 // This pattern is automatically generated from aarch64_ad.m4
13949 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13950 instruct AddExtI_shift(iRegLNoSp dst, iRegL src1, iRegIorL2I src2, immIExt lshift, rFlagsReg cr)
13951 %{
13952 match(Set dst (AddL src1 (LShiftL (ConvI2L src2) lshift)));
13953 ins_cost(1.9 * INSN_COST);
13954 format %{ "add $dst, $src1, $src2, sxtw #lshift" %}
13955
13956 ins_encode %{
13957 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13958 as_Register($src2$$reg), ext::sxtw, ($lshift$$constant));
13959 %}
13960 ins_pipe(ialu_reg_reg_shift);
13961 %}
13962
13963 // This pattern is automatically generated from aarch64_ad.m4
13964 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13965 instruct SubExtI_shift(iRegLNoSp dst, iRegL src1, iRegIorL2I src2, immIExt lshift, rFlagsReg cr)
13966 %{
13967 match(Set dst (SubL src1 (LShiftL (ConvI2L src2) lshift)));
13968 ins_cost(1.9 * INSN_COST);
13969 format %{ "sub $dst, $src1, $src2, sxtw #lshift" %}
13970
13971 ins_encode %{
13972 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13973 as_Register($src2$$reg), ext::sxtw, ($lshift$$constant));
13974 %}
13975 ins_pipe(ialu_reg_reg_shift);
13976 %}
13977
13978 // This pattern is automatically generated from aarch64_ad.m4
13979 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13980 instruct AddExtL_uxtb_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_255 mask, immIExt lshift, rFlagsReg cr)
13981 %{
13982 match(Set dst (AddL src1 (LShiftL (AndL src2 mask) lshift)));
13983 ins_cost(1.9 * INSN_COST);
13984 format %{ "add $dst, $src1, $src2, uxtb #lshift" %}
13985
13986 ins_encode %{
13987 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13988 as_Register($src2$$reg), ext::uxtb, ($lshift$$constant));
13989 %}
13990 ins_pipe(ialu_reg_reg_shift);
13991 %}
13992
13993 // This pattern is automatically generated from aarch64_ad.m4
13994 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13995 instruct AddExtL_uxth_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_65535 mask, immIExt lshift, rFlagsReg cr)
13996 %{
13997 match(Set dst (AddL src1 (LShiftL (AndL src2 mask) lshift)));
13998 ins_cost(1.9 * INSN_COST);
13999 format %{ "add $dst, $src1, $src2, uxth #lshift" %}
14000
14001 ins_encode %{
14002 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
14003 as_Register($src2$$reg), ext::uxth, ($lshift$$constant));
14004 %}
14005 ins_pipe(ialu_reg_reg_shift);
14006 %}
14007
14008 // This pattern is automatically generated from aarch64_ad.m4
14009 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
14010 instruct AddExtL_uxtw_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_4294967295 mask, immIExt lshift, rFlagsReg cr)
14011 %{
14012 match(Set dst (AddL src1 (LShiftL (AndL src2 mask) lshift)));
14013 ins_cost(1.9 * INSN_COST);
14014 format %{ "add $dst, $src1, $src2, uxtw #lshift" %}
14015
14016 ins_encode %{
14017 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
14018 as_Register($src2$$reg), ext::uxtw, ($lshift$$constant));
14019 %}
14020 ins_pipe(ialu_reg_reg_shift);
14021 %}
14022
14023 // This pattern is automatically generated from aarch64_ad.m4
14024 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
14025 instruct SubExtL_uxtb_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_255 mask, immIExt lshift, rFlagsReg cr)
14026 %{
14027 match(Set dst (SubL src1 (LShiftL (AndL src2 mask) lshift)));
14028 ins_cost(1.9 * INSN_COST);
14029 format %{ "sub $dst, $src1, $src2, uxtb #lshift" %}
14030
14031 ins_encode %{
14032 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
14033 as_Register($src2$$reg), ext::uxtb, ($lshift$$constant));
14034 %}
14035 ins_pipe(ialu_reg_reg_shift);
14036 %}
14037
14038 // This pattern is automatically generated from aarch64_ad.m4
14039 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
14040 instruct SubExtL_uxth_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_65535 mask, immIExt lshift, rFlagsReg cr)
14041 %{
14042 match(Set dst (SubL src1 (LShiftL (AndL src2 mask) lshift)));
14043 ins_cost(1.9 * INSN_COST);
14044 format %{ "sub $dst, $src1, $src2, uxth #lshift" %}
14045
14046 ins_encode %{
14047 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
14048 as_Register($src2$$reg), ext::uxth, ($lshift$$constant));
14049 %}
14050 ins_pipe(ialu_reg_reg_shift);
14051 %}
14052
14053 // This pattern is automatically generated from aarch64_ad.m4
14054 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
14055 instruct SubExtL_uxtw_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_4294967295 mask, immIExt lshift, rFlagsReg cr)
14056 %{
14057 match(Set dst (SubL src1 (LShiftL (AndL src2 mask) lshift)));
14058 ins_cost(1.9 * INSN_COST);
14059 format %{ "sub $dst, $src1, $src2, uxtw #lshift" %}
14060
14061 ins_encode %{
14062 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
14063 as_Register($src2$$reg), ext::uxtw, ($lshift$$constant));
14064 %}
14065 ins_pipe(ialu_reg_reg_shift);
14066 %}
14067
14068 // This pattern is automatically generated from aarch64_ad.m4
14069 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
14070 instruct AddExtI_uxtb_and_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_255 mask, immIExt lshift, rFlagsReg cr)
14071 %{
14072 match(Set dst (AddI src1 (LShiftI (AndI src2 mask) lshift)));
14073 ins_cost(1.9 * INSN_COST);
14074 format %{ "addw $dst, $src1, $src2, uxtb #lshift" %}
14075
14076 ins_encode %{
14077 __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
14078 as_Register($src2$$reg), ext::uxtb, ($lshift$$constant));
14079 %}
14080 ins_pipe(ialu_reg_reg_shift);
14081 %}
14082
14083 // This pattern is automatically generated from aarch64_ad.m4
14084 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
14085 instruct AddExtI_uxth_and_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_65535 mask, immIExt lshift, rFlagsReg cr)
14086 %{
14087 match(Set dst (AddI src1 (LShiftI (AndI src2 mask) lshift)));
14088 ins_cost(1.9 * INSN_COST);
14089 format %{ "addw $dst, $src1, $src2, uxth #lshift" %}
14090
14091 ins_encode %{
14092 __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
14093 as_Register($src2$$reg), ext::uxth, ($lshift$$constant));
14094 %}
14095 ins_pipe(ialu_reg_reg_shift);
14096 %}
14097
14098 // This pattern is automatically generated from aarch64_ad.m4
14099 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
14100 instruct SubExtI_uxtb_and_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_255 mask, immIExt lshift, rFlagsReg cr)
14101 %{
14102 match(Set dst (SubI src1 (LShiftI (AndI src2 mask) lshift)));
14103 ins_cost(1.9 * INSN_COST);
14104 format %{ "subw $dst, $src1, $src2, uxtb #lshift" %}
14105
14106 ins_encode %{
14107 __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
14108 as_Register($src2$$reg), ext::uxtb, ($lshift$$constant));
14109 %}
14110 ins_pipe(ialu_reg_reg_shift);
14111 %}
14112
14113 // This pattern is automatically generated from aarch64_ad.m4
14114 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
14115 instruct SubExtI_uxth_and_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_65535 mask, immIExt lshift, rFlagsReg cr)
14116 %{
14117 match(Set dst (SubI src1 (LShiftI (AndI src2 mask) lshift)));
14118 ins_cost(1.9 * INSN_COST);
14119 format %{ "subw $dst, $src1, $src2, uxth #lshift" %}
14120
14121 ins_encode %{
14122 __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
14123 as_Register($src2$$reg), ext::uxth, ($lshift$$constant));
14124 %}
14125 ins_pipe(ialu_reg_reg_shift);
14126 %}
14127
14128
14129
14130 // END This section of the file is automatically generated. Do not edit --------------
14131
14132
14133 // ============================================================================
14134 // Floating Point Arithmetic Instructions
14135
14136 instruct addF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
14137 match(Set dst (AddF src1 src2));
14138
14139 ins_cost(INSN_COST * 5);
14140 format %{ "fadds $dst, $src1, $src2" %}
14141
14142 ins_encode %{
14143 __ fadds(as_FloatRegister($dst$$reg),
14144 as_FloatRegister($src1$$reg),
14145 as_FloatRegister($src2$$reg));
14146 %}
14147
14148 ins_pipe(fp_dop_reg_reg_s);
14149 %}
14150
14151 instruct addD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
14152 match(Set dst (AddD src1 src2));
14153
14154 ins_cost(INSN_COST * 5);
14155 format %{ "faddd $dst, $src1, $src2" %}
14156
14157 ins_encode %{
14158 __ faddd(as_FloatRegister($dst$$reg),
14159 as_FloatRegister($src1$$reg),
14160 as_FloatRegister($src2$$reg));
14161 %}
14162
14163 ins_pipe(fp_dop_reg_reg_d);
14164 %}
14165
14166 instruct subF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
14167 match(Set dst (SubF src1 src2));
14168
14169 ins_cost(INSN_COST * 5);
14170 format %{ "fsubs $dst, $src1, $src2" %}
14171
14172 ins_encode %{
14173 __ fsubs(as_FloatRegister($dst$$reg),
14174 as_FloatRegister($src1$$reg),
14175 as_FloatRegister($src2$$reg));
14176 %}
14177
14178 ins_pipe(fp_dop_reg_reg_s);
14179 %}
14180
14181 instruct subD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
14182 match(Set dst (SubD src1 src2));
14183
14184 ins_cost(INSN_COST * 5);
14185 format %{ "fsubd $dst, $src1, $src2" %}
14186
14187 ins_encode %{
14188 __ fsubd(as_FloatRegister($dst$$reg),
14189 as_FloatRegister($src1$$reg),
14190 as_FloatRegister($src2$$reg));
14191 %}
14192
14193 ins_pipe(fp_dop_reg_reg_d);
14194 %}
14195
14196 instruct mulF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
14197 match(Set dst (MulF src1 src2));
14198
14199 ins_cost(INSN_COST * 6);
14200 format %{ "fmuls $dst, $src1, $src2" %}
14201
14202 ins_encode %{
14203 __ fmuls(as_FloatRegister($dst$$reg),
14204 as_FloatRegister($src1$$reg),
14205 as_FloatRegister($src2$$reg));
14206 %}
14207
14208 ins_pipe(fp_dop_reg_reg_s);
14209 %}
14210
14211 instruct mulD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
14212 match(Set dst (MulD src1 src2));
14213
14214 ins_cost(INSN_COST * 6);
14215 format %{ "fmuld $dst, $src1, $src2" %}
14216
14217 ins_encode %{
14218 __ fmuld(as_FloatRegister($dst$$reg),
14219 as_FloatRegister($src1$$reg),
14220 as_FloatRegister($src2$$reg));
14221 %}
14222
14223 ins_pipe(fp_dop_reg_reg_d);
14224 %}
14225
14226 // src1 * src2 + src3
14227 instruct maddF_reg_reg(vRegF dst, vRegF src1, vRegF src2, vRegF src3) %{
14228 predicate(UseFMA);
14229 match(Set dst (FmaF src3 (Binary src1 src2)));
14230
14231 format %{ "fmadds $dst, $src1, $src2, $src3" %}
14232
14233 ins_encode %{
14234 __ fmadds(as_FloatRegister($dst$$reg),
14235 as_FloatRegister($src1$$reg),
14236 as_FloatRegister($src2$$reg),
14237 as_FloatRegister($src3$$reg));
14238 %}
14239
14240 ins_pipe(pipe_class_default);
14241 %}
14242
14243 // src1 * src2 + src3
14244 instruct maddD_reg_reg(vRegD dst, vRegD src1, vRegD src2, vRegD src3) %{
14245 predicate(UseFMA);
14246 match(Set dst (FmaD src3 (Binary src1 src2)));
14247
14248 format %{ "fmaddd $dst, $src1, $src2, $src3" %}
14249
14250 ins_encode %{
14251 __ fmaddd(as_FloatRegister($dst$$reg),
14252 as_FloatRegister($src1$$reg),
14253 as_FloatRegister($src2$$reg),
14254 as_FloatRegister($src3$$reg));
14255 %}
14256
14257 ins_pipe(pipe_class_default);
14258 %}
14259
14260 // -src1 * src2 + src3
14261 instruct msubF_reg_reg(vRegF dst, vRegF src1, vRegF src2, vRegF src3) %{
14262 predicate(UseFMA);
14263 match(Set dst (FmaF src3 (Binary (NegF src1) src2)));
14264 match(Set dst (FmaF src3 (Binary src1 (NegF src2))));
14265
14266 format %{ "fmsubs $dst, $src1, $src2, $src3" %}
14267
14268 ins_encode %{
14269 __ fmsubs(as_FloatRegister($dst$$reg),
14270 as_FloatRegister($src1$$reg),
14271 as_FloatRegister($src2$$reg),
14272 as_FloatRegister($src3$$reg));
14273 %}
14274
14275 ins_pipe(pipe_class_default);
14276 %}
14277
14278 // -src1 * src2 + src3
14279 instruct msubD_reg_reg(vRegD dst, vRegD src1, vRegD src2, vRegD src3) %{
14280 predicate(UseFMA);
14281 match(Set dst (FmaD src3 (Binary (NegD src1) src2)));
14282 match(Set dst (FmaD src3 (Binary src1 (NegD src2))));
14283
14284 format %{ "fmsubd $dst, $src1, $src2, $src3" %}
14285
14286 ins_encode %{
14287 __ fmsubd(as_FloatRegister($dst$$reg),
14288 as_FloatRegister($src1$$reg),
14289 as_FloatRegister($src2$$reg),
14290 as_FloatRegister($src3$$reg));
14291 %}
14292
14293 ins_pipe(pipe_class_default);
14294 %}
14295
14296 // -src1 * src2 - src3
14297 instruct mnaddF_reg_reg(vRegF dst, vRegF src1, vRegF src2, vRegF src3) %{
14298 predicate(UseFMA);
14299 match(Set dst (FmaF (NegF src3) (Binary (NegF src1) src2)));
14300 match(Set dst (FmaF (NegF src3) (Binary src1 (NegF src2))));
14301
14302 format %{ "fnmadds $dst, $src1, $src2, $src3" %}
14303
14304 ins_encode %{
14305 __ fnmadds(as_FloatRegister($dst$$reg),
14306 as_FloatRegister($src1$$reg),
14307 as_FloatRegister($src2$$reg),
14308 as_FloatRegister($src3$$reg));
14309 %}
14310
14311 ins_pipe(pipe_class_default);
14312 %}
14313
14314 // -src1 * src2 - src3
14315 instruct mnaddD_reg_reg(vRegD dst, vRegD src1, vRegD src2, vRegD src3) %{
14316 predicate(UseFMA);
14317 match(Set dst (FmaD (NegD src3) (Binary (NegD src1) src2)));
14318 match(Set dst (FmaD (NegD src3) (Binary src1 (NegD src2))));
14319
14320 format %{ "fnmaddd $dst, $src1, $src2, $src3" %}
14321
14322 ins_encode %{
14323 __ fnmaddd(as_FloatRegister($dst$$reg),
14324 as_FloatRegister($src1$$reg),
14325 as_FloatRegister($src2$$reg),
14326 as_FloatRegister($src3$$reg));
14327 %}
14328
14329 ins_pipe(pipe_class_default);
14330 %}
14331
14332 // src1 * src2 - src3
14333 instruct mnsubF_reg_reg(vRegF dst, vRegF src1, vRegF src2, vRegF src3, immF0 zero) %{
14334 predicate(UseFMA);
14335 match(Set dst (FmaF (NegF src3) (Binary src1 src2)));
14336
14337 format %{ "fnmsubs $dst, $src1, $src2, $src3" %}
14338
14339 ins_encode %{
14340 __ fnmsubs(as_FloatRegister($dst$$reg),
14341 as_FloatRegister($src1$$reg),
14342 as_FloatRegister($src2$$reg),
14343 as_FloatRegister($src3$$reg));
14344 %}
14345
14346 ins_pipe(pipe_class_default);
14347 %}
14348
14349 // src1 * src2 - src3
14350 instruct mnsubD_reg_reg(vRegD dst, vRegD src1, vRegD src2, vRegD src3, immD0 zero) %{
14351 predicate(UseFMA);
14352 match(Set dst (FmaD (NegD src3) (Binary src1 src2)));
14353
14354 format %{ "fnmsubd $dst, $src1, $src2, $src3" %}
14355
14356 ins_encode %{
14357 // n.b. insn name should be fnmsubd
14358 __ fnmsub(as_FloatRegister($dst$$reg),
14359 as_FloatRegister($src1$$reg),
14360 as_FloatRegister($src2$$reg),
14361 as_FloatRegister($src3$$reg));
14362 %}
14363
14364 ins_pipe(pipe_class_default);
14365 %}
14366
14367
14368 // Math.max(FF)F
14369 instruct maxF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
14370 match(Set dst (MaxF src1 src2));
14371
14372 format %{ "fmaxs $dst, $src1, $src2" %}
14373 ins_encode %{
14374 __ fmaxs(as_FloatRegister($dst$$reg),
14375 as_FloatRegister($src1$$reg),
14376 as_FloatRegister($src2$$reg));
14377 %}
14378
14379 ins_pipe(fp_dop_reg_reg_s);
14380 %}
14381
14382 // Math.min(FF)F
14383 instruct minF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
14384 match(Set dst (MinF src1 src2));
14385
14386 format %{ "fmins $dst, $src1, $src2" %}
14387 ins_encode %{
14388 __ fmins(as_FloatRegister($dst$$reg),
14389 as_FloatRegister($src1$$reg),
14390 as_FloatRegister($src2$$reg));
14391 %}
14392
14393 ins_pipe(fp_dop_reg_reg_s);
14394 %}
14395
14396 // Math.max(DD)D
14397 instruct maxD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
14398 match(Set dst (MaxD src1 src2));
14399
14400 format %{ "fmaxd $dst, $src1, $src2" %}
14401 ins_encode %{
14402 __ fmaxd(as_FloatRegister($dst$$reg),
14403 as_FloatRegister($src1$$reg),
14404 as_FloatRegister($src2$$reg));
14405 %}
14406
14407 ins_pipe(fp_dop_reg_reg_d);
14408 %}
14409
14410 // Math.min(DD)D
14411 instruct minD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
14412 match(Set dst (MinD src1 src2));
14413
14414 format %{ "fmind $dst, $src1, $src2" %}
14415 ins_encode %{
14416 __ fmind(as_FloatRegister($dst$$reg),
14417 as_FloatRegister($src1$$reg),
14418 as_FloatRegister($src2$$reg));
14419 %}
14420
14421 ins_pipe(fp_dop_reg_reg_d);
14422 %}
14423
14424
14425 instruct divF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
14426 match(Set dst (DivF src1 src2));
14427
14428 ins_cost(INSN_COST * 18);
14429 format %{ "fdivs $dst, $src1, $src2" %}
14430
14431 ins_encode %{
14432 __ fdivs(as_FloatRegister($dst$$reg),
14433 as_FloatRegister($src1$$reg),
14434 as_FloatRegister($src2$$reg));
14435 %}
14436
14437 ins_pipe(fp_div_s);
14438 %}
14439
14440 instruct divD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
14441 match(Set dst (DivD src1 src2));
14442
14443 ins_cost(INSN_COST * 32);
14444 format %{ "fdivd $dst, $src1, $src2" %}
14445
14446 ins_encode %{
14447 __ fdivd(as_FloatRegister($dst$$reg),
14448 as_FloatRegister($src1$$reg),
14449 as_FloatRegister($src2$$reg));
14450 %}
14451
14452 ins_pipe(fp_div_d);
14453 %}
14454
14455 instruct negF_reg_reg(vRegF dst, vRegF src) %{
14456 match(Set dst (NegF src));
14457
14458 ins_cost(INSN_COST * 3);
14459 format %{ "fneg $dst, $src" %}
14460
14461 ins_encode %{
14462 __ fnegs(as_FloatRegister($dst$$reg),
14463 as_FloatRegister($src$$reg));
14464 %}
14465
14466 ins_pipe(fp_uop_s);
14467 %}
14468
14469 instruct negD_reg_reg(vRegD dst, vRegD src) %{
14470 match(Set dst (NegD src));
14471
14472 ins_cost(INSN_COST * 3);
14473 format %{ "fnegd $dst, $src" %}
14474
14475 ins_encode %{
14476 __ fnegd(as_FloatRegister($dst$$reg),
14477 as_FloatRegister($src$$reg));
14478 %}
14479
14480 ins_pipe(fp_uop_d);
14481 %}
14482
14483 instruct absI_reg(iRegINoSp dst, iRegIorL2I src, rFlagsReg cr)
14484 %{
14485 match(Set dst (AbsI src));
14486
14487 effect(KILL cr);
14488 ins_cost(INSN_COST * 2);
14489 format %{ "cmpw $src, zr\n\t"
14490 "cnegw $dst, $src, Assembler::LT\t# int abs"
14491 %}
14492
14493 ins_encode %{
14494 __ cmpw(as_Register($src$$reg), zr);
14495 __ cnegw(as_Register($dst$$reg), as_Register($src$$reg), Assembler::LT);
14496 %}
14497 ins_pipe(pipe_class_default);
14498 %}
14499
14500 instruct absL_reg(iRegLNoSp dst, iRegL src, rFlagsReg cr)
14501 %{
14502 match(Set dst (AbsL src));
14503
14504 effect(KILL cr);
14505 ins_cost(INSN_COST * 2);
14506 format %{ "cmp $src, zr\n\t"
14507 "cneg $dst, $src, Assembler::LT\t# long abs"
14508 %}
14509
14510 ins_encode %{
14511 __ cmp(as_Register($src$$reg), zr);
14512 __ cneg(as_Register($dst$$reg), as_Register($src$$reg), Assembler::LT);
14513 %}
14514 ins_pipe(pipe_class_default);
14515 %}
14516
14517 instruct absF_reg(vRegF dst, vRegF src) %{
14518 match(Set dst (AbsF src));
14519
14520 ins_cost(INSN_COST * 3);
14521 format %{ "fabss $dst, $src" %}
14522 ins_encode %{
14523 __ fabss(as_FloatRegister($dst$$reg),
14524 as_FloatRegister($src$$reg));
14525 %}
14526
14527 ins_pipe(fp_uop_s);
14528 %}
14529
14530 instruct absD_reg(vRegD dst, vRegD src) %{
14531 match(Set dst (AbsD src));
14532
14533 ins_cost(INSN_COST * 3);
14534 format %{ "fabsd $dst, $src" %}
14535 ins_encode %{
14536 __ fabsd(as_FloatRegister($dst$$reg),
14537 as_FloatRegister($src$$reg));
14538 %}
14539
14540 ins_pipe(fp_uop_d);
14541 %}
14542
14543 instruct absdF_reg(vRegF dst, vRegF src1, vRegF src2) %{
14544 match(Set dst (AbsF (SubF src1 src2)));
14545
14546 ins_cost(INSN_COST * 3);
14547 format %{ "fabds $dst, $src1, $src2" %}
14548 ins_encode %{
14549 __ fabds(as_FloatRegister($dst$$reg),
14550 as_FloatRegister($src1$$reg),
14551 as_FloatRegister($src2$$reg));
14552 %}
14553
14554 ins_pipe(fp_uop_s);
14555 %}
14556
14557 instruct absdD_reg(vRegD dst, vRegD src1, vRegD src2) %{
14558 match(Set dst (AbsD (SubD src1 src2)));
14559
14560 ins_cost(INSN_COST * 3);
14561 format %{ "fabdd $dst, $src1, $src2" %}
14562 ins_encode %{
14563 __ fabdd(as_FloatRegister($dst$$reg),
14564 as_FloatRegister($src1$$reg),
14565 as_FloatRegister($src2$$reg));
14566 %}
14567
14568 ins_pipe(fp_uop_d);
14569 %}
14570
14571 instruct sqrtD_reg(vRegD dst, vRegD src) %{
14572 match(Set dst (SqrtD src));
14573
14574 ins_cost(INSN_COST * 50);
14575 format %{ "fsqrtd $dst, $src" %}
14576 ins_encode %{
14577 __ fsqrtd(as_FloatRegister($dst$$reg),
14578 as_FloatRegister($src$$reg));
14579 %}
14580
14581 ins_pipe(fp_div_s);
14582 %}
14583
14584 instruct sqrtF_reg(vRegF dst, vRegF src) %{
14585 match(Set dst (SqrtF src));
14586
14587 ins_cost(INSN_COST * 50);
14588 format %{ "fsqrts $dst, $src" %}
14589 ins_encode %{
14590 __ fsqrts(as_FloatRegister($dst$$reg),
14591 as_FloatRegister($src$$reg));
14592 %}
14593
14594 ins_pipe(fp_div_d);
14595 %}
14596
14597 // Math.rint, floor, ceil
14598 instruct roundD_reg(vRegD dst, vRegD src, immI rmode) %{
14599 match(Set dst (RoundDoubleMode src rmode));
14600 format %{ "frint $dst, $src, $rmode" %}
14601 ins_encode %{
14602 switch ($rmode$$constant) {
14603 case RoundDoubleModeNode::rmode_rint:
14604 __ frintnd(as_FloatRegister($dst$$reg),
14605 as_FloatRegister($src$$reg));
14606 break;
14607 case RoundDoubleModeNode::rmode_floor:
14608 __ frintmd(as_FloatRegister($dst$$reg),
14609 as_FloatRegister($src$$reg));
14610 break;
14611 case RoundDoubleModeNode::rmode_ceil:
14612 __ frintpd(as_FloatRegister($dst$$reg),
14613 as_FloatRegister($src$$reg));
14614 break;
14615 }
14616 %}
14617 ins_pipe(fp_uop_d);
14618 %}
14619
14620 instruct copySignD_reg(vRegD dst, vRegD src1, vRegD src2, vRegD zero) %{
14621 match(Set dst (CopySignD src1 (Binary src2 zero)));
14622 effect(TEMP_DEF dst, USE src1, USE src2, USE zero);
14623 format %{ "CopySignD $dst $src1 $src2" %}
14624 ins_encode %{
14625 FloatRegister dst = as_FloatRegister($dst$$reg),
14626 src1 = as_FloatRegister($src1$$reg),
14627 src2 = as_FloatRegister($src2$$reg),
14628 zero = as_FloatRegister($zero$$reg);
14629 __ fnegd(dst, zero);
14630 __ bsl(dst, __ T8B, src2, src1);
14631 %}
14632 ins_pipe(fp_uop_d);
14633 %}
14634
14635 instruct copySignF_reg(vRegF dst, vRegF src1, vRegF src2) %{
14636 match(Set dst (CopySignF src1 src2));
14637 effect(TEMP_DEF dst, USE src1, USE src2);
14638 format %{ "CopySignF $dst $src1 $src2" %}
14639 ins_encode %{
14640 FloatRegister dst = as_FloatRegister($dst$$reg),
14641 src1 = as_FloatRegister($src1$$reg),
14642 src2 = as_FloatRegister($src2$$reg);
14643 __ movi(dst, __ T2S, 0x80, 24);
14644 __ bsl(dst, __ T8B, src2, src1);
14645 %}
14646 ins_pipe(fp_uop_d);
14647 %}
14648
14649 instruct signumD_reg(vRegD dst, vRegD src, vRegD zero, vRegD one) %{
14650 match(Set dst (SignumD src (Binary zero one)));
14651 effect(TEMP_DEF dst, USE src, USE zero, USE one);
14652 format %{ "signumD $dst, $src" %}
14653 ins_encode %{
14654 FloatRegister src = as_FloatRegister($src$$reg),
14655 dst = as_FloatRegister($dst$$reg),
14656 zero = as_FloatRegister($zero$$reg),
14657 one = as_FloatRegister($one$$reg);
14658 __ facgtd(dst, src, zero); // dst=0 for +-0.0 and NaN. 0xFFF..F otherwise
14659 __ ushrd(dst, dst, 1); // dst=0 for +-0.0 and NaN. 0x7FF..F otherwise
14660 // Bit selection instruction gets bit from "one" for each enabled bit in
14661 // "dst", otherwise gets a bit from "src". For "src" that contains +-0.0 or
14662 // NaN the whole "src" will be copied because "dst" is zero. For all other
14663 // "src" values dst is 0x7FF..F, which means only the sign bit is copied
14664 // from "src", and all other bits are copied from 1.0.
14665 __ bsl(dst, __ T8B, one, src);
14666 %}
14667 ins_pipe(fp_uop_d);
14668 %}
14669
14670 instruct signumF_reg(vRegF dst, vRegF src, vRegF zero, vRegF one) %{
14671 match(Set dst (SignumF src (Binary zero one)));
14672 effect(TEMP_DEF dst, USE src, USE zero, USE one);
14673 format %{ "signumF $dst, $src" %}
14674 ins_encode %{
14675 FloatRegister src = as_FloatRegister($src$$reg),
14676 dst = as_FloatRegister($dst$$reg),
14677 zero = as_FloatRegister($zero$$reg),
14678 one = as_FloatRegister($one$$reg);
14679 __ facgts(dst, src, zero); // dst=0 for +-0.0 and NaN. 0xFFF..F otherwise
14680 __ ushr(dst, __ T2S, dst, 1); // dst=0 for +-0.0 and NaN. 0x7FF..F otherwise
14681 // Bit selection instruction gets bit from "one" for each enabled bit in
14682 // "dst", otherwise gets a bit from "src". For "src" that contains +-0.0 or
14683 // NaN the whole "src" will be copied because "dst" is zero. For all other
14684 // "src" values dst is 0x7FF..F, which means only the sign bit is copied
14685 // from "src", and all other bits are copied from 1.0.
14686 __ bsl(dst, __ T8B, one, src);
14687 %}
14688 ins_pipe(fp_uop_d);
14689 %}
14690
14691 instruct onspinwait() %{
14692 match(OnSpinWait);
14693 ins_cost(INSN_COST);
14694
14695 format %{ "onspinwait" %}
14696
14697 ins_encode %{
14698 __ spin_wait();
14699 %}
14700 ins_pipe(pipe_class_empty);
14701 %}
14702
14703 // ============================================================================
14704 // Logical Instructions
14705
14706 // Integer Logical Instructions
14707
14708 // And Instructions
14709
14710
14711 instruct andI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, rFlagsReg cr) %{
14712 match(Set dst (AndI src1 src2));
14713
14714 format %{ "andw $dst, $src1, $src2\t# int" %}
14715
14716 ins_cost(INSN_COST);
14717 ins_encode %{
14718 __ andw(as_Register($dst$$reg),
14719 as_Register($src1$$reg),
14720 as_Register($src2$$reg));
14721 %}
14722
14723 ins_pipe(ialu_reg_reg);
14724 %}
14725
14726 instruct andI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immILog src2, rFlagsReg cr) %{
14727 match(Set dst (AndI src1 src2));
14728
14729 format %{ "andsw $dst, $src1, $src2\t# int" %}
14730
14731 ins_cost(INSN_COST);
14732 ins_encode %{
14733 __ andw(as_Register($dst$$reg),
14734 as_Register($src1$$reg),
14735 (uint64_t)($src2$$constant));
14736 %}
14737
14738 ins_pipe(ialu_reg_imm);
14739 %}
14740
14741 // Or Instructions
14742
14743 instruct orI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
14744 match(Set dst (OrI src1 src2));
14745
14746 format %{ "orrw $dst, $src1, $src2\t# int" %}
14747
14748 ins_cost(INSN_COST);
14749 ins_encode %{
14750 __ orrw(as_Register($dst$$reg),
14751 as_Register($src1$$reg),
14752 as_Register($src2$$reg));
14753 %}
14754
14755 ins_pipe(ialu_reg_reg);
14756 %}
14757
14758 instruct orI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immILog src2) %{
14759 match(Set dst (OrI src1 src2));
14760
14761 format %{ "orrw $dst, $src1, $src2\t# int" %}
14762
14763 ins_cost(INSN_COST);
14764 ins_encode %{
14765 __ orrw(as_Register($dst$$reg),
14766 as_Register($src1$$reg),
14767 (uint64_t)($src2$$constant));
14768 %}
14769
14770 ins_pipe(ialu_reg_imm);
14771 %}
14772
14773 // Xor Instructions
14774
14775 instruct xorI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
14776 match(Set dst (XorI src1 src2));
14777
14778 format %{ "eorw $dst, $src1, $src2\t# int" %}
14779
14780 ins_cost(INSN_COST);
14781 ins_encode %{
14782 __ eorw(as_Register($dst$$reg),
14783 as_Register($src1$$reg),
14784 as_Register($src2$$reg));
14785 %}
14786
14787 ins_pipe(ialu_reg_reg);
14788 %}
14789
14790 instruct xorI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immILog src2) %{
14791 match(Set dst (XorI src1 src2));
14792
14793 format %{ "eorw $dst, $src1, $src2\t# int" %}
14794
14795 ins_cost(INSN_COST);
14796 ins_encode %{
14797 __ eorw(as_Register($dst$$reg),
14798 as_Register($src1$$reg),
14799 (uint64_t)($src2$$constant));
14800 %}
14801
14802 ins_pipe(ialu_reg_imm);
14803 %}
14804
14805 // Long Logical Instructions
14806 // TODO
14807
14808 instruct andL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2, rFlagsReg cr) %{
14809 match(Set dst (AndL src1 src2));
14810
14811 format %{ "and $dst, $src1, $src2\t# int" %}
14812
14813 ins_cost(INSN_COST);
14814 ins_encode %{
14815 __ andr(as_Register($dst$$reg),
14816 as_Register($src1$$reg),
14817 as_Register($src2$$reg));
14818 %}
14819
14820 ins_pipe(ialu_reg_reg);
14821 %}
14822
14823 instruct andL_reg_imm(iRegLNoSp dst, iRegL src1, immLLog src2, rFlagsReg cr) %{
14824 match(Set dst (AndL src1 src2));
14825
14826 format %{ "and $dst, $src1, $src2\t# int" %}
14827
14828 ins_cost(INSN_COST);
14829 ins_encode %{
14830 __ andr(as_Register($dst$$reg),
14831 as_Register($src1$$reg),
14832 (uint64_t)($src2$$constant));
14833 %}
14834
14835 ins_pipe(ialu_reg_imm);
14836 %}
14837
14838 // Or Instructions
14839
14840 instruct orL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
14841 match(Set dst (OrL src1 src2));
14842
14843 format %{ "orr $dst, $src1, $src2\t# int" %}
14844
14845 ins_cost(INSN_COST);
14846 ins_encode %{
14847 __ orr(as_Register($dst$$reg),
14848 as_Register($src1$$reg),
14849 as_Register($src2$$reg));
14850 %}
14851
14852 ins_pipe(ialu_reg_reg);
14853 %}
14854
14855 instruct orL_reg_imm(iRegLNoSp dst, iRegL src1, immLLog src2) %{
14856 match(Set dst (OrL src1 src2));
14857
14858 format %{ "orr $dst, $src1, $src2\t# int" %}
14859
14860 ins_cost(INSN_COST);
14861 ins_encode %{
14862 __ orr(as_Register($dst$$reg),
14863 as_Register($src1$$reg),
14864 (uint64_t)($src2$$constant));
14865 %}
14866
14867 ins_pipe(ialu_reg_imm);
14868 %}
14869
14870 // Xor Instructions
14871
14872 instruct xorL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
14873 match(Set dst (XorL src1 src2));
14874
14875 format %{ "eor $dst, $src1, $src2\t# int" %}
14876
14877 ins_cost(INSN_COST);
14878 ins_encode %{
14879 __ eor(as_Register($dst$$reg),
14880 as_Register($src1$$reg),
14881 as_Register($src2$$reg));
14882 %}
14883
14884 ins_pipe(ialu_reg_reg);
14885 %}
14886
14887 instruct xorL_reg_imm(iRegLNoSp dst, iRegL src1, immLLog src2) %{
14888 match(Set dst (XorL src1 src2));
14889
14890 ins_cost(INSN_COST);
14891 format %{ "eor $dst, $src1, $src2\t# int" %}
14892
14893 ins_encode %{
14894 __ eor(as_Register($dst$$reg),
14895 as_Register($src1$$reg),
14896 (uint64_t)($src2$$constant));
14897 %}
14898
14899 ins_pipe(ialu_reg_imm);
14900 %}
14901
14902 instruct convI2L_reg_reg(iRegLNoSp dst, iRegIorL2I src)
14903 %{
14904 match(Set dst (ConvI2L src));
14905
14906 ins_cost(INSN_COST);
14907 format %{ "sxtw $dst, $src\t# i2l" %}
14908 ins_encode %{
14909 __ sbfm($dst$$Register, $src$$Register, 0, 31);
14910 %}
14911 ins_pipe(ialu_reg_shift);
14912 %}
14913
14914 // this pattern occurs in bigmath arithmetic
14915 instruct convUI2L_reg_reg(iRegLNoSp dst, iRegIorL2I src, immL_32bits mask)
14916 %{
14917 match(Set dst (AndL (ConvI2L src) mask));
14918
14919 ins_cost(INSN_COST);
14920 format %{ "ubfm $dst, $src, 0, 31\t# ui2l" %}
14921 ins_encode %{
14922 __ ubfm($dst$$Register, $src$$Register, 0, 31);
14923 %}
14924
14925 ins_pipe(ialu_reg_shift);
14926 %}
14927
14928 instruct convL2I_reg(iRegINoSp dst, iRegL src) %{
14929 match(Set dst (ConvL2I src));
14930
14931 ins_cost(INSN_COST);
14932 format %{ "movw $dst, $src \t// l2i" %}
14933
14934 ins_encode %{
14935 __ movw(as_Register($dst$$reg), as_Register($src$$reg));
14936 %}
14937
14938 ins_pipe(ialu_reg);
14939 %}
14940
14941 instruct convI2B(iRegINoSp dst, iRegIorL2I src, rFlagsReg cr)
14942 %{
14943 match(Set dst (Conv2B src));
14944 effect(KILL cr);
14945
14946 format %{
14947 "cmpw $src, zr\n\t"
14948 "cset $dst, ne"
14949 %}
14950
14951 ins_encode %{
14952 __ cmpw(as_Register($src$$reg), zr);
14953 __ cset(as_Register($dst$$reg), Assembler::NE);
14954 %}
14955
14956 ins_pipe(ialu_reg);
14957 %}
14958
14959 instruct convP2B(iRegINoSp dst, iRegP src, rFlagsReg cr)
14960 %{
14961 match(Set dst (Conv2B src));
14962 effect(KILL cr);
14963
14964 format %{
14965 "cmp $src, zr\n\t"
14966 "cset $dst, ne"
14967 %}
14968
14969 ins_encode %{
14970 __ cmp(as_Register($src$$reg), zr);
14971 __ cset(as_Register($dst$$reg), Assembler::NE);
14972 %}
14973
14974 ins_pipe(ialu_reg);
14975 %}
14976
14977 instruct convD2F_reg(vRegF dst, vRegD src) %{
14978 match(Set dst (ConvD2F src));
14979
14980 ins_cost(INSN_COST * 5);
14981 format %{ "fcvtd $dst, $src \t// d2f" %}
14982
14983 ins_encode %{
14984 __ fcvtd(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg));
14985 %}
14986
14987 ins_pipe(fp_d2f);
14988 %}
14989
14990 instruct convF2D_reg(vRegD dst, vRegF src) %{
14991 match(Set dst (ConvF2D src));
14992
14993 ins_cost(INSN_COST * 5);
14994 format %{ "fcvts $dst, $src \t// f2d" %}
14995
14996 ins_encode %{
14997 __ fcvts(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg));
14998 %}
14999
15000 ins_pipe(fp_f2d);
15001 %}
15002
15003 instruct convF2I_reg_reg(iRegINoSp dst, vRegF src) %{
15004 match(Set dst (ConvF2I src));
15005
15006 ins_cost(INSN_COST * 5);
15007 format %{ "fcvtzsw $dst, $src \t// f2i" %}
15008
15009 ins_encode %{
15010 __ fcvtzsw(as_Register($dst$$reg), as_FloatRegister($src$$reg));
15011 %}
15012
15013 ins_pipe(fp_f2i);
15014 %}
15015
15016 instruct convF2L_reg_reg(iRegLNoSp dst, vRegF src) %{
15017 match(Set dst (ConvF2L src));
15018
15019 ins_cost(INSN_COST * 5);
15020 format %{ "fcvtzs $dst, $src \t// f2l" %}
15021
15022 ins_encode %{
15023 __ fcvtzs(as_Register($dst$$reg), as_FloatRegister($src$$reg));
15024 %}
15025
15026 ins_pipe(fp_f2l);
15027 %}
15028
15029 instruct convI2F_reg_reg(vRegF dst, iRegIorL2I src) %{
15030 match(Set dst (ConvI2F src));
15031
15032 ins_cost(INSN_COST * 5);
15033 format %{ "scvtfws $dst, $src \t// i2f" %}
15034
15035 ins_encode %{
15036 __ scvtfws(as_FloatRegister($dst$$reg), as_Register($src$$reg));
15037 %}
15038
15039 ins_pipe(fp_i2f);
15040 %}
15041
15042 instruct convL2F_reg_reg(vRegF dst, iRegL src) %{
15043 match(Set dst (ConvL2F src));
15044
15045 ins_cost(INSN_COST * 5);
15046 format %{ "scvtfs $dst, $src \t// l2f" %}
15047
15048 ins_encode %{
15049 __ scvtfs(as_FloatRegister($dst$$reg), as_Register($src$$reg));
15050 %}
15051
15052 ins_pipe(fp_l2f);
15053 %}
15054
15055 instruct convD2I_reg_reg(iRegINoSp dst, vRegD src) %{
15056 match(Set dst (ConvD2I src));
15057
15058 ins_cost(INSN_COST * 5);
15059 format %{ "fcvtzdw $dst, $src \t// d2i" %}
15060
15061 ins_encode %{
15062 __ fcvtzdw(as_Register($dst$$reg), as_FloatRegister($src$$reg));
15063 %}
15064
15065 ins_pipe(fp_d2i);
15066 %}
15067
15068 instruct convD2L_reg_reg(iRegLNoSp dst, vRegD src) %{
15069 match(Set dst (ConvD2L src));
15070
15071 ins_cost(INSN_COST * 5);
15072 format %{ "fcvtzd $dst, $src \t// d2l" %}
15073
15074 ins_encode %{
15075 __ fcvtzd(as_Register($dst$$reg), as_FloatRegister($src$$reg));
15076 %}
15077
15078 ins_pipe(fp_d2l);
15079 %}
15080
15081 instruct convI2D_reg_reg(vRegD dst, iRegIorL2I src) %{
15082 match(Set dst (ConvI2D src));
15083
15084 ins_cost(INSN_COST * 5);
15085 format %{ "scvtfwd $dst, $src \t// i2d" %}
15086
15087 ins_encode %{
15088 __ scvtfwd(as_FloatRegister($dst$$reg), as_Register($src$$reg));
15089 %}
15090
15091 ins_pipe(fp_i2d);
15092 %}
15093
15094 instruct convL2D_reg_reg(vRegD dst, iRegL src) %{
15095 match(Set dst (ConvL2D src));
15096
15097 ins_cost(INSN_COST * 5);
15098 format %{ "scvtfd $dst, $src \t// l2d" %}
15099
15100 ins_encode %{
15101 __ scvtfd(as_FloatRegister($dst$$reg), as_Register($src$$reg));
15102 %}
15103
15104 ins_pipe(fp_l2d);
15105 %}
15106
15107 // stack <-> reg and reg <-> reg shuffles with no conversion
15108
15109 instruct MoveF2I_stack_reg(iRegINoSp dst, stackSlotF src) %{
15110
15111 match(Set dst (MoveF2I src));
15112
15113 effect(DEF dst, USE src);
15114
15115 ins_cost(4 * INSN_COST);
15116
15117 format %{ "ldrw $dst, $src\t# MoveF2I_stack_reg" %}
15118
15119 ins_encode %{
15120 __ ldrw($dst$$Register, Address(sp, $src$$disp));
15121 %}
15122
15123 ins_pipe(iload_reg_reg);
15124
15125 %}
15126
15127 instruct MoveI2F_stack_reg(vRegF dst, stackSlotI src) %{
15128
15129 match(Set dst (MoveI2F src));
15130
15131 effect(DEF dst, USE src);
15132
15133 ins_cost(4 * INSN_COST);
15134
15135 format %{ "ldrs $dst, $src\t# MoveI2F_stack_reg" %}
15136
15137 ins_encode %{
15138 __ ldrs(as_FloatRegister($dst$$reg), Address(sp, $src$$disp));
15139 %}
15140
15141 ins_pipe(pipe_class_memory);
15142
15143 %}
15144
15145 instruct MoveD2L_stack_reg(iRegLNoSp dst, stackSlotD src) %{
15146
15147 match(Set dst (MoveD2L src));
15148
15149 effect(DEF dst, USE src);
15150
15151 ins_cost(4 * INSN_COST);
15152
15153 format %{ "ldr $dst, $src\t# MoveD2L_stack_reg" %}
15154
15155 ins_encode %{
15156 __ ldr($dst$$Register, Address(sp, $src$$disp));
15157 %}
15158
15159 ins_pipe(iload_reg_reg);
15160
15161 %}
15162
15163 instruct MoveL2D_stack_reg(vRegD dst, stackSlotL src) %{
15164
15165 match(Set dst (MoveL2D src));
15166
15167 effect(DEF dst, USE src);
15168
15169 ins_cost(4 * INSN_COST);
15170
15171 format %{ "ldrd $dst, $src\t# MoveL2D_stack_reg" %}
15172
15173 ins_encode %{
15174 __ ldrd(as_FloatRegister($dst$$reg), Address(sp, $src$$disp));
15175 %}
15176
15177 ins_pipe(pipe_class_memory);
15178
15179 %}
15180
15181 instruct MoveF2I_reg_stack(stackSlotI dst, vRegF src) %{
15182
15183 match(Set dst (MoveF2I src));
15184
15185 effect(DEF dst, USE src);
15186
15187 ins_cost(INSN_COST);
15188
15189 format %{ "strs $src, $dst\t# MoveF2I_reg_stack" %}
15190
15191 ins_encode %{
15192 __ strs(as_FloatRegister($src$$reg), Address(sp, $dst$$disp));
15193 %}
15194
15195 ins_pipe(pipe_class_memory);
15196
15197 %}
15198
15199 instruct MoveI2F_reg_stack(stackSlotF dst, iRegI src) %{
15200
15201 match(Set dst (MoveI2F src));
15202
15203 effect(DEF dst, USE src);
15204
15205 ins_cost(INSN_COST);
15206
15207 format %{ "strw $src, $dst\t# MoveI2F_reg_stack" %}
15208
15209 ins_encode %{
15210 __ strw($src$$Register, Address(sp, $dst$$disp));
15211 %}
15212
15213 ins_pipe(istore_reg_reg);
15214
15215 %}
15216
15217 instruct MoveD2L_reg_stack(stackSlotL dst, vRegD src) %{
15218
15219 match(Set dst (MoveD2L src));
15220
15221 effect(DEF dst, USE src);
15222
15223 ins_cost(INSN_COST);
15224
15225 format %{ "strd $dst, $src\t# MoveD2L_reg_stack" %}
15226
15227 ins_encode %{
15228 __ strd(as_FloatRegister($src$$reg), Address(sp, $dst$$disp));
15229 %}
15230
15231 ins_pipe(pipe_class_memory);
15232
15233 %}
15234
15235 instruct MoveL2D_reg_stack(stackSlotD dst, iRegL src) %{
15236
15237 match(Set dst (MoveL2D src));
15238
15239 effect(DEF dst, USE src);
15240
15241 ins_cost(INSN_COST);
15242
15243 format %{ "str $src, $dst\t# MoveL2D_reg_stack" %}
15244
15245 ins_encode %{
15246 __ str($src$$Register, Address(sp, $dst$$disp));
15247 %}
15248
15249 ins_pipe(istore_reg_reg);
15250
15251 %}
15252
15253 instruct MoveF2I_reg_reg(iRegINoSp dst, vRegF src) %{
15254
15255 match(Set dst (MoveF2I src));
15256
15257 effect(DEF dst, USE src);
15258
15259 ins_cost(INSN_COST);
15260
15261 format %{ "fmovs $dst, $src\t# MoveF2I_reg_reg" %}
15262
15263 ins_encode %{
15264 __ fmovs($dst$$Register, as_FloatRegister($src$$reg));
15265 %}
15266
15267 ins_pipe(fp_f2i);
15268
15269 %}
15270
15271 instruct MoveI2F_reg_reg(vRegF dst, iRegI src) %{
15272
15273 match(Set dst (MoveI2F src));
15274
15275 effect(DEF dst, USE src);
15276
15277 ins_cost(INSN_COST);
15278
15279 format %{ "fmovs $dst, $src\t# MoveI2F_reg_reg" %}
15280
15281 ins_encode %{
15282 __ fmovs(as_FloatRegister($dst$$reg), $src$$Register);
15283 %}
15284
15285 ins_pipe(fp_i2f);
15286
15287 %}
15288
15289 instruct MoveD2L_reg_reg(iRegLNoSp dst, vRegD src) %{
15290
15291 match(Set dst (MoveD2L src));
15292
15293 effect(DEF dst, USE src);
15294
15295 ins_cost(INSN_COST);
15296
15297 format %{ "fmovd $dst, $src\t# MoveD2L_reg_reg" %}
15298
15299 ins_encode %{
15300 __ fmovd($dst$$Register, as_FloatRegister($src$$reg));
15301 %}
15302
15303 ins_pipe(fp_d2l);
15304
15305 %}
15306
15307 instruct MoveL2D_reg_reg(vRegD dst, iRegL src) %{
15308
15309 match(Set dst (MoveL2D src));
15310
15311 effect(DEF dst, USE src);
15312
15313 ins_cost(INSN_COST);
15314
15315 format %{ "fmovd $dst, $src\t# MoveL2D_reg_reg" %}
15316
15317 ins_encode %{
15318 __ fmovd(as_FloatRegister($dst$$reg), $src$$Register);
15319 %}
15320
15321 ins_pipe(fp_l2d);
15322
15323 %}
15324
15325 // ============================================================================
15326 // clearing of an array
15327
15328 instruct clearArray_reg_reg_immL0(iRegL_R11 cnt, iRegP_R10 base, immL0 zero, Universe dummy, rFlagsReg cr)
15329 %{
15330 match(Set dummy (ClearArray (Binary cnt base) zero));
15331 effect(USE_KILL cnt, USE_KILL base, KILL cr);
15332
15333 ins_cost(4 * INSN_COST);
15334 format %{ "ClearArray $cnt, $base" %}
15335
15336 ins_encode %{
15337 address tpc = __ zero_words($base$$Register, $cnt$$Register);
15338 if (tpc == NULL) {
15339 ciEnv::current()->record_failure("CodeCache is full");
15340 return;
15341 }
15342 %}
15343
15344 ins_pipe(pipe_class_memory);
15345 %}
15346
15347 instruct clearArray_reg_reg(iRegL_R11 cnt, iRegP_R10 base, iRegL val, Universe dummy, rFlagsReg cr)
15348 %{
15349 predicate(((ClearArrayNode*)n)->word_copy_only());
15350 match(Set dummy (ClearArray (Binary cnt base) val));
15351 effect(USE_KILL cnt, USE_KILL base, KILL cr);
15352
15353 ins_cost(4 * INSN_COST);
15354 format %{ "ClearArray $cnt, $base, $val" %}
15355
15356 ins_encode %{
15357 __ fill_words($base$$Register, $cnt$$Register, $val$$Register);
15358 %}
15359
15360 ins_pipe(pipe_class_memory);
15361 %}
15362
15363 instruct clearArray_imm_reg(immL cnt, iRegP_R10 base, iRegL_R11 temp, Universe dummy, rFlagsReg cr)
15364 %{
15365 predicate((uint64_t)n->in(2)->get_long()
15366 < (uint64_t)(BlockZeroingLowLimit >> LogBytesPerWord)
15367 && !((ClearArrayNode*)n)->word_copy_only());
15368 match(Set dummy (ClearArray cnt base));
15369 effect(TEMP temp, USE_KILL base, KILL cr);
15370
15371 ins_cost(4 * INSN_COST);
15372 format %{ "ClearArray $cnt, $base" %}
15373
15374 ins_encode %{
15375 __ zero_words($base$$Register, (uint64_t)$cnt$$constant);
15376 %}
15377
15378 ins_pipe(pipe_class_memory);
15379 %}
15380
15381 // ============================================================================
15382 // Overflow Math Instructions
15383
15384 instruct overflowAddI_reg_reg(rFlagsReg cr, iRegIorL2I op1, iRegIorL2I op2)
15385 %{
15386 match(Set cr (OverflowAddI op1 op2));
15387
15388 format %{ "cmnw $op1, $op2\t# overflow check int" %}
15389 ins_cost(INSN_COST);
15390 ins_encode %{
15391 __ cmnw($op1$$Register, $op2$$Register);
15392 %}
15393
15394 ins_pipe(icmp_reg_reg);
15395 %}
15396
15397 instruct overflowAddI_reg_imm(rFlagsReg cr, iRegIorL2I op1, immIAddSub op2)
15398 %{
15399 match(Set cr (OverflowAddI op1 op2));
15400
15401 format %{ "cmnw $op1, $op2\t# overflow check int" %}
15402 ins_cost(INSN_COST);
15403 ins_encode %{
15404 __ cmnw($op1$$Register, $op2$$constant);
15405 %}
15406
15407 ins_pipe(icmp_reg_imm);
15408 %}
15409
15410 instruct overflowAddL_reg_reg(rFlagsReg cr, iRegL op1, iRegL op2)
15411 %{
15412 match(Set cr (OverflowAddL op1 op2));
15413
15414 format %{ "cmn $op1, $op2\t# overflow check long" %}
15415 ins_cost(INSN_COST);
15416 ins_encode %{
15417 __ cmn($op1$$Register, $op2$$Register);
15418 %}
15419
15420 ins_pipe(icmp_reg_reg);
15421 %}
15422
15423 instruct overflowAddL_reg_imm(rFlagsReg cr, iRegL op1, immLAddSub op2)
15424 %{
15425 match(Set cr (OverflowAddL op1 op2));
15426
15427 format %{ "cmn $op1, $op2\t# overflow check long" %}
15428 ins_cost(INSN_COST);
15429 ins_encode %{
15430 __ cmn($op1$$Register, $op2$$constant);
15431 %}
15432
15433 ins_pipe(icmp_reg_imm);
15434 %}
15435
15436 instruct overflowSubI_reg_reg(rFlagsReg cr, iRegIorL2I op1, iRegIorL2I op2)
15437 %{
15438 match(Set cr (OverflowSubI op1 op2));
15439
15440 format %{ "cmpw $op1, $op2\t# overflow check int" %}
15441 ins_cost(INSN_COST);
15442 ins_encode %{
15443 __ cmpw($op1$$Register, $op2$$Register);
15444 %}
15445
15446 ins_pipe(icmp_reg_reg);
15447 %}
15448
15449 instruct overflowSubI_reg_imm(rFlagsReg cr, iRegIorL2I op1, immIAddSub op2)
15450 %{
15451 match(Set cr (OverflowSubI op1 op2));
15452
15453 format %{ "cmpw $op1, $op2\t# overflow check int" %}
15454 ins_cost(INSN_COST);
15455 ins_encode %{
15456 __ cmpw($op1$$Register, $op2$$constant);
15457 %}
15458
15459 ins_pipe(icmp_reg_imm);
15460 %}
15461
15462 instruct overflowSubL_reg_reg(rFlagsReg cr, iRegL op1, iRegL op2)
15463 %{
15464 match(Set cr (OverflowSubL op1 op2));
15465
15466 format %{ "cmp $op1, $op2\t# overflow check long" %}
15467 ins_cost(INSN_COST);
15468 ins_encode %{
15469 __ cmp($op1$$Register, $op2$$Register);
15470 %}
15471
15472 ins_pipe(icmp_reg_reg);
15473 %}
15474
15475 instruct overflowSubL_reg_imm(rFlagsReg cr, iRegL op1, immLAddSub op2)
15476 %{
15477 match(Set cr (OverflowSubL op1 op2));
15478
15479 format %{ "cmp $op1, $op2\t# overflow check long" %}
15480 ins_cost(INSN_COST);
15481 ins_encode %{
15482 __ subs(zr, $op1$$Register, $op2$$constant);
15483 %}
15484
15485 ins_pipe(icmp_reg_imm);
15486 %}
15487
15488 instruct overflowNegI_reg(rFlagsReg cr, immI0 zero, iRegIorL2I op1)
15489 %{
15490 match(Set cr (OverflowSubI zero op1));
15491
15492 format %{ "cmpw zr, $op1\t# overflow check int" %}
15493 ins_cost(INSN_COST);
15494 ins_encode %{
15495 __ cmpw(zr, $op1$$Register);
15496 %}
15497
15498 ins_pipe(icmp_reg_imm);
15499 %}
15500
15501 instruct overflowNegL_reg(rFlagsReg cr, immI0 zero, iRegL op1)
15502 %{
15503 match(Set cr (OverflowSubL zero op1));
15504
15505 format %{ "cmp zr, $op1\t# overflow check long" %}
15506 ins_cost(INSN_COST);
15507 ins_encode %{
15508 __ cmp(zr, $op1$$Register);
15509 %}
15510
15511 ins_pipe(icmp_reg_imm);
15512 %}
15513
15514 instruct overflowMulI_reg(rFlagsReg cr, iRegIorL2I op1, iRegIorL2I op2)
15515 %{
15516 match(Set cr (OverflowMulI op1 op2));
15517
15518 format %{ "smull rscratch1, $op1, $op2\t# overflow check int\n\t"
15519 "cmp rscratch1, rscratch1, sxtw\n\t"
15520 "movw rscratch1, #0x80000000\n\t"
15521 "cselw rscratch1, rscratch1, zr, NE\n\t"
15522 "cmpw rscratch1, #1" %}
15523 ins_cost(5 * INSN_COST);
15524 ins_encode %{
15525 __ smull(rscratch1, $op1$$Register, $op2$$Register);
15526 __ subs(zr, rscratch1, rscratch1, ext::sxtw); // NE => overflow
15527 __ movw(rscratch1, 0x80000000); // Develop 0 (EQ),
15528 __ cselw(rscratch1, rscratch1, zr, Assembler::NE); // or 0x80000000 (NE)
15529 __ cmpw(rscratch1, 1); // 0x80000000 - 1 => VS
15530 %}
15531
15532 ins_pipe(pipe_slow);
15533 %}
15534
15535 instruct overflowMulI_reg_branch(cmpOp cmp, iRegIorL2I op1, iRegIorL2I op2, label labl, rFlagsReg cr)
15536 %{
15537 match(If cmp (OverflowMulI op1 op2));
15538 predicate(n->in(1)->as_Bool()->_test._test == BoolTest::overflow
15539 || n->in(1)->as_Bool()->_test._test == BoolTest::no_overflow);
15540 effect(USE labl, KILL cr);
15541
15542 format %{ "smull rscratch1, $op1, $op2\t# overflow check int\n\t"
15543 "cmp rscratch1, rscratch1, sxtw\n\t"
15544 "b$cmp $labl" %}
15545 ins_cost(3 * INSN_COST); // Branch is rare so treat as INSN_COST
15546 ins_encode %{
15547 Label* L = $labl$$label;
15548 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15549 __ smull(rscratch1, $op1$$Register, $op2$$Register);
15550 __ subs(zr, rscratch1, rscratch1, ext::sxtw); // NE => overflow
15551 __ br(cond == Assembler::VS ? Assembler::NE : Assembler::EQ, *L);
15552 %}
15553
15554 ins_pipe(pipe_serial);
15555 %}
15556
15557 instruct overflowMulL_reg(rFlagsReg cr, iRegL op1, iRegL op2)
15558 %{
15559 match(Set cr (OverflowMulL op1 op2));
15560
15561 format %{ "mul rscratch1, $op1, $op2\t#overflow check long\n\t"
15562 "smulh rscratch2, $op1, $op2\n\t"
15563 "cmp rscratch2, rscratch1, ASR #63\n\t"
15564 "movw rscratch1, #0x80000000\n\t"
15565 "cselw rscratch1, rscratch1, zr, NE\n\t"
15566 "cmpw rscratch1, #1" %}
15567 ins_cost(6 * INSN_COST);
15568 ins_encode %{
15569 __ mul(rscratch1, $op1$$Register, $op2$$Register); // Result bits 0..63
15570 __ smulh(rscratch2, $op1$$Register, $op2$$Register); // Result bits 64..127
15571 __ cmp(rscratch2, rscratch1, Assembler::ASR, 63); // Top is pure sign ext
15572 __ movw(rscratch1, 0x80000000); // Develop 0 (EQ),
15573 __ cselw(rscratch1, rscratch1, zr, Assembler::NE); // or 0x80000000 (NE)
15574 __ cmpw(rscratch1, 1); // 0x80000000 - 1 => VS
15575 %}
15576
15577 ins_pipe(pipe_slow);
15578 %}
15579
15580 instruct overflowMulL_reg_branch(cmpOp cmp, iRegL op1, iRegL op2, label labl, rFlagsReg cr)
15581 %{
15582 match(If cmp (OverflowMulL op1 op2));
15583 predicate(n->in(1)->as_Bool()->_test._test == BoolTest::overflow
15584 || n->in(1)->as_Bool()->_test._test == BoolTest::no_overflow);
15585 effect(USE labl, KILL cr);
15586
15587 format %{ "mul rscratch1, $op1, $op2\t#overflow check long\n\t"
15588 "smulh rscratch2, $op1, $op2\n\t"
15589 "cmp rscratch2, rscratch1, ASR #63\n\t"
15590 "b$cmp $labl" %}
15591 ins_cost(4 * INSN_COST); // Branch is rare so treat as INSN_COST
15592 ins_encode %{
15593 Label* L = $labl$$label;
15594 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15595 __ mul(rscratch1, $op1$$Register, $op2$$Register); // Result bits 0..63
15596 __ smulh(rscratch2, $op1$$Register, $op2$$Register); // Result bits 64..127
15597 __ cmp(rscratch2, rscratch1, Assembler::ASR, 63); // Top is pure sign ext
15598 __ br(cond == Assembler::VS ? Assembler::NE : Assembler::EQ, *L);
15599 %}
15600
15601 ins_pipe(pipe_serial);
15602 %}
15603
15604 // ============================================================================
15605 // Compare Instructions
15606
15607 instruct compI_reg_reg(rFlagsReg cr, iRegI op1, iRegI op2)
15608 %{
15609 match(Set cr (CmpI op1 op2));
15610
15611 effect(DEF cr, USE op1, USE op2);
15612
15613 ins_cost(INSN_COST);
15614 format %{ "cmpw $op1, $op2" %}
15615
15616 ins_encode(aarch64_enc_cmpw(op1, op2));
15617
15618 ins_pipe(icmp_reg_reg);
15619 %}
15620
15621 instruct compI_reg_immI0(rFlagsReg cr, iRegI op1, immI0 zero)
15622 %{
15623 match(Set cr (CmpI op1 zero));
15624
15625 effect(DEF cr, USE op1);
15626
15627 ins_cost(INSN_COST);
15628 format %{ "cmpw $op1, 0" %}
15629
15630 ins_encode(aarch64_enc_cmpw_imm_addsub(op1, zero));
15631
15632 ins_pipe(icmp_reg_imm);
15633 %}
15634
15635 instruct compI_reg_immIAddSub(rFlagsReg cr, iRegI op1, immIAddSub op2)
15636 %{
15637 match(Set cr (CmpI op1 op2));
15638
15639 effect(DEF cr, USE op1);
15640
15641 ins_cost(INSN_COST);
15642 format %{ "cmpw $op1, $op2" %}
15643
15644 ins_encode(aarch64_enc_cmpw_imm_addsub(op1, op2));
15645
15646 ins_pipe(icmp_reg_imm);
15647 %}
15648
15649 instruct compI_reg_immI(rFlagsReg cr, iRegI op1, immI op2)
15650 %{
15651 match(Set cr (CmpI op1 op2));
15652
15653 effect(DEF cr, USE op1);
15654
15655 ins_cost(INSN_COST * 2);
15656 format %{ "cmpw $op1, $op2" %}
15657
15658 ins_encode(aarch64_enc_cmpw_imm(op1, op2));
15659
15660 ins_pipe(icmp_reg_imm);
15661 %}
15662
15663 // Unsigned compare Instructions; really, same as signed compare
15664 // except it should only be used to feed an If or a CMovI which takes a
15665 // cmpOpU.
15666
15667 instruct compU_reg_reg(rFlagsRegU cr, iRegI op1, iRegI op2)
15668 %{
15669 match(Set cr (CmpU op1 op2));
15670
15671 effect(DEF cr, USE op1, USE op2);
15672
15673 ins_cost(INSN_COST);
15674 format %{ "cmpw $op1, $op2\t# unsigned" %}
15675
15676 ins_encode(aarch64_enc_cmpw(op1, op2));
15677
15678 ins_pipe(icmp_reg_reg);
15679 %}
15680
15681 instruct compU_reg_immI0(rFlagsRegU cr, iRegI op1, immI0 zero)
15682 %{
15683 match(Set cr (CmpU op1 zero));
15684
15685 effect(DEF cr, USE op1);
15686
15687 ins_cost(INSN_COST);
15688 format %{ "cmpw $op1, #0\t# unsigned" %}
15689
15690 ins_encode(aarch64_enc_cmpw_imm_addsub(op1, zero));
15691
15692 ins_pipe(icmp_reg_imm);
15693 %}
15694
15695 instruct compU_reg_immIAddSub(rFlagsRegU cr, iRegI op1, immIAddSub op2)
15696 %{
15697 match(Set cr (CmpU op1 op2));
15698
15699 effect(DEF cr, USE op1);
15700
15701 ins_cost(INSN_COST);
15702 format %{ "cmpw $op1, $op2\t# unsigned" %}
15703
15704 ins_encode(aarch64_enc_cmpw_imm_addsub(op1, op2));
15705
15706 ins_pipe(icmp_reg_imm);
15707 %}
15708
15709 instruct compU_reg_immI(rFlagsRegU cr, iRegI op1, immI op2)
15710 %{
15711 match(Set cr (CmpU op1 op2));
15712
15713 effect(DEF cr, USE op1);
15714
15715 ins_cost(INSN_COST * 2);
15716 format %{ "cmpw $op1, $op2\t# unsigned" %}
15717
15718 ins_encode(aarch64_enc_cmpw_imm(op1, op2));
15719
15720 ins_pipe(icmp_reg_imm);
15721 %}
15722
15723 instruct compL_reg_reg(rFlagsReg cr, iRegL op1, iRegL op2)
15724 %{
15725 match(Set cr (CmpL op1 op2));
15726
15727 effect(DEF cr, USE op1, USE op2);
15728
15729 ins_cost(INSN_COST);
15730 format %{ "cmp $op1, $op2" %}
15731
15732 ins_encode(aarch64_enc_cmp(op1, op2));
15733
15734 ins_pipe(icmp_reg_reg);
15735 %}
15736
15737 instruct compL_reg_immL0(rFlagsReg cr, iRegL op1, immL0 zero)
15738 %{
15739 match(Set cr (CmpL op1 zero));
15740
15741 effect(DEF cr, USE op1);
15742
15743 ins_cost(INSN_COST);
15744 format %{ "tst $op1" %}
15745
15746 ins_encode(aarch64_enc_cmp_imm_addsub(op1, zero));
15747
15748 ins_pipe(icmp_reg_imm);
15749 %}
15750
15751 instruct compL_reg_immLAddSub(rFlagsReg cr, iRegL op1, immLAddSub op2)
15752 %{
15753 match(Set cr (CmpL op1 op2));
15754
15755 effect(DEF cr, USE op1);
15756
15757 ins_cost(INSN_COST);
15758 format %{ "cmp $op1, $op2" %}
15759
15760 ins_encode(aarch64_enc_cmp_imm_addsub(op1, op2));
15761
15762 ins_pipe(icmp_reg_imm);
15763 %}
15764
15765 instruct compL_reg_immL(rFlagsReg cr, iRegL op1, immL op2)
15766 %{
15767 match(Set cr (CmpL op1 op2));
15768
15769 effect(DEF cr, USE op1);
15770
15771 ins_cost(INSN_COST * 2);
15772 format %{ "cmp $op1, $op2" %}
15773
15774 ins_encode(aarch64_enc_cmp_imm(op1, op2));
15775
15776 ins_pipe(icmp_reg_imm);
15777 %}
15778
15779 instruct compUL_reg_reg(rFlagsRegU cr, iRegL op1, iRegL op2)
15780 %{
15781 match(Set cr (CmpUL op1 op2));
15782
15783 effect(DEF cr, USE op1, USE op2);
15784
15785 ins_cost(INSN_COST);
15786 format %{ "cmp $op1, $op2" %}
15787
15788 ins_encode(aarch64_enc_cmp(op1, op2));
15789
15790 ins_pipe(icmp_reg_reg);
15791 %}
15792
15793 instruct compUL_reg_immL0(rFlagsRegU cr, iRegL op1, immL0 zero)
15794 %{
15795 match(Set cr (CmpUL op1 zero));
15796
15797 effect(DEF cr, USE op1);
15798
15799 ins_cost(INSN_COST);
15800 format %{ "tst $op1" %}
15801
15802 ins_encode(aarch64_enc_cmp_imm_addsub(op1, zero));
15803
15804 ins_pipe(icmp_reg_imm);
15805 %}
15806
15807 instruct compUL_reg_immLAddSub(rFlagsRegU cr, iRegL op1, immLAddSub op2)
15808 %{
15809 match(Set cr (CmpUL op1 op2));
15810
15811 effect(DEF cr, USE op1);
15812
15813 ins_cost(INSN_COST);
15814 format %{ "cmp $op1, $op2" %}
15815
15816 ins_encode(aarch64_enc_cmp_imm_addsub(op1, op2));
15817
15818 ins_pipe(icmp_reg_imm);
15819 %}
15820
15821 instruct compUL_reg_immL(rFlagsRegU cr, iRegL op1, immL op2)
15822 %{
15823 match(Set cr (CmpUL op1 op2));
15824
15825 effect(DEF cr, USE op1);
15826
15827 ins_cost(INSN_COST * 2);
15828 format %{ "cmp $op1, $op2" %}
15829
15830 ins_encode(aarch64_enc_cmp_imm(op1, op2));
15831
15832 ins_pipe(icmp_reg_imm);
15833 %}
15834
15835 instruct compP_reg_reg(rFlagsRegU cr, iRegP op1, iRegP op2)
15836 %{
15837 match(Set cr (CmpP op1 op2));
15838
15839 effect(DEF cr, USE op1, USE op2);
15840
15841 ins_cost(INSN_COST);
15842 format %{ "cmp $op1, $op2\t // ptr" %}
15843
15844 ins_encode(aarch64_enc_cmpp(op1, op2));
15845
15846 ins_pipe(icmp_reg_reg);
15847 %}
15848
15849 instruct compN_reg_reg(rFlagsRegU cr, iRegN op1, iRegN op2)
15850 %{
15851 match(Set cr (CmpN op1 op2));
15852
15853 effect(DEF cr, USE op1, USE op2);
15854
15855 ins_cost(INSN_COST);
15856 format %{ "cmp $op1, $op2\t // compressed ptr" %}
15857
15858 ins_encode(aarch64_enc_cmpn(op1, op2));
15859
15860 ins_pipe(icmp_reg_reg);
15861 %}
15862
15863 instruct testP_reg(rFlagsRegU cr, iRegP op1, immP0 zero)
15864 %{
15865 match(Set cr (CmpP op1 zero));
15866
15867 effect(DEF cr, USE op1, USE zero);
15868
15869 ins_cost(INSN_COST);
15870 format %{ "cmp $op1, 0\t // ptr" %}
15871
15872 ins_encode(aarch64_enc_testp(op1));
15873
15874 ins_pipe(icmp_reg_imm);
15875 %}
15876
15877 instruct testN_reg(rFlagsRegU cr, iRegN op1, immN0 zero)
15878 %{
15879 match(Set cr (CmpN op1 zero));
15880
15881 effect(DEF cr, USE op1, USE zero);
15882
15883 ins_cost(INSN_COST);
15884 format %{ "cmp $op1, 0\t // compressed ptr" %}
15885
15886 ins_encode(aarch64_enc_testn(op1));
15887
15888 ins_pipe(icmp_reg_imm);
15889 %}
15890
15891 // FP comparisons
15892 //
15893 // n.b. CmpF/CmpD set a normal flags reg which then gets compared
15894 // using normal cmpOp. See declaration of rFlagsReg for details.
15895
15896 instruct compF_reg_reg(rFlagsReg cr, vRegF src1, vRegF src2)
15897 %{
15898 match(Set cr (CmpF src1 src2));
15899
15900 ins_cost(3 * INSN_COST);
15901 format %{ "fcmps $src1, $src2" %}
15902
15903 ins_encode %{
15904 __ fcmps(as_FloatRegister($src1$$reg), as_FloatRegister($src2$$reg));
15905 %}
15906
15907 ins_pipe(pipe_class_compare);
15908 %}
15909
15910 instruct compF_reg_zero(rFlagsReg cr, vRegF src1, immF0 src2)
15911 %{
15912 match(Set cr (CmpF src1 src2));
15913
15914 ins_cost(3 * INSN_COST);
15915 format %{ "fcmps $src1, 0.0" %}
15916
15917 ins_encode %{
15918 __ fcmps(as_FloatRegister($src1$$reg), 0.0);
15919 %}
15920
15921 ins_pipe(pipe_class_compare);
15922 %}
15923 // FROM HERE
15924
15925 instruct compD_reg_reg(rFlagsReg cr, vRegD src1, vRegD src2)
15926 %{
15927 match(Set cr (CmpD src1 src2));
15928
15929 ins_cost(3 * INSN_COST);
15930 format %{ "fcmpd $src1, $src2" %}
15931
15932 ins_encode %{
15933 __ fcmpd(as_FloatRegister($src1$$reg), as_FloatRegister($src2$$reg));
15934 %}
15935
15936 ins_pipe(pipe_class_compare);
15937 %}
15938
15939 instruct compD_reg_zero(rFlagsReg cr, vRegD src1, immD0 src2)
15940 %{
15941 match(Set cr (CmpD src1 src2));
15942
15943 ins_cost(3 * INSN_COST);
15944 format %{ "fcmpd $src1, 0.0" %}
15945
15946 ins_encode %{
15947 __ fcmpd(as_FloatRegister($src1$$reg), 0.0);
15948 %}
15949
15950 ins_pipe(pipe_class_compare);
15951 %}
15952
15953 instruct compF3_reg_reg(iRegINoSp dst, vRegF src1, vRegF src2, rFlagsReg cr)
15954 %{
15955 match(Set dst (CmpF3 src1 src2));
15956 effect(KILL cr);
15957
15958 ins_cost(5 * INSN_COST);
15959 format %{ "fcmps $src1, $src2\n\t"
15960 "csinvw($dst, zr, zr, eq\n\t"
15961 "csnegw($dst, $dst, $dst, lt)"
15962 %}
15963
15964 ins_encode %{
15965 Label done;
15966 FloatRegister s1 = as_FloatRegister($src1$$reg);
15967 FloatRegister s2 = as_FloatRegister($src2$$reg);
15968 Register d = as_Register($dst$$reg);
15969 __ fcmps(s1, s2);
15970 // installs 0 if EQ else -1
15971 __ csinvw(d, zr, zr, Assembler::EQ);
15972 // keeps -1 if less or unordered else installs 1
15973 __ csnegw(d, d, d, Assembler::LT);
15974 __ bind(done);
15975 %}
15976
15977 ins_pipe(pipe_class_default);
15978
15979 %}
15980
15981 instruct compD3_reg_reg(iRegINoSp dst, vRegD src1, vRegD src2, rFlagsReg cr)
15982 %{
15983 match(Set dst (CmpD3 src1 src2));
15984 effect(KILL cr);
15985
15986 ins_cost(5 * INSN_COST);
15987 format %{ "fcmpd $src1, $src2\n\t"
15988 "csinvw($dst, zr, zr, eq\n\t"
15989 "csnegw($dst, $dst, $dst, lt)"
15990 %}
15991
15992 ins_encode %{
15993 Label done;
15994 FloatRegister s1 = as_FloatRegister($src1$$reg);
15995 FloatRegister s2 = as_FloatRegister($src2$$reg);
15996 Register d = as_Register($dst$$reg);
15997 __ fcmpd(s1, s2);
15998 // installs 0 if EQ else -1
15999 __ csinvw(d, zr, zr, Assembler::EQ);
16000 // keeps -1 if less or unordered else installs 1
16001 __ csnegw(d, d, d, Assembler::LT);
16002 __ bind(done);
16003 %}
16004 ins_pipe(pipe_class_default);
16005
16006 %}
16007
16008 instruct compF3_reg_immF0(iRegINoSp dst, vRegF src1, immF0 zero, rFlagsReg cr)
16009 %{
16010 match(Set dst (CmpF3 src1 zero));
16011 effect(KILL cr);
16012
16013 ins_cost(5 * INSN_COST);
16014 format %{ "fcmps $src1, 0.0\n\t"
16015 "csinvw($dst, zr, zr, eq\n\t"
16016 "csnegw($dst, $dst, $dst, lt)"
16017 %}
16018
16019 ins_encode %{
16020 Label done;
16021 FloatRegister s1 = as_FloatRegister($src1$$reg);
16022 Register d = as_Register($dst$$reg);
16023 __ fcmps(s1, 0.0);
16024 // installs 0 if EQ else -1
16025 __ csinvw(d, zr, zr, Assembler::EQ);
16026 // keeps -1 if less or unordered else installs 1
16027 __ csnegw(d, d, d, Assembler::LT);
16028 __ bind(done);
16029 %}
16030
16031 ins_pipe(pipe_class_default);
16032
16033 %}
16034
16035 instruct compD3_reg_immD0(iRegINoSp dst, vRegD src1, immD0 zero, rFlagsReg cr)
16036 %{
16037 match(Set dst (CmpD3 src1 zero));
16038 effect(KILL cr);
16039
16040 ins_cost(5 * INSN_COST);
16041 format %{ "fcmpd $src1, 0.0\n\t"
16042 "csinvw($dst, zr, zr, eq\n\t"
16043 "csnegw($dst, $dst, $dst, lt)"
16044 %}
16045
16046 ins_encode %{
16047 Label done;
16048 FloatRegister s1 = as_FloatRegister($src1$$reg);
16049 Register d = as_Register($dst$$reg);
16050 __ fcmpd(s1, 0.0);
16051 // installs 0 if EQ else -1
16052 __ csinvw(d, zr, zr, Assembler::EQ);
16053 // keeps -1 if less or unordered else installs 1
16054 __ csnegw(d, d, d, Assembler::LT);
16055 __ bind(done);
16056 %}
16057 ins_pipe(pipe_class_default);
16058
16059 %}
16060
16061 instruct cmpLTMask_reg_reg(iRegINoSp dst, iRegIorL2I p, iRegIorL2I q, rFlagsReg cr)
16062 %{
16063 match(Set dst (CmpLTMask p q));
16064 effect(KILL cr);
16065
16066 ins_cost(3 * INSN_COST);
16067
16068 format %{ "cmpw $p, $q\t# cmpLTMask\n\t"
16069 "csetw $dst, lt\n\t"
16070 "subw $dst, zr, $dst"
16071 %}
16072
16073 ins_encode %{
16074 __ cmpw(as_Register($p$$reg), as_Register($q$$reg));
16075 __ csetw(as_Register($dst$$reg), Assembler::LT);
16076 __ subw(as_Register($dst$$reg), zr, as_Register($dst$$reg));
16077 %}
16078
16079 ins_pipe(ialu_reg_reg);
16080 %}
16081
16082 instruct cmpLTMask_reg_zero(iRegINoSp dst, iRegIorL2I src, immI0 zero, rFlagsReg cr)
16083 %{
16084 match(Set dst (CmpLTMask src zero));
16085 effect(KILL cr);
16086
16087 ins_cost(INSN_COST);
16088
16089 format %{ "asrw $dst, $src, #31\t# cmpLTMask0" %}
16090
16091 ins_encode %{
16092 __ asrw(as_Register($dst$$reg), as_Register($src$$reg), 31);
16093 %}
16094
16095 ins_pipe(ialu_reg_shift);
16096 %}
16097
16098 // ============================================================================
16099 // Max and Min
16100
16101 instruct cmovI_reg_reg_lt(iRegINoSp dst, iRegI src1, iRegI src2, rFlagsReg cr)
16102 %{
16103 effect( DEF dst, USE src1, USE src2, USE cr );
16104
16105 ins_cost(INSN_COST * 2);
16106 format %{ "cselw $dst, $src1, $src2 lt\t" %}
16107
16108 ins_encode %{
16109 __ cselw(as_Register($dst$$reg),
16110 as_Register($src1$$reg),
16111 as_Register($src2$$reg),
16112 Assembler::LT);
16113 %}
16114
16115 ins_pipe(icond_reg_reg);
16116 %}
16117
16118 instruct minI_rReg(iRegINoSp dst, iRegI src1, iRegI src2)
16119 %{
16120 match(Set dst (MinI src1 src2));
16121 ins_cost(INSN_COST * 3);
16122
16123 expand %{
16124 rFlagsReg cr;
16125 compI_reg_reg(cr, src1, src2);
16126 cmovI_reg_reg_lt(dst, src1, src2, cr);
16127 %}
16128
16129 %}
16130 // FROM HERE
16131
16132 instruct cmovI_reg_reg_gt(iRegINoSp dst, iRegI src1, iRegI src2, rFlagsReg cr)
16133 %{
16134 effect( DEF dst, USE src1, USE src2, USE cr );
16135
16136 ins_cost(INSN_COST * 2);
16137 format %{ "cselw $dst, $src1, $src2 gt\t" %}
16138
16139 ins_encode %{
16140 __ cselw(as_Register($dst$$reg),
16141 as_Register($src1$$reg),
16142 as_Register($src2$$reg),
16143 Assembler::GT);
16144 %}
16145
16146 ins_pipe(icond_reg_reg);
16147 %}
16148
16149 instruct maxI_rReg(iRegINoSp dst, iRegI src1, iRegI src2)
16150 %{
16151 match(Set dst (MaxI src1 src2));
16152 ins_cost(INSN_COST * 3);
16153 expand %{
16154 rFlagsReg cr;
16155 compI_reg_reg(cr, src1, src2);
16156 cmovI_reg_reg_gt(dst, src1, src2, cr);
16157 %}
16158 %}
16159
16160 // ============================================================================
16161 // Branch Instructions
16162
16163 // Direct Branch.
16164 instruct branch(label lbl)
16165 %{
16166 match(Goto);
16167
16168 effect(USE lbl);
16169
16170 ins_cost(BRANCH_COST);
16171 format %{ "b $lbl" %}
16172
16173 ins_encode(aarch64_enc_b(lbl));
16174
16175 ins_pipe(pipe_branch);
16176 %}
16177
16178 // Conditional Near Branch
16179 instruct branchCon(cmpOp cmp, rFlagsReg cr, label lbl)
16180 %{
16181 // Same match rule as `branchConFar'.
16182 match(If cmp cr);
16183
16184 effect(USE lbl);
16185
16186 ins_cost(BRANCH_COST);
16187 // If set to 1 this indicates that the current instruction is a
16188 // short variant of a long branch. This avoids using this
16189 // instruction in first-pass matching. It will then only be used in
16190 // the `Shorten_branches' pass.
16191 // ins_short_branch(1);
16192 format %{ "b$cmp $lbl" %}
16193
16194 ins_encode(aarch64_enc_br_con(cmp, lbl));
16195
16196 ins_pipe(pipe_branch_cond);
16197 %}
16198
16199 // Conditional Near Branch Unsigned
16200 instruct branchConU(cmpOpU cmp, rFlagsRegU cr, label lbl)
16201 %{
16202 // Same match rule as `branchConFar'.
16203 match(If cmp cr);
16204
16205 effect(USE lbl);
16206
16207 ins_cost(BRANCH_COST);
16208 // If set to 1 this indicates that the current instruction is a
16209 // short variant of a long branch. This avoids using this
16210 // instruction in first-pass matching. It will then only be used in
16211 // the `Shorten_branches' pass.
16212 // ins_short_branch(1);
16213 format %{ "b$cmp $lbl\t# unsigned" %}
16214
16215 ins_encode(aarch64_enc_br_conU(cmp, lbl));
16216
16217 ins_pipe(pipe_branch_cond);
16218 %}
16219
16220 // Make use of CBZ and CBNZ. These instructions, as well as being
16221 // shorter than (cmp; branch), have the additional benefit of not
16222 // killing the flags.
16223
16224 instruct cmpI_imm0_branch(cmpOpEqNe cmp, iRegIorL2I op1, immI0 op2, label labl, rFlagsReg cr) %{
16225 match(If cmp (CmpI op1 op2));
16226 effect(USE labl);
16227
16228 ins_cost(BRANCH_COST);
16229 format %{ "cbw$cmp $op1, $labl" %}
16230 ins_encode %{
16231 Label* L = $labl$$label;
16232 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
16233 if (cond == Assembler::EQ)
16234 __ cbzw($op1$$Register, *L);
16235 else
16236 __ cbnzw($op1$$Register, *L);
16237 %}
16238 ins_pipe(pipe_cmp_branch);
16239 %}
16240
16241 instruct cmpL_imm0_branch(cmpOpEqNe cmp, iRegL op1, immL0 op2, label labl, rFlagsReg cr) %{
16242 match(If cmp (CmpL op1 op2));
16243 effect(USE labl);
16244
16245 ins_cost(BRANCH_COST);
16246 format %{ "cb$cmp $op1, $labl" %}
16247 ins_encode %{
16248 Label* L = $labl$$label;
16249 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
16250 if (cond == Assembler::EQ)
16251 __ cbz($op1$$Register, *L);
16252 else
16253 __ cbnz($op1$$Register, *L);
16254 %}
16255 ins_pipe(pipe_cmp_branch);
16256 %}
16257
16258 instruct cmpP_imm0_branch(cmpOpEqNe cmp, iRegP op1, immP0 op2, label labl, rFlagsReg cr) %{
16259 match(If cmp (CmpP op1 op2));
16260 effect(USE labl);
16261
16262 ins_cost(BRANCH_COST);
16263 format %{ "cb$cmp $op1, $labl" %}
16264 ins_encode %{
16265 Label* L = $labl$$label;
16266 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
16267 if (cond == Assembler::EQ)
16268 __ cbz($op1$$Register, *L);
16269 else
16270 __ cbnz($op1$$Register, *L);
16271 %}
16272 ins_pipe(pipe_cmp_branch);
16273 %}
16274
16275 instruct cmpN_imm0_branch(cmpOpEqNe cmp, iRegN op1, immN0 op2, label labl, rFlagsReg cr) %{
16276 match(If cmp (CmpN op1 op2));
16277 effect(USE labl);
16278
16279 ins_cost(BRANCH_COST);
16280 format %{ "cbw$cmp $op1, $labl" %}
16281 ins_encode %{
16282 Label* L = $labl$$label;
16283 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
16284 if (cond == Assembler::EQ)
16285 __ cbzw($op1$$Register, *L);
16286 else
16287 __ cbnzw($op1$$Register, *L);
16288 %}
16289 ins_pipe(pipe_cmp_branch);
16290 %}
16291
16292 instruct cmpP_narrowOop_imm0_branch(cmpOpEqNe cmp, iRegN oop, immP0 zero, label labl, rFlagsReg cr) %{
16293 match(If cmp (CmpP (DecodeN oop) zero));
16294 effect(USE labl);
16295
16296 ins_cost(BRANCH_COST);
16297 format %{ "cb$cmp $oop, $labl" %}
16298 ins_encode %{
16299 Label* L = $labl$$label;
16300 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
16301 if (cond == Assembler::EQ)
16302 __ cbzw($oop$$Register, *L);
16303 else
16304 __ cbnzw($oop$$Register, *L);
16305 %}
16306 ins_pipe(pipe_cmp_branch);
16307 %}
16308
16309 instruct cmpUI_imm0_branch(cmpOpUEqNeLtGe cmp, iRegIorL2I op1, immI0 op2, label labl, rFlagsRegU cr) %{
16310 match(If cmp (CmpU op1 op2));
16311 effect(USE labl);
16312
16313 ins_cost(BRANCH_COST);
16314 format %{ "cbw$cmp $op1, $labl" %}
16315 ins_encode %{
16316 Label* L = $labl$$label;
16317 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
16318 if (cond == Assembler::EQ || cond == Assembler::LS)
16319 __ cbzw($op1$$Register, *L);
16320 else
16321 __ cbnzw($op1$$Register, *L);
16322 %}
16323 ins_pipe(pipe_cmp_branch);
16324 %}
16325
16326 instruct cmpUL_imm0_branch(cmpOpUEqNeLtGe cmp, iRegL op1, immL0 op2, label labl, rFlagsRegU cr) %{
16327 match(If cmp (CmpUL op1 op2));
16328 effect(USE labl);
16329
16330 ins_cost(BRANCH_COST);
16331 format %{ "cb$cmp $op1, $labl" %}
16332 ins_encode %{
16333 Label* L = $labl$$label;
16334 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
16335 if (cond == Assembler::EQ || cond == Assembler::LS)
16336 __ cbz($op1$$Register, *L);
16337 else
16338 __ cbnz($op1$$Register, *L);
16339 %}
16340 ins_pipe(pipe_cmp_branch);
16341 %}
16342
16343 // Test bit and Branch
16344
16345 // Patterns for short (< 32KiB) variants
16346 instruct cmpL_branch_sign(cmpOpLtGe cmp, iRegL op1, immL0 op2, label labl) %{
16347 match(If cmp (CmpL op1 op2));
16348 effect(USE labl);
16349
16350 ins_cost(BRANCH_COST);
16351 format %{ "cb$cmp $op1, $labl # long" %}
16352 ins_encode %{
16353 Label* L = $labl$$label;
16354 Assembler::Condition cond =
16355 ((Assembler::Condition)$cmp$$cmpcode == Assembler::LT) ? Assembler::NE : Assembler::EQ;
16356 __ tbr(cond, $op1$$Register, 63, *L);
16357 %}
16358 ins_pipe(pipe_cmp_branch);
16359 ins_short_branch(1);
16360 %}
16361
16362 instruct cmpI_branch_sign(cmpOpLtGe cmp, iRegIorL2I op1, immI0 op2, label labl) %{
16363 match(If cmp (CmpI op1 op2));
16364 effect(USE labl);
16365
16366 ins_cost(BRANCH_COST);
16367 format %{ "cb$cmp $op1, $labl # int" %}
16368 ins_encode %{
16369 Label* L = $labl$$label;
16370 Assembler::Condition cond =
16371 ((Assembler::Condition)$cmp$$cmpcode == Assembler::LT) ? Assembler::NE : Assembler::EQ;
16372 __ tbr(cond, $op1$$Register, 31, *L);
16373 %}
16374 ins_pipe(pipe_cmp_branch);
16375 ins_short_branch(1);
16376 %}
16377
16378 instruct cmpL_branch_bit(cmpOpEqNe cmp, iRegL op1, immL op2, immL0 op3, label labl) %{
16379 match(If cmp (CmpL (AndL op1 op2) op3));
16380 predicate(is_power_of_2((julong)n->in(2)->in(1)->in(2)->get_long()));
16381 effect(USE labl);
16382
16383 ins_cost(BRANCH_COST);
16384 format %{ "tb$cmp $op1, $op2, $labl" %}
16385 ins_encode %{
16386 Label* L = $labl$$label;
16387 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
16388 int bit = exact_log2_long($op2$$constant);
16389 __ tbr(cond, $op1$$Register, bit, *L);
16390 %}
16391 ins_pipe(pipe_cmp_branch);
16392 ins_short_branch(1);
16393 %}
16394
16395 instruct cmpI_branch_bit(cmpOpEqNe cmp, iRegIorL2I op1, immI op2, immI0 op3, label labl) %{
16396 match(If cmp (CmpI (AndI op1 op2) op3));
16397 predicate(is_power_of_2((juint)n->in(2)->in(1)->in(2)->get_int()));
16398 effect(USE labl);
16399
16400 ins_cost(BRANCH_COST);
16401 format %{ "tb$cmp $op1, $op2, $labl" %}
16402 ins_encode %{
16403 Label* L = $labl$$label;
16404 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
16405 int bit = exact_log2((juint)$op2$$constant);
16406 __ tbr(cond, $op1$$Register, bit, *L);
16407 %}
16408 ins_pipe(pipe_cmp_branch);
16409 ins_short_branch(1);
16410 %}
16411
16412 // And far variants
16413 instruct far_cmpL_branch_sign(cmpOpLtGe cmp, iRegL op1, immL0 op2, label labl) %{
16414 match(If cmp (CmpL op1 op2));
16415 effect(USE labl);
16416
16417 ins_cost(BRANCH_COST);
16418 format %{ "cb$cmp $op1, $labl # long" %}
16419 ins_encode %{
16420 Label* L = $labl$$label;
16421 Assembler::Condition cond =
16422 ((Assembler::Condition)$cmp$$cmpcode == Assembler::LT) ? Assembler::NE : Assembler::EQ;
16423 __ tbr(cond, $op1$$Register, 63, *L, /*far*/true);
16424 %}
16425 ins_pipe(pipe_cmp_branch);
16426 %}
16427
16428 instruct far_cmpI_branch_sign(cmpOpLtGe cmp, iRegIorL2I op1, immI0 op2, label labl) %{
16429 match(If cmp (CmpI op1 op2));
16430 effect(USE labl);
16431
16432 ins_cost(BRANCH_COST);
16433 format %{ "cb$cmp $op1, $labl # int" %}
16434 ins_encode %{
16435 Label* L = $labl$$label;
16436 Assembler::Condition cond =
16437 ((Assembler::Condition)$cmp$$cmpcode == Assembler::LT) ? Assembler::NE : Assembler::EQ;
16438 __ tbr(cond, $op1$$Register, 31, *L, /*far*/true);
16439 %}
16440 ins_pipe(pipe_cmp_branch);
16441 %}
16442
16443 instruct far_cmpL_branch_bit(cmpOpEqNe cmp, iRegL op1, immL op2, immL0 op3, label labl) %{
16444 match(If cmp (CmpL (AndL op1 op2) op3));
16445 predicate(is_power_of_2((julong)n->in(2)->in(1)->in(2)->get_long()));
16446 effect(USE labl);
16447
16448 ins_cost(BRANCH_COST);
16449 format %{ "tb$cmp $op1, $op2, $labl" %}
16450 ins_encode %{
16451 Label* L = $labl$$label;
16452 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
16453 int bit = exact_log2_long($op2$$constant);
16454 __ tbr(cond, $op1$$Register, bit, *L, /*far*/true);
16455 %}
16456 ins_pipe(pipe_cmp_branch);
16457 %}
16458
16459 instruct far_cmpI_branch_bit(cmpOpEqNe cmp, iRegIorL2I op1, immI op2, immI0 op3, label labl) %{
16460 match(If cmp (CmpI (AndI op1 op2) op3));
16461 predicate(is_power_of_2((juint)n->in(2)->in(1)->in(2)->get_int()));
16462 effect(USE labl);
16463
16464 ins_cost(BRANCH_COST);
16465 format %{ "tb$cmp $op1, $op2, $labl" %}
16466 ins_encode %{
16467 Label* L = $labl$$label;
16468 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
16469 int bit = exact_log2((juint)$op2$$constant);
16470 __ tbr(cond, $op1$$Register, bit, *L, /*far*/true);
16471 %}
16472 ins_pipe(pipe_cmp_branch);
16473 %}
16474
16475 // Test bits
16476
16477 instruct cmpL_and(cmpOp cmp, iRegL op1, immL op2, immL0 op3, rFlagsReg cr) %{
16478 match(Set cr (CmpL (AndL op1 op2) op3));
16479 predicate(Assembler::operand_valid_for_logical_immediate
16480 (/*is_32*/false, n->in(1)->in(2)->get_long()));
16481
16482 ins_cost(INSN_COST);
16483 format %{ "tst $op1, $op2 # long" %}
16484 ins_encode %{
16485 __ tst($op1$$Register, $op2$$constant);
16486 %}
16487 ins_pipe(ialu_reg_reg);
16488 %}
16489
16490 instruct cmpI_and(cmpOp cmp, iRegIorL2I op1, immI op2, immI0 op3, rFlagsReg cr) %{
16491 match(Set cr (CmpI (AndI op1 op2) op3));
16492 predicate(Assembler::operand_valid_for_logical_immediate
16493 (/*is_32*/true, n->in(1)->in(2)->get_int()));
16494
16495 ins_cost(INSN_COST);
16496 format %{ "tst $op1, $op2 # int" %}
16497 ins_encode %{
16498 __ tstw($op1$$Register, $op2$$constant);
16499 %}
16500 ins_pipe(ialu_reg_reg);
16501 %}
16502
16503 instruct cmpL_and_reg(cmpOp cmp, iRegL op1, iRegL op2, immL0 op3, rFlagsReg cr) %{
16504 match(Set cr (CmpL (AndL op1 op2) op3));
16505
16506 ins_cost(INSN_COST);
16507 format %{ "tst $op1, $op2 # long" %}
16508 ins_encode %{
16509 __ tst($op1$$Register, $op2$$Register);
16510 %}
16511 ins_pipe(ialu_reg_reg);
16512 %}
16513
16514 instruct cmpI_and_reg(cmpOp cmp, iRegIorL2I op1, iRegIorL2I op2, immI0 op3, rFlagsReg cr) %{
16515 match(Set cr (CmpI (AndI op1 op2) op3));
16516
16517 ins_cost(INSN_COST);
16518 format %{ "tstw $op1, $op2 # int" %}
16519 ins_encode %{
16520 __ tstw($op1$$Register, $op2$$Register);
16521 %}
16522 ins_pipe(ialu_reg_reg);
16523 %}
16524
16525
16526 // Conditional Far Branch
16527 // Conditional Far Branch Unsigned
16528 // TODO: fixme
16529
16530 // counted loop end branch near
16531 instruct branchLoopEnd(cmpOp cmp, rFlagsReg cr, label lbl)
16532 %{
16533 match(CountedLoopEnd cmp cr);
16534
16535 effect(USE lbl);
16536
16537 ins_cost(BRANCH_COST);
16538 // short variant.
16539 // ins_short_branch(1);
16540 format %{ "b$cmp $lbl \t// counted loop end" %}
16541
16542 ins_encode(aarch64_enc_br_con(cmp, lbl));
16543
16544 ins_pipe(pipe_branch);
16545 %}
16546
16547 // counted loop end branch near Unsigned
16548 instruct branchLoopEndU(cmpOpU cmp, rFlagsRegU cr, label lbl)
16549 %{
16550 match(CountedLoopEnd cmp cr);
16551
16552 effect(USE lbl);
16553
16554 ins_cost(BRANCH_COST);
16555 // short variant.
16556 // ins_short_branch(1);
16557 format %{ "b$cmp $lbl \t// counted loop end unsigned" %}
16558
16559 ins_encode(aarch64_enc_br_conU(cmp, lbl));
16560
16561 ins_pipe(pipe_branch);
16562 %}
16563
16564 // counted loop end branch far
16565 // counted loop end branch far unsigned
16566 // TODO: fixme
16567
16568 // ============================================================================
16569 // inlined locking and unlocking
16570
16571 instruct cmpFastLock(rFlagsReg cr, iRegP object, iRegP box, iRegPNoSp tmp, iRegPNoSp tmp2)
16572 %{
16573 match(Set cr (FastLock object box));
16574 effect(TEMP tmp, TEMP tmp2);
16575
16576 // TODO
16577 // identify correct cost
16578 ins_cost(5 * INSN_COST);
16579 format %{ "fastlock $object,$box\t! kills $tmp,$tmp2" %}
16580
16581 ins_encode(aarch64_enc_fast_lock(object, box, tmp, tmp2));
16582
16583 ins_pipe(pipe_serial);
16584 %}
16585
16586 instruct cmpFastUnlock(rFlagsReg cr, iRegP object, iRegP box, iRegPNoSp tmp, iRegPNoSp tmp2)
16587 %{
16588 match(Set cr (FastUnlock object box));
16589 effect(TEMP tmp, TEMP tmp2);
16590
16591 ins_cost(5 * INSN_COST);
16592 format %{ "fastunlock $object,$box\t! kills $tmp, $tmp2" %}
16593
16594 ins_encode(aarch64_enc_fast_unlock(object, box, tmp, tmp2));
16595
16596 ins_pipe(pipe_serial);
16597 %}
16598
16599
16600 // ============================================================================
16601 // Safepoint Instructions
16602
16603 // TODO
16604 // provide a near and far version of this code
16605
16606 instruct safePoint(rFlagsReg cr, iRegP poll)
16607 %{
16608 match(SafePoint poll);
16609 effect(KILL cr);
16610
16611 format %{
16612 "ldrw zr, [$poll]\t# Safepoint: poll for GC"
16613 %}
16614 ins_encode %{
16615 __ read_polling_page(as_Register($poll$$reg), relocInfo::poll_type);
16616 %}
16617 ins_pipe(pipe_serial); // ins_pipe(iload_reg_mem);
16618 %}
16619
16620
16621 // ============================================================================
16622 // Procedure Call/Return Instructions
16623
16624 // Call Java Static Instruction
16625
16626 instruct CallStaticJavaDirect(method meth)
16627 %{
16628 match(CallStaticJava);
16629
16630 effect(USE meth);
16631
16632 ins_cost(CALL_COST);
16633
16634 format %{ "call,static $meth \t// ==> " %}
16635
16636 ins_encode(aarch64_enc_java_static_call(meth),
16637 aarch64_enc_call_epilog);
16638
16639 ins_pipe(pipe_class_call);
16640 %}
16641
16642 // TO HERE
16643
16644 // Call Java Dynamic Instruction
16645 instruct CallDynamicJavaDirect(method meth)
16646 %{
16647 match(CallDynamicJava);
16648
16649 effect(USE meth);
16650
16651 ins_cost(CALL_COST);
16652
16653 format %{ "CALL,dynamic $meth \t// ==> " %}
16654
16655 ins_encode(aarch64_enc_java_dynamic_call(meth),
16656 aarch64_enc_call_epilog);
16657
16658 ins_pipe(pipe_class_call);
16659 %}
16660
16661 // Call Runtime Instruction
16662
16663 instruct CallRuntimeDirect(method meth)
16664 %{
16665 match(CallRuntime);
16666
16667 effect(USE meth);
16668
16669 ins_cost(CALL_COST);
16670
16671 format %{ "CALL, runtime $meth" %}
16672
16673 ins_encode( aarch64_enc_java_to_runtime(meth) );
16674
16675 ins_pipe(pipe_class_call);
16676 %}
16677
16678 // Call Runtime Instruction
16679
16680 instruct CallLeafDirect(method meth)
16681 %{
16682 match(CallLeaf);
16683
16684 effect(USE meth);
16685
16686 ins_cost(CALL_COST);
16687
16688 format %{ "CALL, runtime leaf $meth" %}
16689
16690 ins_encode( aarch64_enc_java_to_runtime(meth) );
16691
16692 ins_pipe(pipe_class_call);
16693 %}
16694
16695 // Call Runtime Instruction
16696
16697 // entry point is null, target holds the address to call
16698 instruct CallLeafNoFPIndirect(iRegP target)
16699 %{
16700 predicate(n->as_Call()->entry_point() == NULL);
16701
16702 match(CallLeafNoFP target);
16703
16704 ins_cost(CALL_COST);
16705
16706 format %{ "CALL, runtime leaf nofp indirect $target" %}
16707
16708 ins_encode %{
16709 __ blr($target$$Register);
16710 %}
16711
16712 ins_pipe(pipe_class_call);
16713 %}
16714
16715 instruct CallLeafNoFPDirect(method meth)
16716 %{
16717 predicate(n->as_Call()->entry_point() != NULL);
16718
16719 match(CallLeafNoFP);
16720
16721 effect(USE meth);
16722
16723 ins_cost(CALL_COST);
16724
16725 format %{ "CALL, runtime leaf nofp $meth" %}
16726
16727 ins_encode( aarch64_enc_java_to_runtime(meth) );
16728
16729 ins_pipe(pipe_class_call);
16730 %}
16731
16732 instruct CallNativeDirect(method meth)
16733 %{
16734 match(CallNative);
16735
16736 effect(USE meth);
16737
16738 ins_cost(CALL_COST);
16739
16740 format %{ "CALL, native $meth" %}
16741
16742 ins_encode( aarch64_enc_java_to_runtime(meth) );
16743
16744 ins_pipe(pipe_class_call);
16745 %}
16746
16747 // Tail Call; Jump from runtime stub to Java code.
16748 // Also known as an 'interprocedural jump'.
16749 // Target of jump will eventually return to caller.
16750 // TailJump below removes the return address.
16751 instruct TailCalljmpInd(iRegPNoSp jump_target, inline_cache_RegP method_ptr)
16752 %{
16753 match(TailCall jump_target method_ptr);
16754
16755 ins_cost(CALL_COST);
16756
16757 format %{ "br $jump_target\t# $method_ptr holds method" %}
16758
16759 ins_encode(aarch64_enc_tail_call(jump_target));
16760
16761 ins_pipe(pipe_class_call);
16762 %}
16763
16764 instruct TailjmpInd(iRegPNoSp jump_target, iRegP_R0 ex_oop)
16765 %{
16766 match(TailJump jump_target ex_oop);
16767
16768 ins_cost(CALL_COST);
16769
16770 format %{ "br $jump_target\t# $ex_oop holds exception oop" %}
16771
16772 ins_encode(aarch64_enc_tail_jmp(jump_target));
16773
16774 ins_pipe(pipe_class_call);
16775 %}
16776
16777 // Create exception oop: created by stack-crawling runtime code.
16778 // Created exception is now available to this handler, and is setup
16779 // just prior to jumping to this handler. No code emitted.
16780 // TODO check
16781 // should ex_oop be in r0? intel uses rax, ppc cannot use r0 so uses rarg1
16782 instruct CreateException(iRegP_R0 ex_oop)
16783 %{
16784 match(Set ex_oop (CreateEx));
16785
16786 format %{ " -- \t// exception oop; no code emitted" %}
16787
16788 size(0);
16789
16790 ins_encode( /*empty*/ );
16791
16792 ins_pipe(pipe_class_empty);
16793 %}
16794
16795 // Rethrow exception: The exception oop will come in the first
16796 // argument position. Then JUMP (not call) to the rethrow stub code.
16797 instruct RethrowException() %{
16798 match(Rethrow);
16799 ins_cost(CALL_COST);
16800
16801 format %{ "b rethrow_stub" %}
16802
16803 ins_encode( aarch64_enc_rethrow() );
16804
16805 ins_pipe(pipe_class_call);
16806 %}
16807
16808
16809 // Return Instruction
16810 // epilog node loads ret address into lr as part of frame pop
16811 instruct Ret()
16812 %{
16813 match(Return);
16814
16815 format %{ "ret\t// return register" %}
16816
16817 ins_encode( aarch64_enc_ret() );
16818
16819 ins_pipe(pipe_branch);
16820 %}
16821
16822 // Die now.
16823 instruct ShouldNotReachHere() %{
16824 match(Halt);
16825
16826 ins_cost(CALL_COST);
16827 format %{ "ShouldNotReachHere" %}
16828
16829 ins_encode %{
16830 if (is_reachable()) {
16831 __ stop(_halt_reason);
16832 }
16833 %}
16834
16835 ins_pipe(pipe_class_default);
16836 %}
16837
16838 // ============================================================================
16839 // Partial Subtype Check
16840 //
16841 // superklass array for an instance of the superklass. Set a hidden
16842 // internal cache on a hit (cache is checked with exposed code in
16843 // gen_subtype_check()). Return NZ for a miss or zero for a hit. The
16844 // encoding ALSO sets flags.
16845
16846 instruct partialSubtypeCheck(iRegP_R4 sub, iRegP_R0 super, iRegP_R2 temp, iRegP_R5 result, rFlagsReg cr)
16847 %{
16848 match(Set result (PartialSubtypeCheck sub super));
16849 effect(KILL cr, KILL temp);
16850
16851 ins_cost(1100); // slightly larger than the next version
16852 format %{ "partialSubtypeCheck $result, $sub, $super" %}
16853
16854 ins_encode(aarch64_enc_partial_subtype_check(sub, super, temp, result));
16855
16856 opcode(0x1); // Force zero of result reg on hit
16857
16858 ins_pipe(pipe_class_memory);
16859 %}
16860
16861 instruct partialSubtypeCheckVsZero(iRegP_R4 sub, iRegP_R0 super, iRegP_R2 temp, iRegP_R5 result, immP0 zero, rFlagsReg cr)
16862 %{
16863 match(Set cr (CmpP (PartialSubtypeCheck sub super) zero));
16864 effect(KILL temp, KILL result);
16865
16866 ins_cost(1100); // slightly larger than the next version
16867 format %{ "partialSubtypeCheck $result, $sub, $super == 0" %}
16868
16869 ins_encode(aarch64_enc_partial_subtype_check(sub, super, temp, result));
16870
16871 opcode(0x0); // Don't zero result reg on hit
16872
16873 ins_pipe(pipe_class_memory);
16874 %}
16875
16876 instruct string_compareU(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
16877 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2, rFlagsReg cr)
16878 %{
16879 predicate(((StrCompNode*)n)->encoding() == StrIntrinsicNode::UU);
16880 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
16881 effect(KILL tmp1, KILL tmp2, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
16882
16883 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # KILL $tmp1" %}
16884 ins_encode %{
16885 // Count is in 8-bit bytes; non-Compact chars are 16 bits.
16886 __ string_compare($str1$$Register, $str2$$Register,
16887 $cnt1$$Register, $cnt2$$Register, $result$$Register,
16888 $tmp1$$Register, $tmp2$$Register,
16889 fnoreg, fnoreg, fnoreg, StrIntrinsicNode::UU);
16890 %}
16891 ins_pipe(pipe_class_memory);
16892 %}
16893
16894 instruct string_compareL(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
16895 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2, rFlagsReg cr)
16896 %{
16897 predicate(((StrCompNode*)n)->encoding() == StrIntrinsicNode::LL);
16898 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
16899 effect(KILL tmp1, KILL tmp2, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
16900
16901 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # KILL $tmp1" %}
16902 ins_encode %{
16903 __ string_compare($str1$$Register, $str2$$Register,
16904 $cnt1$$Register, $cnt2$$Register, $result$$Register,
16905 $tmp1$$Register, $tmp2$$Register,
16906 fnoreg, fnoreg, fnoreg, StrIntrinsicNode::LL);
16907 %}
16908 ins_pipe(pipe_class_memory);
16909 %}
16910
16911 instruct string_compareUL(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
16912 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
16913 vRegD_V0 vtmp1, vRegD_V1 vtmp2, vRegD_V2 vtmp3, rFlagsReg cr)
16914 %{
16915 predicate(((StrCompNode*)n)->encoding() == StrIntrinsicNode::UL);
16916 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
16917 effect(KILL tmp1, KILL tmp2, KILL vtmp1, KILL vtmp2, KILL vtmp3,
16918 USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
16919
16920 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # KILL $tmp1, $tmp2, $vtmp1, $vtmp2, $vtmp3" %}
16921 ins_encode %{
16922 __ string_compare($str1$$Register, $str2$$Register,
16923 $cnt1$$Register, $cnt2$$Register, $result$$Register,
16924 $tmp1$$Register, $tmp2$$Register,
16925 $vtmp1$$FloatRegister, $vtmp2$$FloatRegister,
16926 $vtmp3$$FloatRegister, StrIntrinsicNode::UL);
16927 %}
16928 ins_pipe(pipe_class_memory);
16929 %}
16930
16931 instruct string_compareLU(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
16932 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
16933 vRegD_V0 vtmp1, vRegD_V1 vtmp2, vRegD_V2 vtmp3, rFlagsReg cr)
16934 %{
16935 predicate(((StrCompNode*)n)->encoding() == StrIntrinsicNode::LU);
16936 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
16937 effect(KILL tmp1, KILL tmp2, KILL vtmp1, KILL vtmp2, KILL vtmp3,
16938 USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
16939
16940 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # KILL $tmp1, $tmp2, $vtmp1, $vtmp2, $vtmp3" %}
16941 ins_encode %{
16942 __ string_compare($str1$$Register, $str2$$Register,
16943 $cnt1$$Register, $cnt2$$Register, $result$$Register,
16944 $tmp1$$Register, $tmp2$$Register,
16945 $vtmp1$$FloatRegister, $vtmp2$$FloatRegister,
16946 $vtmp3$$FloatRegister,StrIntrinsicNode::LU);
16947 %}
16948 ins_pipe(pipe_class_memory);
16949 %}
16950
16951 instruct string_indexofUU(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2, iRegI_R2 cnt2,
16952 iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2, iRegINoSp tmp3,
16953 iRegINoSp tmp4, iRegINoSp tmp5, iRegINoSp tmp6, rFlagsReg cr)
16954 %{
16955 predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UU);
16956 match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 cnt2)));
16957 effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2,
16958 TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, TEMP tmp5, TEMP tmp6, KILL cr);
16959 format %{ "String IndexOf $str1,$cnt1,$str2,$cnt2 -> $result (UU)" %}
16960
16961 ins_encode %{
16962 __ string_indexof($str1$$Register, $str2$$Register,
16963 $cnt1$$Register, $cnt2$$Register,
16964 $tmp1$$Register, $tmp2$$Register,
16965 $tmp3$$Register, $tmp4$$Register,
16966 $tmp5$$Register, $tmp6$$Register,
16967 -1, $result$$Register, StrIntrinsicNode::UU);
16968 %}
16969 ins_pipe(pipe_class_memory);
16970 %}
16971
16972 instruct string_indexofLL(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2, iRegI_R2 cnt2,
16973 iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2, iRegINoSp tmp3,
16974 iRegINoSp tmp4, iRegINoSp tmp5, iRegINoSp tmp6, rFlagsReg cr)
16975 %{
16976 predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::LL);
16977 match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 cnt2)));
16978 effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2,
16979 TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, TEMP tmp5, TEMP tmp6, KILL cr);
16980 format %{ "String IndexOf $str1,$cnt1,$str2,$cnt2 -> $result (LL)" %}
16981
16982 ins_encode %{
16983 __ string_indexof($str1$$Register, $str2$$Register,
16984 $cnt1$$Register, $cnt2$$Register,
16985 $tmp1$$Register, $tmp2$$Register,
16986 $tmp3$$Register, $tmp4$$Register,
16987 $tmp5$$Register, $tmp6$$Register,
16988 -1, $result$$Register, StrIntrinsicNode::LL);
16989 %}
16990 ins_pipe(pipe_class_memory);
16991 %}
16992
16993 instruct string_indexofUL(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2, iRegI_R2 cnt2,
16994 iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2, iRegINoSp tmp3,
16995 iRegINoSp tmp4, iRegINoSp tmp5, iRegINoSp tmp6, rFlagsReg cr)
16996 %{
16997 predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UL);
16998 match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 cnt2)));
16999 effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2,
17000 TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, TEMP tmp5, TEMP tmp6, KILL cr);
17001 format %{ "String IndexOf $str1,$cnt1,$str2,$cnt2 -> $result (UL)" %}
17002
17003 ins_encode %{
17004 __ string_indexof($str1$$Register, $str2$$Register,
17005 $cnt1$$Register, $cnt2$$Register,
17006 $tmp1$$Register, $tmp2$$Register,
17007 $tmp3$$Register, $tmp4$$Register,
17008 $tmp5$$Register, $tmp6$$Register,
17009 -1, $result$$Register, StrIntrinsicNode::UL);
17010 %}
17011 ins_pipe(pipe_class_memory);
17012 %}
17013
17014 instruct string_indexof_conUU(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2,
17015 immI_le_4 int_cnt2, iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2,
17016 iRegINoSp tmp3, iRegINoSp tmp4, rFlagsReg cr)
17017 %{
17018 predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UU);
17019 match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 int_cnt2)));
17020 effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1,
17021 TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, KILL cr);
17022 format %{ "String IndexOf $str1,$cnt1,$str2,$int_cnt2 -> $result (UU)" %}
17023
17024 ins_encode %{
17025 int icnt2 = (int)$int_cnt2$$constant;
17026 __ string_indexof($str1$$Register, $str2$$Register,
17027 $cnt1$$Register, zr,
17028 $tmp1$$Register, $tmp2$$Register,
17029 $tmp3$$Register, $tmp4$$Register, zr, zr,
17030 icnt2, $result$$Register, StrIntrinsicNode::UU);
17031 %}
17032 ins_pipe(pipe_class_memory);
17033 %}
17034
17035 instruct string_indexof_conLL(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2,
17036 immI_le_4 int_cnt2, iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2,
17037 iRegINoSp tmp3, iRegINoSp tmp4, rFlagsReg cr)
17038 %{
17039 predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::LL);
17040 match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 int_cnt2)));
17041 effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1,
17042 TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, KILL cr);
17043 format %{ "String IndexOf $str1,$cnt1,$str2,$int_cnt2 -> $result (LL)" %}
17044
17045 ins_encode %{
17046 int icnt2 = (int)$int_cnt2$$constant;
17047 __ string_indexof($str1$$Register, $str2$$Register,
17048 $cnt1$$Register, zr,
17049 $tmp1$$Register, $tmp2$$Register,
17050 $tmp3$$Register, $tmp4$$Register, zr, zr,
17051 icnt2, $result$$Register, StrIntrinsicNode::LL);
17052 %}
17053 ins_pipe(pipe_class_memory);
17054 %}
17055
17056 instruct string_indexof_conUL(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2,
17057 immI_1 int_cnt2, iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2,
17058 iRegINoSp tmp3, iRegINoSp tmp4, rFlagsReg cr)
17059 %{
17060 predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UL);
17061 match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 int_cnt2)));
17062 effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1,
17063 TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, KILL cr);
17064 format %{ "String IndexOf $str1,$cnt1,$str2,$int_cnt2 -> $result (UL)" %}
17065
17066 ins_encode %{
17067 int icnt2 = (int)$int_cnt2$$constant;
17068 __ string_indexof($str1$$Register, $str2$$Register,
17069 $cnt1$$Register, zr,
17070 $tmp1$$Register, $tmp2$$Register,
17071 $tmp3$$Register, $tmp4$$Register, zr, zr,
17072 icnt2, $result$$Register, StrIntrinsicNode::UL);
17073 %}
17074 ins_pipe(pipe_class_memory);
17075 %}
17076
17077 instruct string_indexof_char(iRegP_R1 str1, iRegI_R2 cnt1, iRegI_R3 ch,
17078 iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2,
17079 iRegINoSp tmp3, rFlagsReg cr)
17080 %{
17081 match(Set result (StrIndexOfChar (Binary str1 cnt1) ch));
17082 predicate((UseSVE == 0) && (((StrIndexOfCharNode*)n)->encoding() == StrIntrinsicNode::U));
17083 effect(USE_KILL str1, USE_KILL cnt1, USE_KILL ch,
17084 TEMP tmp1, TEMP tmp2, TEMP tmp3, KILL cr);
17085
17086 format %{ "StringUTF16 IndexOf char[] $str1,$cnt1,$ch -> $result" %}
17087
17088 ins_encode %{
17089 __ string_indexof_char($str1$$Register, $cnt1$$Register, $ch$$Register,
17090 $result$$Register, $tmp1$$Register, $tmp2$$Register,
17091 $tmp3$$Register);
17092 %}
17093 ins_pipe(pipe_class_memory);
17094 %}
17095
17096 instruct stringL_indexof_char(iRegP_R1 str1, iRegI_R2 cnt1, iRegI_R3 ch,
17097 iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2,
17098 iRegINoSp tmp3, rFlagsReg cr)
17099 %{
17100 match(Set result (StrIndexOfChar (Binary str1 cnt1) ch));
17101 predicate((UseSVE == 0) && (((StrIndexOfCharNode*)n)->encoding() == StrIntrinsicNode::L));
17102 effect(USE_KILL str1, USE_KILL cnt1, USE_KILL ch,
17103 TEMP tmp1, TEMP tmp2, TEMP tmp3, KILL cr);
17104
17105 format %{ "StringLatin1 IndexOf char[] $str1,$cnt1,$ch -> $result" %}
17106
17107 ins_encode %{
17108 __ stringL_indexof_char($str1$$Register, $cnt1$$Register, $ch$$Register,
17109 $result$$Register, $tmp1$$Register, $tmp2$$Register,
17110 $tmp3$$Register);
17111 %}
17112 ins_pipe(pipe_class_memory);
17113 %}
17114
17115 instruct string_equalsL(iRegP_R1 str1, iRegP_R3 str2, iRegI_R4 cnt,
17116 iRegI_R0 result, rFlagsReg cr)
17117 %{
17118 predicate(((StrEqualsNode*)n)->encoding() == StrIntrinsicNode::LL);
17119 match(Set result (StrEquals (Binary str1 str2) cnt));
17120 effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt, KILL cr);
17121
17122 format %{ "String Equals $str1,$str2,$cnt -> $result" %}
17123 ins_encode %{
17124 // Count is in 8-bit bytes; non-Compact chars are 16 bits.
17125 __ string_equals($str1$$Register, $str2$$Register,
17126 $result$$Register, $cnt$$Register, 1);
17127 %}
17128 ins_pipe(pipe_class_memory);
17129 %}
17130
17131 instruct string_equalsU(iRegP_R1 str1, iRegP_R3 str2, iRegI_R4 cnt,
17132 iRegI_R0 result, rFlagsReg cr)
17133 %{
17134 predicate(((StrEqualsNode*)n)->encoding() == StrIntrinsicNode::UU);
17135 match(Set result (StrEquals (Binary str1 str2) cnt));
17136 effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt, KILL cr);
17137
17138 format %{ "String Equals $str1,$str2,$cnt -> $result" %}
17139 ins_encode %{
17140 // Count is in 8-bit bytes; non-Compact chars are 16 bits.
17141 __ string_equals($str1$$Register, $str2$$Register,
17142 $result$$Register, $cnt$$Register, 2);
17143 %}
17144 ins_pipe(pipe_class_memory);
17145 %}
17146
17147 instruct array_equalsB(iRegP_R1 ary1, iRegP_R2 ary2, iRegI_R0 result,
17148 iRegP_R3 tmp1, iRegP_R4 tmp2, iRegP_R5 tmp3,
17149 iRegP_R10 tmp, rFlagsReg cr)
17150 %{
17151 predicate(((AryEqNode*)n)->encoding() == StrIntrinsicNode::LL);
17152 match(Set result (AryEq ary1 ary2));
17153 effect(KILL tmp, USE_KILL ary1, USE_KILL ary2, TEMP tmp1, TEMP tmp2, TEMP tmp3, KILL cr);
17154
17155 format %{ "Array Equals $ary1,ary2 -> $result // KILL $tmp" %}
17156 ins_encode %{
17157 address tpc = __ arrays_equals($ary1$$Register, $ary2$$Register,
17158 $tmp1$$Register, $tmp2$$Register, $tmp3$$Register,
17159 $result$$Register, $tmp$$Register, 1);
17160 if (tpc == NULL) {
17161 ciEnv::current()->record_failure("CodeCache is full");
17162 return;
17163 }
17164 %}
17165 ins_pipe(pipe_class_memory);
17166 %}
17167
17168 instruct array_equalsC(iRegP_R1 ary1, iRegP_R2 ary2, iRegI_R0 result,
17169 iRegP_R3 tmp1, iRegP_R4 tmp2, iRegP_R5 tmp3,
17170 iRegP_R10 tmp, rFlagsReg cr)
17171 %{
17172 predicate(((AryEqNode*)n)->encoding() == StrIntrinsicNode::UU);
17173 match(Set result (AryEq ary1 ary2));
17174 effect(KILL tmp, USE_KILL ary1, USE_KILL ary2, TEMP tmp1, TEMP tmp2, TEMP tmp3, KILL cr);
17175
17176 format %{ "Array Equals $ary1,ary2 -> $result // KILL $tmp" %}
17177 ins_encode %{
17178 address tpc = __ arrays_equals($ary1$$Register, $ary2$$Register,
17179 $tmp1$$Register, $tmp2$$Register, $tmp3$$Register,
17180 $result$$Register, $tmp$$Register, 2);
17181 if (tpc == NULL) {
17182 ciEnv::current()->record_failure("CodeCache is full");
17183 return;
17184 }
17185 %}
17186 ins_pipe(pipe_class_memory);
17187 %}
17188
17189 instruct count_positives(iRegP_R1 ary1, iRegI_R2 len, iRegI_R0 result, rFlagsReg cr)
17190 %{
17191 match(Set result (CountPositives ary1 len));
17192 effect(USE_KILL ary1, USE_KILL len, KILL cr);
17193 format %{ "count positives byte[] $ary1,$len -> $result" %}
17194 ins_encode %{
17195 address tpc = __ count_positives($ary1$$Register, $len$$Register, $result$$Register);
17196 if (tpc == NULL) {
17197 ciEnv::current()->record_failure("CodeCache is full");
17198 return;
17199 }
17200 %}
17201 ins_pipe( pipe_slow );
17202 %}
17203
17204 // fast char[] to byte[] compression
17205 instruct string_compress(iRegP_R2 src, iRegP_R1 dst, iRegI_R3 len,
17206 vRegD_V0 tmp1, vRegD_V1 tmp2,
17207 vRegD_V2 tmp3, vRegD_V3 tmp4,
17208 iRegI_R0 result, rFlagsReg cr)
17209 %{
17210 match(Set result (StrCompressedCopy src (Binary dst len)));
17211 effect(TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4,
17212 USE_KILL src, USE_KILL dst, USE len, KILL cr);
17213
17214 format %{ "String Compress $src,$dst,$len -> $result // KILL $src,$dst" %}
17215 ins_encode %{
17216 __ char_array_compress($src$$Register, $dst$$Register, $len$$Register,
17217 $result$$Register,
17218 $tmp1$$FloatRegister, $tmp2$$FloatRegister,
17219 $tmp3$$FloatRegister, $tmp4$$FloatRegister);
17220 %}
17221 ins_pipe(pipe_slow);
17222 %}
17223
17224 // fast byte[] to char[] inflation
17225 instruct string_inflate(Universe dummy, iRegP_R0 src, iRegP_R1 dst, iRegI_R2 len,
17226 vRegD_V0 tmp1, vRegD_V1 tmp2, vRegD_V2 tmp3, iRegP_R3 tmp4, rFlagsReg cr)
17227 %{
17228 match(Set dummy (StrInflatedCopy src (Binary dst len)));
17229 effect(TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, USE_KILL src, USE_KILL dst, USE_KILL len, KILL cr);
17230
17231 format %{ "String Inflate $src,$dst // KILL $tmp1, $tmp2" %}
17232 ins_encode %{
17233 address tpc = __ byte_array_inflate($src$$Register, $dst$$Register, $len$$Register,
17234 $tmp1$$FloatRegister, $tmp2$$FloatRegister,
17235 $tmp3$$FloatRegister, $tmp4$$Register);
17236 if (tpc == NULL) {
17237 ciEnv::current()->record_failure("CodeCache is full");
17238 return;
17239 }
17240 %}
17241 ins_pipe(pipe_class_memory);
17242 %}
17243
17244 // encode char[] to byte[] in ISO_8859_1
17245 instruct encode_iso_array(iRegP_R2 src, iRegP_R1 dst, iRegI_R3 len,
17246 vRegD_V0 vtmp0, vRegD_V1 vtmp1,
17247 vRegD_V2 vtmp2, vRegD_V3 vtmp3,
17248 iRegI_R0 result, rFlagsReg cr)
17249 %{
17250 predicate(!((EncodeISOArrayNode*)n)->is_ascii());
17251 match(Set result (EncodeISOArray src (Binary dst len)));
17252 effect(USE_KILL src, USE_KILL dst, USE len,
17253 KILL vtmp0, KILL vtmp1, KILL vtmp2, KILL vtmp3, KILL cr);
17254
17255 format %{ "Encode ISO array $src,$dst,$len -> $result" %}
17256 ins_encode %{
17257 __ encode_iso_array($src$$Register, $dst$$Register, $len$$Register,
17258 $result$$Register, false,
17259 $vtmp0$$FloatRegister, $vtmp1$$FloatRegister,
17260 $vtmp2$$FloatRegister, $vtmp3$$FloatRegister);
17261 %}
17262 ins_pipe(pipe_class_memory);
17263 %}
17264
17265 instruct encode_ascii_array(iRegP_R2 src, iRegP_R1 dst, iRegI_R3 len,
17266 vRegD_V0 vtmp0, vRegD_V1 vtmp1,
17267 vRegD_V2 vtmp2, vRegD_V3 vtmp3,
17268 iRegI_R0 result, rFlagsReg cr)
17269 %{
17270 predicate(((EncodeISOArrayNode*)n)->is_ascii());
17271 match(Set result (EncodeISOArray src (Binary dst len)));
17272 effect(USE_KILL src, USE_KILL dst, USE len,
17273 KILL vtmp0, KILL vtmp1, KILL vtmp2, KILL vtmp3, KILL cr);
17274
17275 format %{ "Encode ASCII array $src,$dst,$len -> $result" %}
17276 ins_encode %{
17277 __ encode_iso_array($src$$Register, $dst$$Register, $len$$Register,
17278 $result$$Register, true,
17279 $vtmp0$$FloatRegister, $vtmp1$$FloatRegister,
17280 $vtmp2$$FloatRegister, $vtmp3$$FloatRegister);
17281 %}
17282 ins_pipe(pipe_class_memory);
17283 %}
17284
17285 // ============================================================================
17286 // This name is KNOWN by the ADLC and cannot be changed.
17287 // The ADLC forces a 'TypeRawPtr::BOTTOM' output type
17288 // for this guy.
17289 instruct tlsLoadP(thread_RegP dst)
17290 %{
17291 match(Set dst (ThreadLocal));
17292
17293 ins_cost(0);
17294
17295 format %{ " -- \t// $dst=Thread::current(), empty" %}
17296
17297 size(0);
17298
17299 ins_encode( /*empty*/ );
17300
17301 ins_pipe(pipe_class_empty);
17302 %}
17303
17304 //----------PEEPHOLE RULES-----------------------------------------------------
17305 // These must follow all instruction definitions as they use the names
17306 // defined in the instructions definitions.
17307 //
17308 // peepmatch ( root_instr_name [preceding_instruction]* );
17309 //
17310 // peepconstraint %{
17311 // (instruction_number.operand_name relational_op instruction_number.operand_name
17312 // [, ...] );
17313 // // instruction numbers are zero-based using left to right order in peepmatch
17314 //
17315 // peepreplace ( instr_name ( [instruction_number.operand_name]* ) );
17316 // // provide an instruction_number.operand_name for each operand that appears
17317 // // in the replacement instruction's match rule
17318 //
17319 // ---------VM FLAGS---------------------------------------------------------
17320 //
17321 // All peephole optimizations can be turned off using -XX:-OptoPeephole
17322 //
17323 // Each peephole rule is given an identifying number starting with zero and
17324 // increasing by one in the order seen by the parser. An individual peephole
17325 // can be enabled, and all others disabled, by using -XX:OptoPeepholeAt=#
17326 // on the command-line.
17327 //
17328 // ---------CURRENT LIMITATIONS----------------------------------------------
17329 //
17330 // Only match adjacent instructions in same basic block
17331 // Only equality constraints
17332 // Only constraints between operands, not (0.dest_reg == RAX_enc)
17333 // Only one replacement instruction
17334 //
17335 // ---------EXAMPLE----------------------------------------------------------
17336 //
17337 // // pertinent parts of existing instructions in architecture description
17338 // instruct movI(iRegINoSp dst, iRegI src)
17339 // %{
17340 // match(Set dst (CopyI src));
17341 // %}
17342 //
17343 // instruct incI_iReg(iRegINoSp dst, immI1 src, rFlagsReg cr)
17344 // %{
17345 // match(Set dst (AddI dst src));
17346 // effect(KILL cr);
17347 // %}
17348 //
17349 // // Change (inc mov) to lea
17350 // peephole %{
17351 // // increment preceeded by register-register move
17352 // peepmatch ( incI_iReg movI );
17353 // // require that the destination register of the increment
17354 // // match the destination register of the move
17355 // peepconstraint ( 0.dst == 1.dst );
17356 // // construct a replacement instruction that sets
17357 // // the destination to ( move's source register + one )
17358 // peepreplace ( leaI_iReg_immI( 0.dst 1.src 0.src ) );
17359 // %}
17360 //
17361
17362 // Implementation no longer uses movX instructions since
17363 // machine-independent system no longer uses CopyX nodes.
17364 //
17365 // peephole
17366 // %{
17367 // peepmatch (incI_iReg movI);
17368 // peepconstraint (0.dst == 1.dst);
17369 // peepreplace (leaI_iReg_immI(0.dst 1.src 0.src));
17370 // %}
17371
17372 // peephole
17373 // %{
17374 // peepmatch (decI_iReg movI);
17375 // peepconstraint (0.dst == 1.dst);
17376 // peepreplace (leaI_iReg_immI(0.dst 1.src 0.src));
17377 // %}
17378
17379 // peephole
17380 // %{
17381 // peepmatch (addI_iReg_imm movI);
17382 // peepconstraint (0.dst == 1.dst);
17383 // peepreplace (leaI_iReg_immI(0.dst 1.src 0.src));
17384 // %}
17385
17386 // peephole
17387 // %{
17388 // peepmatch (incL_iReg movL);
17389 // peepconstraint (0.dst == 1.dst);
17390 // peepreplace (leaL_iReg_immL(0.dst 1.src 0.src));
17391 // %}
17392
17393 // peephole
17394 // %{
17395 // peepmatch (decL_iReg movL);
17396 // peepconstraint (0.dst == 1.dst);
17397 // peepreplace (leaL_iReg_immL(0.dst 1.src 0.src));
17398 // %}
17399
17400 // peephole
17401 // %{
17402 // peepmatch (addL_iReg_imm movL);
17403 // peepconstraint (0.dst == 1.dst);
17404 // peepreplace (leaL_iReg_immL(0.dst 1.src 0.src));
17405 // %}
17406
17407 // peephole
17408 // %{
17409 // peepmatch (addP_iReg_imm movP);
17410 // peepconstraint (0.dst == 1.dst);
17411 // peepreplace (leaP_iReg_imm(0.dst 1.src 0.src));
17412 // %}
17413
17414 // // Change load of spilled value to only a spill
17415 // instruct storeI(memory mem, iRegI src)
17416 // %{
17417 // match(Set mem (StoreI mem src));
17418 // %}
17419 //
17420 // instruct loadI(iRegINoSp dst, memory mem)
17421 // %{
17422 // match(Set dst (LoadI mem));
17423 // %}
17424 //
17425
17426 //----------SMARTSPILL RULES---------------------------------------------------
17427 // These must follow all instruction definitions as they use the names
17428 // defined in the instructions definitions.
17429
17430 // Local Variables:
17431 // mode: c++
17432 // End: