1 dnl Copyright (c) 2025, Oracle and/or its affiliates. All rights reserved.
2 dnl DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
3 dnl
4 dnl This code is free software; you can redistribute it and/or modify it
5 dnl under the terms of the GNU General Public License version 2 only, as
6 dnl published by the Free Software Foundation.
7 dnl
8 dnl This code is distributed in the hope that it will be useful, but WITHOUT
9 dnl ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
10 dnl FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
11 dnl version 2 for more details (a copy is included in the LICENSE file that
12 dnl accompanied this code).
13 dnl
14 dnl You should have received a copy of the GNU General Public License version
15 dnl 2 along with this work; if not, write to the Free Software Foundation,
16 dnl Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
17 dnl
18 dnl Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
19 dnl or visit www.oracle.com if you need additional information or have any
20 dnl questions.
21 dnl
22 // BEGIN This section of the file is automatically generated from shenandoah_aarch64.m4.
23
24
25 define(`STORE_INSN',
26 `
27 // This pattern is generated automatically from shenandoah_aarch64.m4.
28 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
29 instruct store_$1_$2_shenandoah(indirect mem, iReg$1 src, iRegPNoSp tmp, rFlagsReg cr)
30 %{
31 match(Set mem (Store$1 mem src));
32 predicate(UseShenandoahGC && ifelse($2,Volatile,'needs_releasing_store(n)`,'!needs_releasing_store(n)`) && n->as_Store()->barrier_data() != 0);
33 effect(TEMP tmp, KILL cr);
34 ins_cost(ifelse($2,Volatile,VOLATILE_REF_COST,3*INSN_COST));
35 format %{ "$3 $src, $mem" %}
36 ins_encode %{
37 Address gcs_addr(rthread, in_bytes(ShenandoahThreadLocalData::gc_state_offset()));
38 __ ldrb($tmp$$Register, gcs_addr);
39
40 ShenandoahBarrierSet::assembler()->satb_barrier_c2(this, masm,
41 $mem$$Register /* addr, used in slow path only */,
42 noreg /* pre_val, used in slow path only */,
43 $tmp$$Register /* gc_state on fas path */,
44 false /* unused in this case */);
45
46 __ $3($src$$Register, $mem$$Register);
47
48 ShenandoahBarrierSet::assembler()->card_barrier_c2(this, masm,
49 $mem$$Register /* addr */,
50 $tmp$$Register /* tmp */);
51 %}
52 ins_pipe(pipe_class_memory);
53 %}')dnl
54 STORE_INSN(P,Normal,str)
55 STORE_INSN(P,Volatile,stlr)
56 STORE_INSN(N,Normal,strw)
57 STORE_INSN(N,Volatile,stlrw)
58
59
60
61
62
63
64
65
66
67 define(`ENCODEP_AND_STORE_INSN',
68 `
69 // This pattern is generated automatically from shenandoah_aarch64.m4.
70 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
71 instruct encodePAndStoreN_$1_shenandoah(indirect mem, iRegP src, iRegPNoSp tmp, rFlagsReg cr)
72 %{
73 match(Set mem (StoreN mem (EncodeP src)));
74 predicate(UseShenandoahGC && ifelse($1,Volatile,'needs_releasing_store(n)`,'!needs_releasing_store(n)`) && n->as_Store()->barrier_data() != 0);
75 effect(TEMP tmp, KILL cr);
76 ins_cost(ifelse($1,Volatile,VOLATILE_REF_COST,4*INSN_COST));
77 format %{ "encode_heap_oop $tmp, $src\n\t"
78 "$2 $tmp, $mem\t# compressed ptr" %}
79 ins_encode %{
80 Address gcs_addr(rthread, in_bytes(ShenandoahThreadLocalData::gc_state_offset()));
81 __ ldrb($tmp$$Register, gcs_addr);
82
83 ShenandoahBarrierSet::assembler()->satb_barrier_c2(this, masm,
84 $mem$$Register /* addr, used in slow path only */,
85 noreg /* pre_val, used in slow path only */,
86 $tmp$$Register /* gc_state on fas path */,
87 false /* encoded_preval */);
88
89 if ((barrier_data() & ShenandoahBarrierNotNull) == 0) {
90 __ encode_heap_oop($tmp$$Register, $src$$Register);
91 } else {
92 __ encode_heap_oop_not_null($tmp$$Register, $src$$Register);
93 }
94
95 __ $2($tmp$$Register, $mem$$Register);
96
97 ShenandoahBarrierSet::assembler()->card_barrier_c2(this, masm,
98 $mem$$Register /* addr */,
99 $tmp$$Register /* tmp */);
100 %}
101 ins_pipe(pipe_class_memory);
102 %}')dnl
103 ENCODEP_AND_STORE_INSN(Normal,strw)
104 ENCODEP_AND_STORE_INSN(Volatile,stlrw)
105
106
107
108
109
110
111
112
113
114 define(`CMPANDSWP_INSN',
115 `
116 // This pattern is generated automatically from shenandoah_aarch64.m4.
117 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
118 instruct compareAndSwap_$1_$2_shenandoah(iRegINoSp res, indirect mem, iReg$1NoSp oldval, iReg$1 newval, iRegPNoSp tmp1, iRegPNoSp tmp2, rFlagsReg cr)
119 %{
120 match(Set res (CompareAndSwap$1 mem (Binary oldval newval)));
121 predicate(UseShenandoahGC && ifelse($2,Volatile,'needs_acquiring_load_exclusive(n)`,'!needs_acquiring_load_exclusive(n)`) && n->as_LoadStore()->barrier_data() != 0);
122 ins_cost(ifelse($2,Volatile,VOLATILE_REF_COST + 3*INSN_COST,VOLATILE_REF_COST));
123 effect(TEMP_DEF res, TEMP tmp1, TEMP tmp2, KILL cr);
124 format %{
125 "cmpxchg_$1_$2_shenandoah $mem, $oldval, $newval\t# (ptr) if $mem == $oldval then $mem <-- $newval"
126 %}
127 ins_encode %{
128 assert_different_registers($tmp1$$Register, $tmp2$$Register);
129 guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
130
131 Address gcs_addr(rthread, in_bytes(ShenandoahThreadLocalData::gc_state_offset()));
132 __ ldrb($tmp1$$Register, gcs_addr);
133
134 bool is_narrow = ifelse($1,N,'true`,'false`);
135 ShenandoahBarrierSet::assembler()->satb_barrier_c2(this, masm,
136 noreg /* addr */,
137 $oldval$$Register /* pre_val, only used in slow path */,
138 $tmp1$$Register /* gc_state */,
139 is_narrow /* encoded_preval */);
140
141 bool is_acquire = ifelse($2,Volatile,'true`,'false`);
142 ShenandoahBarrierSet::assembler()->cmpxchg_oop_c2(this, masm,
143 $mem$$base$$Register,
144 $oldval$$Register,
145 $newval$$Register,
146 $res$$Register,
147 $tmp1$$Register, /* gc_state */
148 $tmp2$$Register,
149 /*acquire*/ is_acquire,
150 /*release*/ true,
151 /*weak*/ false,
152 /*is_cae*/ false);
153
154 ShenandoahBarrierSet::assembler()->card_barrier_c2(this, masm,
155 $mem$$Register /* addr */,
156 $tmp1$$Register /* tmp */);
157 %}
158
159 ins_pipe(pipe_slow);
160 %}')dnl
161 CMPANDSWP_INSN(P,Normal)
162 CMPANDSWP_INSN(N,Normal)
163 CMPANDSWP_INSN(P,Volatile)
164 CMPANDSWP_INSN(N,Volatile)
165
166
167
168
169
170 define(`CMPANDXCGH_INSN',
171 `
172 // This pattern is generated automatically from shenandoah_aarch64.m4.
173 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
174 instruct compareAndExchange_$1_$2_shenandoah(iReg$1NoSp res, indirect mem, iReg$1 oldval, iReg$1 newval, iRegPNoSp tmp1, iRegPNoSp tmp2, rFlagsReg cr)
175 %{
176 match(Set res (CompareAndExchange$1 mem (Binary oldval newval)));
177 predicate(UseShenandoahGC && ifelse($2,Volatile,'needs_acquiring_load_exclusive(n)`,'!needs_acquiring_load_exclusive(n)`) && n->as_LoadStore()->barrier_data() != 0);
178 ins_cost(ifelse($2,Volatile,VOLATILE_REF_COST + 3*INSN_COST,2*VOLATILE_REF_COST));
179 effect(TEMP_DEF res, TEMP tmp1, TEMP tmp2, KILL cr);
180 format %{
181 "cmpxchg_$1_$2_shenandoah $mem, $oldval, $newval\t# (ptr) if $mem == $oldval then $mem <-- $newval"
182 %}
183 ins_encode %{
184 assert_different_registers($tmp1$$Register, $tmp2$$Register);
185 guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
186
187 Address gcs_addr(rthread, in_bytes(ShenandoahThreadLocalData::gc_state_offset()));
188 __ ldrb($tmp1$$Register, gcs_addr);
189
190 bool is_narrow = ifelse($1,N,'true`,'false`);
191 ShenandoahBarrierSet::assembler()->satb_barrier_c2(this, masm,
192 noreg /* addr */,
193 $oldval$$Register /* pre_val */,
194 $tmp1$$Register /* gc_state */,
195 is_narrow /* encoded_preval */);
196
197 bool is_acquire = ifelse($2,Volatile,'true`,'false`);
198 ShenandoahBarrierSet::assembler()->cmpxchg_oop_c2(this, masm,
199 $mem$$base$$Register,
200 $oldval$$Register,
201 $newval$$Register,
202 $res$$Register,
203 $tmp1$$Register, /* gc_state */
204 $tmp2$$Register,
205 /*acquire*/ is_acquire,
206 /*release*/ true,
207 /*weak*/ false,
208 /*is_cae*/ true);
209
210 bool maybe_null = (this->bottom_type()->make_ptr()->ptr() != TypePtr::NotNull);
211 ShenandoahBarrierSet::assembler()->load_ref_barrier_c2(this, masm,
212 $res$$Register /* obj */,
213 $mem$$Register /* addr */,
214 is_narrow /* narrow */,
215 maybe_null,
216 $tmp1$$Register /* gc_state */);
217
218 ShenandoahBarrierSet::assembler()->card_barrier_c2(this, masm,
219 $mem$$Register /* addr */,
220 $tmp1$$Register /* tmp */);
221 %}
222 ins_pipe(pipe_slow);
223 %}')dnl
224 CMPANDXCGH_INSN(N,Normal)
225 CMPANDXCGH_INSN(P,Normal)
226 CMPANDXCGH_INSN(N,Volatile)
227 CMPANDXCGH_INSN(P,Volatile)
228
229
230
231
232
233
234
235 define(`WEAKCMPANDSWAP_INSN',
236 `
237 // This pattern is generated automatically from shenandoah_aarch64.m4.
238 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
239 instruct weakCompareAndSwap_$1_$2_shenandoah(iRegINoSp res, indirect mem, iReg$1NoSp oldval, iReg$1 newval, iRegPNoSp tmp1, iRegPNoSp tmp2, rFlagsReg cr)
240 %{
241 match(Set res (WeakCompareAndSwap$1 mem (Binary oldval newval)));
242 predicate(UseShenandoahGC && ifelse($2,Volatile,'needs_acquiring_load_exclusive(n)`,'!needs_acquiring_load_exclusive(n)`) && n->as_LoadStore()->barrier_data() != 0);
243 ins_cost(VOLATILE_REF_COST);
244 effect(TEMP_DEF res, TEMP tmp1, TEMP tmp2, KILL cr);
245 format %{
246 "cmpxchg_oop_c2 $res = $mem, $oldval, $newval\t# ($1, weak, $2) if $mem == $oldval then $mem <-- $newval\n\t"
247 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
248 %}
249 ins_encode %{
250 assert_different_registers($tmp1$$Register, $tmp2$$Register);
251 guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
252
253 Address gcs_addr(rthread, in_bytes(ShenandoahThreadLocalData::gc_state_offset()));
254 __ ldrb($tmp1$$Register, gcs_addr);
255
256 bool is_narrow = ifelse($1,N,'true`,'false`);
257 ShenandoahBarrierSet::assembler()->satb_barrier_c2(this, masm,
258 noreg /* addr */,
259 $oldval$$Register /* pre_val */,
260 $tmp1$$Register /* gc_state */,
261 is_narrow /* encoded_preval */);
262
263 bool is_acquire = ifelse($2,Volatile,'true`,'false`);
264 ShenandoahBarrierSet::assembler()->cmpxchg_oop_c2(this, masm,
265 $mem$$base$$Register,
266 $oldval$$Register,
267 $newval$$Register,
268 $res$$Register,
269 $tmp1$$Register,
270 $tmp2$$Register,
271 /*acquire*/ is_acquire,
272 /*release*/ true,
273 /*weak*/ true,
274 /*is_cae*/ false);
275
276 ShenandoahBarrierSet::assembler()->card_barrier_c2(this, masm,
277 $mem$$Register /* addr */,
278 $tmp1$$Register /* tmp */);
279 %}
280 ins_pipe(pipe_slow);
281 %}')dnl
282 WEAKCMPANDSWAP_INSN(N,Normal)
283 WEAKCMPANDSWAP_INSN(P,Normal)
284 WEAKCMPANDSWAP_INSN(N,Volatile)
285 WEAKCMPANDSWAP_INSN(P,Volatile)
286
287
288 define(`GETANDSET_INSN',
289 `
290 // This pattern is generated automatically from shenandoah_aarch64.m4.
291 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
292 instruct getAndSet_$1_$2_shenandoah(indirect mem, iReg$1 newval, iReg$1NoSp preval, iRegPNoSp tmp, rFlagsReg cr)
293 %{
294 match(Set preval (GetAndSet$1 mem newval));
295 predicate(UseShenandoahGC && ifelse($2,Volatile,'needs_acquiring_load_exclusive(n)`,'!needs_acquiring_load_exclusive(n)`) && n->as_LoadStore()->barrier_data() != 0);
296 effect(TEMP_DEF preval, TEMP tmp, KILL cr);
297 ins_cost(ifelse($2,Volatile,VOLATILE_REF_COST + 3*INSN_COST,2*VOLATILE_REF_COST));
298 format %{ "$3 $preval, $newval, [$mem]" %}
299 ins_encode %{
300 __ $3($preval$$Register, $newval$$Register, $mem$$Register);
301
302 Address gcs_addr(rthread, in_bytes(ShenandoahThreadLocalData::gc_state_offset()));
303 __ ldrb($tmp$$Register, gcs_addr);
304
305 bool is_narrow = ifelse($1,N,'true`,'false`);
306 ShenandoahBarrierSet::assembler()->satb_barrier_c2(this, masm,
307 noreg /* addr */,
308 $preval$$Register /* pre_val */,
309 $tmp$$Register /* gc_state */,
310 is_narrow /* encoded_preval */);
311
312 bool maybe_null = (this->bottom_type()->make_ptr()->ptr() != TypePtr::NotNull);
313 ShenandoahBarrierSet::assembler()->load_ref_barrier_c2(this, masm,
314 $preval$$Register /* obj */,
315 $mem$$Register /* addr */,
316 is_narrow /* narrow */,
317 maybe_null,
318 $tmp$$Register /* gc_state */);
319
320 ShenandoahBarrierSet::assembler()->card_barrier_c2(this, masm,
321 $mem$$Register /* addr */,
322 $tmp$$Register /* tmp */);
323 %}
324 ins_pipe(pipe_serial);
325 %}')dnl
326 GETANDSET_INSN(P,Normal,atomic_xchg)
327 GETANDSET_INSN(P,Volatile,atomic_xchgal)
328 GETANDSET_INSN(N,Normal,atomic_xchgw)
329 GETANDSET_INSN(N,Volatile,atomic_xchgalw)
330
331 define(`LOAD_INSN',
332 `
333 // This pattern is generated automatically from shenandoah_aarch64.m4.
334 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
335 instruct load_$1_$2_shenandoah(iReg$1NoSp dst, indirect mem, iRegPNoSp tmp, rFlagsReg cr)
336 %{
337 match(Set dst (Load$1 mem));
338 predicate(UseShenandoahGC && ifelse($2,Volatile,'needs_acquiring_load(n)`,'!needs_acquiring_load(n)`) && n->as_Load()->barrier_data() != 0);
339 effect(TEMP_DEF dst, TEMP tmp, KILL cr);
340 ins_cost(ifelse($2,Volatile,VOLATILE_REF_COST,3*INSN_COST));
341 format %{ "$3 $dst, $mem\t# ptr" %}
342 ins_encode %{
343 __ $3($dst$$Register, $mem$$Register);
344
345 Address gcs_addr(rthread, in_bytes(ShenandoahThreadLocalData::gc_state_offset()));
346 __ ldrb($tmp$$Register, gcs_addr);
347
348 bool is_narrow = ifelse($1,N,'true`,'false`);
349 ShenandoahBarrierSet::assembler()->satb_barrier_c2(this, masm,
350 noreg /* obj */,
351 $dst$$Register /* pre_val, in this case it will be only used in the slowpath as tmp. */,
352 $tmp$$Register /* gc_state */,
353 is_narrow /* encoded_preval */);
354
355 bool maybe_null = (this->bottom_type()->make_ptr()->ptr() != TypePtr::NotNull);
356 ShenandoahBarrierSet::assembler()->load_ref_barrier_c2(this, masm,
357 $dst$$Register /* obj */,
358 $mem$$Register /* addr */,
359 is_narrow /* narrow */,
360 maybe_null,
361 $tmp$$Register /* gc_state */);
362 %}
363 ins_pipe(pipe_class_memory);
364 %}')dnl
365 LOAD_INSN(P,Normal,ldr)
366 LOAD_INSN(P,Volatile,ldar)
367 LOAD_INSN(N,Normal,ldrw)
368 LOAD_INSN(N,Volatile,ldarw)
369
370 // END This section of the file is automatically generated from shenandoah_aarch64.m4.