1 //
2 // Copyright (c) 2024, 2025, Oracle and/or its affiliates. All rights reserved.
3 // DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
4 //
5 // This code is free software; you can redistribute it and/or modify it
6 // under the terms of the GNU General Public License version 2 only, as
7 // published by the Free Software Foundation.
8 //
9 // This code is distributed in the hope that it will be useful, but WITHOUT
10 // ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 // FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
12 // version 2 for more details (a copy is included in the LICENSE file that
13 // accompanied this code).
14 //
15 // You should have received a copy of the GNU General Public License version
16 // 2 along with this work; if not, write to the Free Software Foundation,
17 // Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
18 //
19 // Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
20 // or visit www.oracle.com if you need additional information or have any
21 // questions.
22 //
23
24 source_hpp %{
25
26 #include "gc/g1/c2/g1BarrierSetC2.hpp"
27 #include "gc/shared/gc_globals.hpp"
28
29 %}
30
31 source %{
32
33 #include "gc/g1/g1BarrierSetAssembler_aarch64.hpp"
34 #include "gc/g1/g1BarrierSetRuntime.hpp"
35
36 static void write_barrier_pre(MacroAssembler* masm,
37 const MachNode* node,
38 Register obj,
39 Register pre_val,
40 Register tmp1,
41 Register tmp2,
42 RegSet preserve = RegSet(),
43 RegSet no_preserve = RegSet()) {
44 if (!G1PreBarrierStubC2::needs_barrier(node)) {
45 return;
46 }
47 Assembler::InlineSkippedInstructionsCounter skip_counter(masm);
48 G1BarrierSetAssembler* g1_asm = static_cast<G1BarrierSetAssembler*>(BarrierSet::barrier_set()->barrier_set_assembler());
49 G1PreBarrierStubC2* const stub = G1PreBarrierStubC2::create(node);
50 for (RegSetIterator<Register> reg = preserve.begin(); *reg != noreg; ++reg) {
51 stub->preserve(*reg);
52 }
53 for (RegSetIterator<Register> reg = no_preserve.begin(); *reg != noreg; ++reg) {
54 stub->dont_preserve(*reg);
55 }
56 g1_asm->g1_write_barrier_pre_c2(masm, obj, pre_val, rthread, tmp1, tmp2, stub);
57 }
58
59 static void write_barrier_post(MacroAssembler* masm,
60 const MachNode* node,
61 Register store_addr,
62 Register new_val,
63 Register tmp1,
64 Register tmp2) {
65 if (!G1BarrierStubC2::needs_post_barrier(node)) {
66 return;
67 }
68 Assembler::InlineSkippedInstructionsCounter skip_counter(masm);
69 G1BarrierSetAssembler* g1_asm = static_cast<G1BarrierSetAssembler*>(BarrierSet::barrier_set()->barrier_set_assembler());
70 bool new_val_may_be_null = G1BarrierStubC2::post_new_val_may_be_null(node);
71 g1_asm->g1_write_barrier_post_c2(masm, store_addr, new_val, rthread, tmp1, tmp2, new_val_may_be_null);
72 }
73
74 %}
75
76 // BEGIN This section of the file is automatically generated. Do not edit --------------
77
78 // This section is generated from g1_aarch64.m4
79
80
81 // This pattern is generated automatically from g1_aarch64.m4.
82 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
83 instruct g1StoreP(indirect mem, iRegP src, iRegPNoSp tmp1, iRegPNoSp tmp2, iRegPNoSp tmp3, rFlagsReg cr)
84 %{
85 predicate(UseG1GC && !needs_releasing_store(n) && n->as_Store()->barrier_data() != 0);
86 match(Set mem (StoreP mem src));
87 effect(TEMP tmp1, TEMP tmp2, TEMP tmp3, KILL cr);
88 ins_cost(INSN_COST);
89 format %{ "str $src, $mem\t# ptr" %}
90 ins_encode %{
91 write_barrier_pre(masm, this,
92 $mem$$Register /* obj */,
93 $tmp1$$Register /* pre_val */,
94 $tmp2$$Register /* tmp1 */,
95 $tmp3$$Register /* tmp2 */,
96 RegSet::of($mem$$Register, $src$$Register) /* preserve */);
97 __ str($src$$Register, $mem$$Register);
98 write_barrier_post(masm, this,
99 $mem$$Register /* store_addr */,
100 $src$$Register /* new_val */,
101 $tmp2$$Register /* tmp1 */,
102 $tmp3$$Register /* tmp2 */);
103 %}
104 ins_pipe(istore_reg_mem);
105 %}
106
107 // This pattern is generated automatically from g1_aarch64.m4.
108 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
109 instruct g1StorePVolatile(indirect mem, iRegP src, iRegPNoSp tmp1, iRegPNoSp tmp2, iRegPNoSp tmp3, rFlagsReg cr)
110 %{
111 predicate(UseG1GC && needs_releasing_store(n) && n->as_Store()->barrier_data() != 0);
112 match(Set mem (StoreP mem src));
113 effect(TEMP tmp1, TEMP tmp2, TEMP tmp3, KILL cr);
114 ins_cost(VOLATILE_REF_COST);
115 format %{ "stlr $src, $mem\t# ptr" %}
116 ins_encode %{
117 write_barrier_pre(masm, this,
118 $mem$$Register /* obj */,
119 $tmp1$$Register /* pre_val */,
120 $tmp2$$Register /* tmp1 */,
121 $tmp3$$Register /* tmp2 */,
122 RegSet::of($mem$$Register, $src$$Register) /* preserve */);
123 __ stlr($src$$Register, $mem$$Register);
124 write_barrier_post(masm, this,
125 $mem$$Register /* store_addr */,
126 $src$$Register /* new_val */,
127 $tmp2$$Register /* tmp1 */,
128 $tmp3$$Register /* tmp2 */);
129 %}
130 ins_pipe(pipe_class_memory);
131 %}
132
133 // This pattern is generated automatically from g1_aarch64.m4.
134 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
135 instruct g1StoreN(indirect mem, iRegN src, iRegPNoSp tmp1, iRegPNoSp tmp2, iRegPNoSp tmp3, rFlagsReg cr)
136 %{
137 predicate(UseG1GC && !needs_releasing_store(n) && n->as_Store()->barrier_data() != 0);
138 match(Set mem (StoreN mem src));
139 effect(TEMP tmp1, TEMP tmp2, TEMP tmp3, KILL cr);
140 ins_cost(INSN_COST);
141 format %{ "strw $src, $mem\t# compressed ptr" %}
142 ins_encode %{
143 write_barrier_pre(masm, this,
144 $mem$$Register /* obj */,
145 $tmp1$$Register /* pre_val */,
146 $tmp2$$Register /* tmp1 */,
147 $tmp3$$Register /* tmp2 */,
148 RegSet::of($mem$$Register, $src$$Register) /* preserve */);
149 __ strw($src$$Register, $mem$$Register);
150 if ((barrier_data() & G1C2BarrierPost) != 0) {
151 if ((barrier_data() & G1C2BarrierPostNotNull) == 0) {
152 __ decode_heap_oop($tmp1$$Register, $src$$Register);
153 } else {
154 __ decode_heap_oop_not_null($tmp1$$Register, $src$$Register);
155 }
156 }
157 write_barrier_post(masm, this,
158 $mem$$Register /* store_addr */,
159 $tmp1$$Register /* new_val */,
160 $tmp2$$Register /* tmp1 */,
161 $tmp3$$Register /* tmp2 */);
162 %}
163 ins_pipe(istore_reg_mem);
164 %}
165
166 // This pattern is generated automatically from g1_aarch64.m4.
167 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
168 instruct g1StoreNVolatile(indirect mem, iRegN src, iRegPNoSp tmp1, iRegPNoSp tmp2, iRegPNoSp tmp3, rFlagsReg cr)
169 %{
170 predicate(UseG1GC && needs_releasing_store(n) && n->as_Store()->barrier_data() != 0);
171 match(Set mem (StoreN mem src));
172 effect(TEMP tmp1, TEMP tmp2, TEMP tmp3, KILL cr);
173 ins_cost(VOLATILE_REF_COST);
174 format %{ "stlrw $src, $mem\t# compressed ptr" %}
175 ins_encode %{
176 write_barrier_pre(masm, this,
177 $mem$$Register /* obj */,
178 $tmp1$$Register /* pre_val */,
179 $tmp2$$Register /* tmp1 */,
180 $tmp3$$Register /* tmp2 */,
181 RegSet::of($mem$$Register, $src$$Register) /* preserve */);
182 __ stlrw($src$$Register, $mem$$Register);
183 if ((barrier_data() & G1C2BarrierPost) != 0) {
184 if ((barrier_data() & G1C2BarrierPostNotNull) == 0) {
185 __ decode_heap_oop($tmp1$$Register, $src$$Register);
186 } else {
187 __ decode_heap_oop_not_null($tmp1$$Register, $src$$Register);
188 }
189 }
190 write_barrier_post(masm, this,
191 $mem$$Register /* store_addr */,
192 $tmp1$$Register /* new_val */,
193 $tmp2$$Register /* tmp1 */,
194 $tmp3$$Register /* tmp2 */);
195 %}
196 ins_pipe(pipe_class_memory);
197 %}
198
199 // This pattern is generated automatically from g1_aarch64.m4.
200 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
201 instruct g1EncodePAndStoreN(indirect mem, iRegP src, iRegPNoSp tmp1, iRegPNoSp tmp2, iRegPNoSp tmp3, rFlagsReg cr)
202 %{
203 predicate(UseG1GC && !needs_releasing_store(n) && n->as_Store()->barrier_data() != 0);
204 match(Set mem (StoreN mem (EncodeP src)));
205 effect(TEMP tmp1, TEMP tmp2, TEMP tmp3, KILL cr);
206 ins_cost(INSN_COST);
207 format %{ "encode_heap_oop $tmp1, $src\n\t"
208 "strw $tmp1, $mem\t# compressed ptr" %}
209 ins_encode %{
210 write_barrier_pre(masm, this,
211 $mem$$Register /* obj */,
212 $tmp1$$Register /* pre_val */,
213 $tmp2$$Register /* tmp1 */,
214 $tmp3$$Register /* tmp2 */,
215 RegSet::of($mem$$Register, $src$$Register) /* preserve */);
216 if ((barrier_data() & G1C2BarrierPostNotNull) == 0) {
217 __ encode_heap_oop($tmp1$$Register, $src$$Register);
218 } else {
219 __ encode_heap_oop_not_null($tmp1$$Register, $src$$Register);
220 }
221 __ strw($tmp1$$Register, $mem$$Register);
222 write_barrier_post(masm, this,
223 $mem$$Register /* store_addr */,
224 $src$$Register /* new_val */,
225 $tmp2$$Register /* tmp1 */,
226 $tmp3$$Register /* tmp2 */);
227 %}
228 ins_pipe(istore_reg_mem);
229 %}
230
231 // This pattern is generated automatically from g1_aarch64.m4.
232 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
233 instruct g1EncodePAndStoreNVolatile(indirect mem, iRegP src, iRegPNoSp tmp1, iRegPNoSp tmp2, iRegPNoSp tmp3, rFlagsReg cr)
234 %{
235 predicate(UseG1GC && needs_releasing_store(n) && n->as_Store()->barrier_data() != 0);
236 match(Set mem (StoreN mem (EncodeP src)));
237 effect(TEMP tmp1, TEMP tmp2, TEMP tmp3, KILL cr);
238 ins_cost(VOLATILE_REF_COST);
239 format %{ "encode_heap_oop $tmp1, $src\n\t"
240 "stlrw $tmp1, $mem\t# compressed ptr" %}
241 ins_encode %{
242 write_barrier_pre(masm, this,
243 $mem$$Register /* obj */,
244 $tmp1$$Register /* pre_val */,
245 $tmp2$$Register /* tmp1 */,
246 $tmp3$$Register /* tmp2 */,
247 RegSet::of($mem$$Register, $src$$Register) /* preserve */);
248 if ((barrier_data() & G1C2BarrierPostNotNull) == 0) {
249 __ encode_heap_oop($tmp1$$Register, $src$$Register);
250 } else {
251 __ encode_heap_oop_not_null($tmp1$$Register, $src$$Register);
252 }
253 __ stlrw($tmp1$$Register, $mem$$Register);
254 write_barrier_post(masm, this,
255 $mem$$Register /* store_addr */,
256 $src$$Register /* new_val */,
257 $tmp2$$Register /* tmp1 */,
258 $tmp3$$Register /* tmp2 */);
259 %}
260 ins_pipe(pipe_class_memory);
261 %}
262
263 // This pattern is generated automatically from g1_aarch64.m4.
264 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
265 instruct g1CompareAndExchangeP(iRegPNoSp res, indirect mem, iRegP oldval, iRegP newval, iRegPNoSp tmp1, iRegPNoSp tmp2, rFlagsReg cr)
266 %{
267 predicate(UseG1GC && !needs_acquiring_load_exclusive(n) && n->as_LoadStore()->barrier_data() != 0);
268 match(Set res (CompareAndExchangeP mem (Binary oldval newval)));
269 effect(TEMP res, TEMP tmp1, TEMP tmp2, KILL cr);
270 ins_cost(2 * VOLATILE_REF_COST);
271 format %{ "cmpxchg $res = $mem, $oldval, $newval\t# ptr" %}
272 ins_encode %{
273 assert_different_registers($oldval$$Register, $mem$$Register);
274 assert_different_registers($newval$$Register, $mem$$Register);
275 // Pass $oldval to the pre-barrier (instead of loading from $mem), because
276 // $oldval is the only value that can be overwritten.
277 // The same holds for g1CompareAndSwapP and its Acq variant.
278 write_barrier_pre(masm, this,
279 noreg /* obj */,
280 $oldval$$Register /* pre_val */,
281 $tmp1$$Register /* tmp1 */,
282 $tmp2$$Register /* tmp2 */,
283 RegSet::of($mem$$Register, $oldval$$Register, $newval$$Register) /* preserve */,
284 RegSet::of($res$$Register) /* no_preserve */);
285 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register, Assembler::xword,
286 memory_order_release, $res$$Register);
287 write_barrier_post(masm, this,
288 $mem$$Register /* store_addr */,
289 $newval$$Register /* new_val */,
290 $tmp1$$Register /* tmp1 */,
291 $tmp2$$Register /* tmp2 */);
292 %}
293 ins_pipe(pipe_slow);
294 %}
295
296 // This pattern is generated automatically from g1_aarch64.m4.
297 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
298 instruct g1CompareAndExchangePAcq(iRegPNoSp res, indirect mem, iRegP oldval, iRegP newval, iRegPNoSp tmp1, iRegPNoSp tmp2, rFlagsReg cr)
299 %{
300 predicate(UseG1GC && needs_acquiring_load_exclusive(n) && n->as_LoadStore()->barrier_data() != 0);
301 match(Set res (CompareAndExchangeP mem (Binary oldval newval)));
302 effect(TEMP res, TEMP tmp1, TEMP tmp2, KILL cr);
303 ins_cost(VOLATILE_REF_COST);
304 format %{ "cmpxchg_acq $res = $mem, $oldval, $newval\t# ptr" %}
305 ins_encode %{
306 assert_different_registers($oldval$$Register, $mem$$Register);
307 assert_different_registers($newval$$Register, $mem$$Register);
308 // Pass $oldval to the pre-barrier (instead of loading from $mem), because
309 // $oldval is the only value that can be overwritten.
310 // The same holds for g1CompareAndSwapP and its Acq variant.
311 write_barrier_pre(masm, this,
312 noreg /* obj */,
313 $oldval$$Register /* pre_val */,
314 $tmp1$$Register /* tmp1 */,
315 $tmp2$$Register /* tmp2 */,
316 RegSet::of($mem$$Register, $oldval$$Register, $newval$$Register) /* preserve */,
317 RegSet::of($res$$Register) /* no_preserve */);
318 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register, Assembler::xword,
319 memory_order_seq_cst, $res$$Register);
320 write_barrier_post(masm, this,
321 $mem$$Register /* store_addr */,
322 $newval$$Register /* new_val */,
323 $tmp1$$Register /* tmp1 */,
324 $tmp2$$Register /* tmp2 */);
325 %}
326 ins_pipe(pipe_slow);
327 %}
328
329 // This pattern is generated automatically from g1_aarch64.m4.
330 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
331 instruct g1CompareAndExchangeN(iRegNNoSp res, indirect mem, iRegN oldval, iRegN newval, iRegPNoSp tmp1, iRegPNoSp tmp2, iRegPNoSp tmp3, rFlagsReg cr)
332 %{
333 predicate(UseG1GC && !needs_acquiring_load_exclusive(n) && n->as_LoadStore()->barrier_data() != 0);
334 match(Set res (CompareAndExchangeN mem (Binary oldval newval)));
335 effect(TEMP res, TEMP tmp1, TEMP tmp2, TEMP tmp3, KILL cr);
336 ins_cost(2 * VOLATILE_REF_COST);
337 format %{ "cmpxchg $res = $mem, $oldval, $newval\t# narrow oop" %}
338 ins_encode %{
339 assert_different_registers($oldval$$Register, $mem$$Register);
340 assert_different_registers($newval$$Register, $mem$$Register);
341 write_barrier_pre(masm, this,
342 $mem$$Register /* obj */,
343 $tmp1$$Register /* pre_val */,
344 $tmp2$$Register /* tmp1 */,
345 $tmp3$$Register /* tmp2 */,
346 RegSet::of($mem$$Register, $oldval$$Register, $newval$$Register) /* preserve */,
347 RegSet::of($res$$Register) /* no_preserve */);
348 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register, Assembler::word,
349 memory_order_release, $res$$Register);
350 __ decode_heap_oop($tmp1$$Register, $newval$$Register);
351 write_barrier_post(masm, this,
352 $mem$$Register /* store_addr */,
353 $tmp1$$Register /* new_val */,
354 $tmp2$$Register /* tmp1 */,
355 $tmp3$$Register /* tmp2 */);
356 %}
357 ins_pipe(pipe_slow);
358 %}
359
360 // This pattern is generated automatically from g1_aarch64.m4.
361 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
362 instruct g1CompareAndExchangeNAcq(iRegNNoSp res, indirect mem, iRegN oldval, iRegN newval, iRegPNoSp tmp1, iRegPNoSp tmp2, iRegPNoSp tmp3, rFlagsReg cr)
363 %{
364 predicate(UseG1GC && needs_acquiring_load_exclusive(n) && n->as_LoadStore()->barrier_data() != 0);
365 match(Set res (CompareAndExchangeN mem (Binary oldval newval)));
366 effect(TEMP res, TEMP tmp1, TEMP tmp2, TEMP tmp3, KILL cr);
367 ins_cost(VOLATILE_REF_COST);
368 format %{ "cmpxchg_acq $res = $mem, $oldval, $newval\t# narrow oop" %}
369 ins_encode %{
370 assert_different_registers($oldval$$Register, $mem$$Register);
371 assert_different_registers($newval$$Register, $mem$$Register);
372 write_barrier_pre(masm, this,
373 $mem$$Register /* obj */,
374 $tmp1$$Register /* pre_val */,
375 $tmp2$$Register /* tmp1 */,
376 $tmp3$$Register /* tmp2 */,
377 RegSet::of($mem$$Register, $oldval$$Register, $newval$$Register) /* preserve */,
378 RegSet::of($res$$Register) /* no_preserve */);
379 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register, Assembler::word,
380 memory_order_seq_cst, $res$$Register);
381 __ decode_heap_oop($tmp1$$Register, $newval$$Register);
382 write_barrier_post(masm, this,
383 $mem$$Register /* store_addr */,
384 $tmp1$$Register /* new_val */,
385 $tmp2$$Register /* tmp1 */,
386 $tmp3$$Register /* tmp2 */);
387 %}
388 ins_pipe(pipe_slow);
389 %}
390
391 // This pattern is generated automatically from g1_aarch64.m4.
392 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
393 instruct g1CompareAndSwapP(iRegINoSp res, indirect mem, iRegP newval, iRegPNoSp tmp1, iRegPNoSp tmp2, iRegP oldval, rFlagsReg cr)
394 %{
395 predicate(UseG1GC && !needs_acquiring_load_exclusive(n) && n->as_LoadStore()->barrier_data() != 0);
396 match(Set res (CompareAndSwapP mem (Binary oldval newval)));
397 match(Set res (WeakCompareAndSwapP mem (Binary oldval newval)));
398 effect(TEMP res, TEMP tmp1, TEMP tmp2, KILL cr);
399 ins_cost(2 * VOLATILE_REF_COST);
400 format %{ "cmpxchg $mem, $oldval, $newval\t# (ptr)\n\t"
401 "cset $res, EQ" %}
402 ins_encode %{
403 assert_different_registers($oldval$$Register, $mem$$Register);
404 assert_different_registers($newval$$Register, $mem$$Register);
405 write_barrier_pre(masm, this,
406 noreg /* obj */,
407 $oldval$$Register /* pre_val */,
408 $tmp1$$Register /* tmp1 */,
409 $tmp2$$Register /* tmp2 */,
410 RegSet::of($mem$$Register, $oldval$$Register, $newval$$Register) /* preserve */,
411 RegSet::of($res$$Register) /* no_preserve */);
412 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register, Assembler::xword, memory_order_release);
413 __ cset($res$$Register, Assembler::EQ);
414 write_barrier_post(masm, this,
415 $mem$$Register /* store_addr */,
416 $newval$$Register /* new_val */,
417 $tmp1$$Register /* tmp1 */,
418 $tmp2$$Register /* tmp2 */);
419 %}
420 ins_pipe(pipe_slow);
421 %}
422
423 // This pattern is generated automatically from g1_aarch64.m4.
424 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
425 instruct g1CompareAndSwapPAcq(iRegINoSp res, indirect mem, iRegP newval, iRegPNoSp tmp1, iRegPNoSp tmp2, iRegP oldval, rFlagsReg cr)
426 %{
427 predicate(UseG1GC && needs_acquiring_load_exclusive(n) && n->as_LoadStore()->barrier_data() != 0);
428 match(Set res (CompareAndSwapP mem (Binary oldval newval)));
429 match(Set res (WeakCompareAndSwapP mem (Binary oldval newval)));
430 effect(TEMP res, TEMP tmp1, TEMP tmp2, KILL cr);
431 ins_cost(VOLATILE_REF_COST);
432 format %{ "cmpxchg_acq $mem, $oldval, $newval\t# (ptr)\n\t"
433 "cset $res, EQ" %}
434 ins_encode %{
435 assert_different_registers($oldval$$Register, $mem$$Register);
436 assert_different_registers($newval$$Register, $mem$$Register);
437 write_barrier_pre(masm, this,
438 noreg /* obj */,
439 $oldval$$Register /* pre_val */,
440 $tmp1$$Register /* tmp1 */,
441 $tmp2$$Register /* tmp2 */,
442 RegSet::of($mem$$Register, $oldval$$Register, $newval$$Register) /* preserve */,
443 RegSet::of($res$$Register) /* no_preserve */);
444 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register, Assembler::xword, memory_order_seq_cst);
445 __ cset($res$$Register, Assembler::EQ);
446 write_barrier_post(masm, this,
447 $mem$$Register /* store_addr */,
448 $newval$$Register /* new_val */,
449 $tmp1$$Register /* tmp1 */,
450 $tmp2$$Register /* tmp2 */);
451 %}
452 ins_pipe(pipe_slow);
453 %}
454
455 // This pattern is generated automatically from g1_aarch64.m4.
456 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
457 instruct g1CompareAndSwapN(iRegINoSp res, indirect mem, iRegN newval, iRegPNoSp tmp1, iRegPNoSp tmp2, iRegPNoSp tmp3, iRegN oldval, rFlagsReg cr)
458 %{
459 predicate(UseG1GC && !needs_acquiring_load_exclusive(n) && n->as_LoadStore()->barrier_data() != 0);
460 match(Set res (CompareAndSwapN mem (Binary oldval newval)));
461 match(Set res (WeakCompareAndSwapN mem (Binary oldval newval)));
462 effect(TEMP res, TEMP tmp1, TEMP tmp2, TEMP tmp3, KILL cr);
463 ins_cost(2 * VOLATILE_REF_COST);
464 format %{ "cmpxchg $mem, $oldval, $newval\t# (narrow oop)\n\t"
465 "cset $res, EQ" %}
466 ins_encode %{
467 assert_different_registers($oldval$$Register, $mem$$Register);
468 assert_different_registers($newval$$Register, $mem$$Register);
469 write_barrier_pre(masm, this,
470 $mem$$Register /* obj */,
471 $tmp1$$Register /* pre_val */,
472 $tmp2$$Register /* tmp1 */,
473 $tmp3$$Register /* tmp2 */,
474 RegSet::of($mem$$Register, $oldval$$Register, $newval$$Register) /* preserve */,
475 RegSet::of($res$$Register) /* no_preserve */);
476 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register, Assembler::word, memory_order_release);
477 __ cset($res$$Register, Assembler::EQ);
478 __ decode_heap_oop($tmp1$$Register, $newval$$Register);
479 write_barrier_post(masm, this,
480 $mem$$Register /* store_addr */,
481 $tmp1$$Register /* new_val */,
482 $tmp2$$Register /* tmp1 */,
483 $tmp3$$Register /* tmp2 */);
484 %}
485 ins_pipe(pipe_slow);
486 %}
487
488 // This pattern is generated automatically from g1_aarch64.m4.
489 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
490 instruct g1CompareAndSwapNAcq(iRegINoSp res, indirect mem, iRegN newval, iRegPNoSp tmp1, iRegPNoSp tmp2, iRegPNoSp tmp3, iRegN oldval, rFlagsReg cr)
491 %{
492 predicate(UseG1GC && needs_acquiring_load_exclusive(n) && n->as_LoadStore()->barrier_data() != 0);
493 match(Set res (CompareAndSwapN mem (Binary oldval newval)));
494 match(Set res (WeakCompareAndSwapN mem (Binary oldval newval)));
495 effect(TEMP res, TEMP tmp1, TEMP tmp2, TEMP tmp3, KILL cr);
496 ins_cost(VOLATILE_REF_COST);
497 format %{ "cmpxchg_acq $mem, $oldval, $newval\t# (narrow oop)\n\t"
498 "cset $res, EQ" %}
499 ins_encode %{
500 assert_different_registers($oldval$$Register, $mem$$Register);
501 assert_different_registers($newval$$Register, $mem$$Register);
502 write_barrier_pre(masm, this,
503 $mem$$Register /* obj */,
504 $tmp1$$Register /* pre_val */,
505 $tmp2$$Register /* tmp1 */,
506 $tmp3$$Register /* tmp2 */,
507 RegSet::of($mem$$Register, $oldval$$Register, $newval$$Register) /* preserve */,
508 RegSet::of($res$$Register) /* no_preserve */);
509 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register, Assembler::word, memory_order_seq_cst);
510 __ cset($res$$Register, Assembler::EQ);
511 __ decode_heap_oop($tmp1$$Register, $newval$$Register);
512 write_barrier_post(masm, this,
513 $mem$$Register /* store_addr */,
514 $tmp1$$Register /* new_val */,
515 $tmp2$$Register /* tmp1 */,
516 $tmp3$$Register /* tmp2 */);
517 %}
518 ins_pipe(pipe_slow);
519 %}
520
521 // This pattern is generated automatically from g1_aarch64.m4.
522 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
523 instruct g1GetAndSetP(indirect mem, iRegP newval, iRegPNoSp tmp1, iRegPNoSp tmp2, iRegPNoSp preval, rFlagsReg cr)
524 %{
525 predicate(UseG1GC && !needs_acquiring_load_exclusive(n) && n->as_LoadStore()->barrier_data() != 0);
526 match(Set preval (GetAndSetP mem newval));
527 effect(TEMP preval, TEMP tmp1, TEMP tmp2, KILL cr);
528 ins_cost(2 * VOLATILE_REF_COST);
529 format %{ "atomic_xchg $preval, $newval, [$mem]" %}
530 ins_encode %{
531 assert_different_registers($mem$$Register, $newval$$Register);
532 write_barrier_pre(masm, this,
533 $mem$$Register /* obj */,
534 $preval$$Register /* pre_val (as a temporary register) */,
535 $tmp1$$Register /* tmp1 */,
536 $tmp2$$Register /* tmp2 */,
537 RegSet::of($mem$$Register, $preval$$Register, $newval$$Register) /* preserve */);
538 __ atomic_xchg($preval$$Register, $newval$$Register, $mem$$Register);
539 write_barrier_post(masm, this,
540 $mem$$Register /* store_addr */,
541 $newval$$Register /* new_val */,
542 $tmp1$$Register /* tmp1 */,
543 $tmp2$$Register /* tmp2 */);
544 %}
545 ins_pipe(pipe_serial);
546 %}
547
548 // This pattern is generated automatically from g1_aarch64.m4.
549 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
550 instruct g1GetAndSetPAcq(indirect mem, iRegP newval, iRegPNoSp tmp1, iRegPNoSp tmp2, iRegPNoSp preval, rFlagsReg cr)
551 %{
552 predicate(UseG1GC && needs_acquiring_load_exclusive(n) && n->as_LoadStore()->barrier_data() != 0);
553 match(Set preval (GetAndSetP mem newval));
554 effect(TEMP preval, TEMP tmp1, TEMP tmp2, KILL cr);
555 ins_cost(VOLATILE_REF_COST);
556 format %{ "atomic_xchg_acq $preval, $newval, [$mem]" %}
557 ins_encode %{
558 assert_different_registers($mem$$Register, $newval$$Register);
559 write_barrier_pre(masm, this,
560 $mem$$Register /* obj */,
561 $preval$$Register /* pre_val (as a temporary register) */,
562 $tmp1$$Register /* tmp1 */,
563 $tmp2$$Register /* tmp2 */,
564 RegSet::of($mem$$Register, $preval$$Register, $newval$$Register) /* preserve */);
565 __ atomic_xchgal($preval$$Register, $newval$$Register, $mem$$Register);
566 write_barrier_post(masm, this,
567 $mem$$Register /* store_addr */,
568 $newval$$Register /* new_val */,
569 $tmp1$$Register /* tmp1 */,
570 $tmp2$$Register /* tmp2 */);
571 %}
572 ins_pipe(pipe_serial);
573 %}
574
575 // This pattern is generated automatically from g1_aarch64.m4.
576 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
577 instruct g1GetAndSetN(indirect mem, iRegN newval, iRegPNoSp tmp1, iRegPNoSp tmp2, iRegPNoSp tmp3, iRegNNoSp preval, rFlagsReg cr)
578 %{
579 predicate(UseG1GC && !needs_acquiring_load_exclusive(n) && n->as_LoadStore()->barrier_data() != 0);
580 match(Set preval (GetAndSetN mem newval));
581 effect(TEMP preval, TEMP tmp1, TEMP tmp2, TEMP tmp3, KILL cr);
582 ins_cost(2 * VOLATILE_REF_COST);
583 format %{ "atomic_xchgw $preval, $newval, [$mem]" %}
584 ins_encode %{
585 assert_different_registers($mem$$Register, $newval$$Register);
586 write_barrier_pre(masm, this,
587 $mem$$Register /* obj */,
588 $tmp1$$Register /* pre_val */,
589 $tmp2$$Register /* tmp1 */,
590 $tmp3$$Register /* tmp2 */,
591 RegSet::of($mem$$Register, $preval$$Register, $newval$$Register) /* preserve */);
592 __ atomic_xchgw($preval$$Register, $newval$$Register, $mem$$Register);
593 __ decode_heap_oop($tmp1$$Register, $newval$$Register);
594 write_barrier_post(masm, this,
595 $mem$$Register /* store_addr */,
596 $tmp1$$Register /* new_val */,
597 $tmp2$$Register /* tmp1 */,
598 $tmp3$$Register /* tmp2 */);
599 %}
600 ins_pipe(pipe_serial);
601 %}
602
603 // This pattern is generated automatically from g1_aarch64.m4.
604 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
605 instruct g1GetAndSetNAcq(indirect mem, iRegN newval, iRegPNoSp tmp1, iRegPNoSp tmp2, iRegPNoSp tmp3, iRegNNoSp preval, rFlagsReg cr)
606 %{
607 predicate(UseG1GC && needs_acquiring_load_exclusive(n) && n->as_LoadStore()->barrier_data() != 0);
608 match(Set preval (GetAndSetN mem newval));
609 effect(TEMP preval, TEMP tmp1, TEMP tmp2, TEMP tmp3, KILL cr);
610 ins_cost(VOLATILE_REF_COST);
611 format %{ "atomic_xchgw_acq $preval, $newval, [$mem]" %}
612 ins_encode %{
613 assert_different_registers($mem$$Register, $newval$$Register);
614 write_barrier_pre(masm, this,
615 $mem$$Register /* obj */,
616 $tmp1$$Register /* pre_val */,
617 $tmp2$$Register /* tmp1 */,
618 $tmp3$$Register /* tmp2 */,
619 RegSet::of($mem$$Register, $preval$$Register, $newval$$Register) /* preserve */);
620 __ atomic_xchgalw($preval$$Register, $newval$$Register, $mem$$Register);
621 __ decode_heap_oop($tmp1$$Register, $newval$$Register);
622 write_barrier_post(masm, this,
623 $mem$$Register /* store_addr */,
624 $tmp1$$Register /* new_val */,
625 $tmp2$$Register /* tmp1 */,
626 $tmp3$$Register /* tmp2 */);
627 %}
628 ins_pipe(pipe_serial);
629 %}
630
631 // This pattern is generated automatically from g1_aarch64.m4.
632 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
633 instruct g1LoadP(iRegPNoSp dst, indirect mem, iRegPNoSp tmp1, iRegPNoSp tmp2, rFlagsReg cr)
634 %{
635 // This instruction does not need an acquiring counterpart because it is only
636 // used for reference loading (Reference::get()). The same holds for g1LoadN.
637 predicate(UseG1GC && !needs_acquiring_load(n) && n->as_Load()->barrier_data() != 0);
638 match(Set dst (LoadP mem));
639 effect(TEMP dst, TEMP tmp1, TEMP tmp2, KILL cr);
640 ins_cost(4 * INSN_COST);
641 format %{ "ldr $dst, $mem\t# ptr" %}
642 ins_encode %{
643 __ ldr($dst$$Register, $mem$$Register);
644 write_barrier_pre(masm, this,
645 noreg /* obj */,
646 $dst$$Register /* pre_val */,
647 $tmp1$$Register /* tmp1 */,
648 $tmp2$$Register /* tmp2 */);
649 %}
650 ins_pipe(iload_reg_mem);
651 %}
652
653 // This pattern is generated automatically from g1_aarch64.m4.
654 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
655 instruct g1LoadN(iRegNNoSp dst, indirect mem, iRegPNoSp tmp1, iRegPNoSp tmp2, iRegPNoSp tmp3, rFlagsReg cr)
656 %{
657 predicate(UseG1GC && !needs_acquiring_load(n) && n->as_Load()->barrier_data() != 0);
658 match(Set dst (LoadN mem));
659 effect(TEMP dst, TEMP tmp1, TEMP tmp2, TEMP tmp3, KILL cr);
660 ins_cost(4 * INSN_COST);
661 format %{ "ldrw $dst, $mem\t# compressed ptr" %}
662 ins_encode %{
663 __ ldrw($dst$$Register, $mem$$Register);
664 if ((barrier_data() & G1C2BarrierPre) != 0) {
665 __ decode_heap_oop($tmp1$$Register, $dst$$Register);
666 write_barrier_pre(masm, this,
667 noreg /* obj */,
668 $tmp1$$Register /* pre_val */,
669 $tmp2$$Register /* tmp1 */,
670 $tmp3$$Register /* tmp2 */);
671 }
672 %}
673 ins_pipe(iload_reg_mem);
674 %}
675
676 // END This section of the file is automatically generated. Do not edit --------------