1 //
2 // Copyright (c) 2024, 2025, Oracle and/or its affiliates. All rights reserved.
3 // DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
4 //
5 // This code is free software; you can redistribute it and/or modify it
6 // under the terms of the GNU General Public License version 2 only, as
7 // published by the Free Software Foundation.
8 //
9 // This code is distributed in the hope that it will be useful, but WITHOUT
10 // ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 // FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
12 // version 2 for more details (a copy is included in the LICENSE file that
13 // accompanied this code).
14 //
15 // You should have received a copy of the GNU General Public License version
16 // 2 along with this work; if not, write to the Free Software Foundation,
17 // Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
18 //
19 // Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
20 // or visit www.oracle.com if you need additional information or have any
21 // questions.
22 //
23
24 source_hpp %{
25
26 #include "gc/g1/c2/g1BarrierSetC2.hpp"
27 #include "gc/shared/gc_globals.hpp"
28
29 %}
30
31 source %{
32
33 #include "gc/g1/g1BarrierSetAssembler_aarch64.hpp"
34 #include "gc/g1/g1BarrierSetRuntime.hpp"
35
36 static void write_barrier_pre(MacroAssembler* masm,
37 const MachNode* node,
38 Register obj,
39 Register pre_val,
40 Register tmp1,
41 Register tmp2,
42 RegSet preserve = RegSet(),
43 RegSet no_preserve = RegSet()) {
44 if (!G1PreBarrierStubC2::needs_barrier(node)) {
45 return;
46 }
47 Assembler::InlineSkippedInstructionsCounter skip_counter(masm);
48 G1BarrierSetAssembler* g1_asm = static_cast<G1BarrierSetAssembler*>(BarrierSet::barrier_set()->barrier_set_assembler());
49 G1PreBarrierStubC2* const stub = G1PreBarrierStubC2::create(node);
50 for (RegSetIterator<Register> reg = preserve.begin(); *reg != noreg; ++reg) {
51 stub->preserve(*reg);
52 }
53 for (RegSetIterator<Register> reg = no_preserve.begin(); *reg != noreg; ++reg) {
54 stub->dont_preserve(*reg);
55 }
56 g1_asm->g1_write_barrier_pre_c2(masm, obj, pre_val, rthread, tmp1, tmp2, stub);
57 }
58
59 static void write_barrier_post(MacroAssembler* masm,
60 const MachNode* node,
61 Register store_addr,
62 Register new_val,
63 Register tmp1,
64 Register tmp2) {
65 if (!G1BarrierStubC2::needs_post_barrier(node)) {
66 return;
67 }
68 Assembler::InlineSkippedInstructionsCounter skip_counter(masm);
69 G1BarrierSetAssembler* g1_asm = static_cast<G1BarrierSetAssembler*>(BarrierSet::barrier_set()->barrier_set_assembler());
70 bool new_val_may_be_null = G1BarrierStubC2::post_new_val_may_be_null(node);
71 g1_asm->g1_write_barrier_post_c2(masm, store_addr, new_val, rthread, tmp1, tmp2, new_val_may_be_null);
72 }
73
74 %}
75
76 // TODO 8350865 (same applies to g1StoreLSpecialTwoOops)
77 // - Can we use an unbound register for src?
78 // - Do no set/overwrite barrier data here, also handle G1C2BarrierPostNotNull
79 // - Is the zero-extend really required in all the places?
80 // - Move this into the .m4?
81 instruct g1StoreLSpecialOneOop(indirect mem, iRegL_R11 src, immI off, iRegPNoSp tmp1, iRegPNoSp tmp2, iRegPNoSp tmp3, iRegPNoSp tmp4, rFlagsReg cr)
82 %{
83 predicate(UseG1GC);
84 match(Set mem (StoreLSpecial mem (Binary src off)));
85 effect(TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, USE_KILL src, KILL cr);
86 ins_cost(INSN_COST);
87 format %{ "str $src, $mem\t# g1StoreLSpecialOneOop" %}
88 ins_encode %{
89 ((MachNode*)this)->set_barrier_data(G1C2BarrierPre | G1C2BarrierPost);
90
91 // Adjust address to point to narrow oop
92 __ add($tmp4$$Register, $mem$$Register, $off$$constant);
93 write_barrier_pre(masm, this,
94 $tmp4$$Register /* obj */,
95 $tmp1$$Register /* pre_val */,
96 $tmp2$$Register /* tmp1 */,
97 $tmp3$$Register /* tmp2 */);
98
99 __ str($src$$Register, $mem$$Register);
100
101 // Shift long value to extract the narrow oop field value and zero-extend it
102 __ lsr($src$$Register, $src$$Register, $off$$constant << LogBitsPerByte);
103 __ ubfm($src$$Register, $src$$Register, 0, 31);
104
105 write_barrier_post(masm, this,
106 $tmp4$$Register /* store_addr */,
107 $src$$Register /* new_val */,
108 $tmp2$$Register /* tmp1 */,
109 $tmp3$$Register /* tmp2 */);
110 %}
111 ins_pipe(istore_reg_mem);
112 %}
113
114 instruct g1StoreLSpecialTwoOops(indirect mem, iRegL_R11 src, iRegPNoSp tmp1, iRegPNoSp tmp2, iRegPNoSp tmp3, iRegPNoSp tmp4, rFlagsReg cr)
115 %{
116 predicate(UseG1GC);
117 match(Set mem (StoreLSpecial mem src));
118 effect(TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, USE_KILL src, KILL cr);
119 ins_cost(INSN_COST);
120 format %{ "str $src, $mem\t# g1StoreLSpecialTwoOops" %}
121 ins_encode %{
122 ((MachNode*)this)->set_barrier_data(G1C2BarrierPre | G1C2BarrierPost);
123
124 write_barrier_pre(masm, this,
125 $mem$$Register /* obj */,
126 $tmp1$$Register /* pre_val */,
127 $tmp2$$Register /* tmp1 */,
128 $tmp3$$Register /* tmp2 */,
129 RegSet::of($mem$$Register, $src$$Register) /* preserve */);
130 // Adjust address to point to the second narrow oop in the long value
131 __ add($tmp4$$Register, $mem$$Register, 4);
132 write_barrier_pre(masm, this,
133 $tmp4$$Register /* obj */,
134 $tmp1$$Register /* pre_val */,
135 $tmp2$$Register /* tmp1 */,
136 $tmp3$$Register /* tmp2 */,
137 RegSet::of($mem$$Register, $src$$Register, $tmp4$$Register) /* preserve */);
138
139 __ str($src$$Register, $mem$$Register);
140
141 // Zero-extend first narrow oop to long
142 __ ubfm($tmp1$$Register, $src$$Register, 0, 31);
143
144 // Shift long value to extract the second narrow oop field value
145 __ lsr($src$$Register, $src$$Register, 32);
146 write_barrier_post(masm, this,
147 $mem$$Register /* store_addr */,
148 $tmp1$$Register /* new_val */,
149 $tmp2$$Register /* tmp1 */,
150 $tmp3$$Register /* tmp2 */);
151 write_barrier_post(masm, this,
152 $tmp4$$Register /* store_addr */,
153 $src$$Register /* new_val */,
154 $tmp2$$Register /* tmp1 */,
155 $tmp3$$Register /* tmp2 */);
156 %}
157 ins_pipe(istore_reg_mem);
158 %}
159
160
161 // BEGIN This section of the file is automatically generated. Do not edit --------------
162
163 // This section is generated from g1_aarch64.m4
164
165
166 // This pattern is generated automatically from g1_aarch64.m4.
167 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
168 instruct g1StoreP(indirect mem, iRegP src, iRegPNoSp tmp1, iRegPNoSp tmp2, iRegPNoSp tmp3, rFlagsReg cr)
169 %{
170 predicate(UseG1GC && !needs_releasing_store(n) && n->as_Store()->barrier_data() != 0);
171 match(Set mem (StoreP mem src));
172 effect(TEMP tmp1, TEMP tmp2, TEMP tmp3, KILL cr);
173 ins_cost(INSN_COST);
174 format %{ "str $src, $mem\t# ptr" %}
175 ins_encode %{
176 write_barrier_pre(masm, this,
177 $mem$$Register /* obj */,
178 $tmp1$$Register /* pre_val */,
179 $tmp2$$Register /* tmp1 */,
180 $tmp3$$Register /* tmp2 */,
181 RegSet::of($mem$$Register, $src$$Register) /* preserve */);
182 __ str($src$$Register, $mem$$Register);
183 write_barrier_post(masm, this,
184 $mem$$Register /* store_addr */,
185 $src$$Register /* new_val */,
186 $tmp2$$Register /* tmp1 */,
187 $tmp3$$Register /* tmp2 */);
188 %}
189 ins_pipe(istore_reg_mem);
190 %}
191
192 // This pattern is generated automatically from g1_aarch64.m4.
193 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
194 instruct g1StorePVolatile(indirect mem, iRegP src, iRegPNoSp tmp1, iRegPNoSp tmp2, iRegPNoSp tmp3, rFlagsReg cr)
195 %{
196 predicate(UseG1GC && needs_releasing_store(n) && n->as_Store()->barrier_data() != 0);
197 match(Set mem (StoreP mem src));
198 effect(TEMP tmp1, TEMP tmp2, TEMP tmp3, KILL cr);
199 ins_cost(VOLATILE_REF_COST);
200 format %{ "stlr $src, $mem\t# ptr" %}
201 ins_encode %{
202 write_barrier_pre(masm, this,
203 $mem$$Register /* obj */,
204 $tmp1$$Register /* pre_val */,
205 $tmp2$$Register /* tmp1 */,
206 $tmp3$$Register /* tmp2 */,
207 RegSet::of($mem$$Register, $src$$Register) /* preserve */);
208 __ stlr($src$$Register, $mem$$Register);
209 write_barrier_post(masm, this,
210 $mem$$Register /* store_addr */,
211 $src$$Register /* new_val */,
212 $tmp2$$Register /* tmp1 */,
213 $tmp3$$Register /* tmp2 */);
214 %}
215 ins_pipe(pipe_class_memory);
216 %}
217
218 // This pattern is generated automatically from g1_aarch64.m4.
219 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
220 instruct g1StoreN(indirect mem, iRegN src, iRegPNoSp tmp1, iRegPNoSp tmp2, iRegPNoSp tmp3, rFlagsReg cr)
221 %{
222 predicate(UseG1GC && !needs_releasing_store(n) && n->as_Store()->barrier_data() != 0);
223 match(Set mem (StoreN mem src));
224 effect(TEMP tmp1, TEMP tmp2, TEMP tmp3, KILL cr);
225 ins_cost(INSN_COST);
226 format %{ "strw $src, $mem\t# compressed ptr" %}
227 ins_encode %{
228 write_barrier_pre(masm, this,
229 $mem$$Register /* obj */,
230 $tmp1$$Register /* pre_val */,
231 $tmp2$$Register /* tmp1 */,
232 $tmp3$$Register /* tmp2 */,
233 RegSet::of($mem$$Register, $src$$Register) /* preserve */);
234 __ strw($src$$Register, $mem$$Register);
235 if ((barrier_data() & G1C2BarrierPost) != 0) {
236 if ((barrier_data() & G1C2BarrierPostNotNull) == 0) {
237 __ decode_heap_oop($tmp1$$Register, $src$$Register);
238 } else {
239 __ decode_heap_oop_not_null($tmp1$$Register, $src$$Register);
240 }
241 }
242 write_barrier_post(masm, this,
243 $mem$$Register /* store_addr */,
244 $tmp1$$Register /* new_val */,
245 $tmp2$$Register /* tmp1 */,
246 $tmp3$$Register /* tmp2 */);
247 %}
248 ins_pipe(istore_reg_mem);
249 %}
250
251 // This pattern is generated automatically from g1_aarch64.m4.
252 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
253 instruct g1StoreNVolatile(indirect mem, iRegN src, iRegPNoSp tmp1, iRegPNoSp tmp2, iRegPNoSp tmp3, rFlagsReg cr)
254 %{
255 predicate(UseG1GC && needs_releasing_store(n) && n->as_Store()->barrier_data() != 0);
256 match(Set mem (StoreN mem src));
257 effect(TEMP tmp1, TEMP tmp2, TEMP tmp3, KILL cr);
258 ins_cost(VOLATILE_REF_COST);
259 format %{ "stlrw $src, $mem\t# compressed ptr" %}
260 ins_encode %{
261 write_barrier_pre(masm, this,
262 $mem$$Register /* obj */,
263 $tmp1$$Register /* pre_val */,
264 $tmp2$$Register /* tmp1 */,
265 $tmp3$$Register /* tmp2 */,
266 RegSet::of($mem$$Register, $src$$Register) /* preserve */);
267 __ stlrw($src$$Register, $mem$$Register);
268 if ((barrier_data() & G1C2BarrierPost) != 0) {
269 if ((barrier_data() & G1C2BarrierPostNotNull) == 0) {
270 __ decode_heap_oop($tmp1$$Register, $src$$Register);
271 } else {
272 __ decode_heap_oop_not_null($tmp1$$Register, $src$$Register);
273 }
274 }
275 write_barrier_post(masm, this,
276 $mem$$Register /* store_addr */,
277 $tmp1$$Register /* new_val */,
278 $tmp2$$Register /* tmp1 */,
279 $tmp3$$Register /* tmp2 */);
280 %}
281 ins_pipe(pipe_class_memory);
282 %}
283
284 // This pattern is generated automatically from g1_aarch64.m4.
285 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
286 instruct g1EncodePAndStoreN(indirect mem, iRegP src, iRegPNoSp tmp1, iRegPNoSp tmp2, iRegPNoSp tmp3, rFlagsReg cr)
287 %{
288 predicate(UseG1GC && !needs_releasing_store(n) && n->as_Store()->barrier_data() != 0);
289 match(Set mem (StoreN mem (EncodeP src)));
290 effect(TEMP tmp1, TEMP tmp2, TEMP tmp3, KILL cr);
291 ins_cost(INSN_COST);
292 format %{ "encode_heap_oop $tmp1, $src\n\t"
293 "strw $tmp1, $mem\t# compressed ptr" %}
294 ins_encode %{
295 write_barrier_pre(masm, this,
296 $mem$$Register /* obj */,
297 $tmp1$$Register /* pre_val */,
298 $tmp2$$Register /* tmp1 */,
299 $tmp3$$Register /* tmp2 */,
300 RegSet::of($mem$$Register, $src$$Register) /* preserve */);
301 if ((barrier_data() & G1C2BarrierPostNotNull) == 0) {
302 __ encode_heap_oop($tmp1$$Register, $src$$Register);
303 } else {
304 __ encode_heap_oop_not_null($tmp1$$Register, $src$$Register);
305 }
306 __ strw($tmp1$$Register, $mem$$Register);
307 write_barrier_post(masm, this,
308 $mem$$Register /* store_addr */,
309 $src$$Register /* new_val */,
310 $tmp2$$Register /* tmp1 */,
311 $tmp3$$Register /* tmp2 */);
312 %}
313 ins_pipe(istore_reg_mem);
314 %}
315
316 // This pattern is generated automatically from g1_aarch64.m4.
317 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
318 instruct g1EncodePAndStoreNVolatile(indirect mem, iRegP src, iRegPNoSp tmp1, iRegPNoSp tmp2, iRegPNoSp tmp3, rFlagsReg cr)
319 %{
320 predicate(UseG1GC && needs_releasing_store(n) && n->as_Store()->barrier_data() != 0);
321 match(Set mem (StoreN mem (EncodeP src)));
322 effect(TEMP tmp1, TEMP tmp2, TEMP tmp3, KILL cr);
323 ins_cost(VOLATILE_REF_COST);
324 format %{ "encode_heap_oop $tmp1, $src\n\t"
325 "stlrw $tmp1, $mem\t# compressed ptr" %}
326 ins_encode %{
327 write_barrier_pre(masm, this,
328 $mem$$Register /* obj */,
329 $tmp1$$Register /* pre_val */,
330 $tmp2$$Register /* tmp1 */,
331 $tmp3$$Register /* tmp2 */,
332 RegSet::of($mem$$Register, $src$$Register) /* preserve */);
333 if ((barrier_data() & G1C2BarrierPostNotNull) == 0) {
334 __ encode_heap_oop($tmp1$$Register, $src$$Register);
335 } else {
336 __ encode_heap_oop_not_null($tmp1$$Register, $src$$Register);
337 }
338 __ stlrw($tmp1$$Register, $mem$$Register);
339 write_barrier_post(masm, this,
340 $mem$$Register /* store_addr */,
341 $src$$Register /* new_val */,
342 $tmp2$$Register /* tmp1 */,
343 $tmp3$$Register /* tmp2 */);
344 %}
345 ins_pipe(pipe_class_memory);
346 %}
347
348 // This pattern is generated automatically from g1_aarch64.m4.
349 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
350 instruct g1CompareAndExchangeP(iRegPNoSp res, indirect mem, iRegP oldval, iRegP newval, iRegPNoSp tmp1, iRegPNoSp tmp2, rFlagsReg cr)
351 %{
352 predicate(UseG1GC && !needs_acquiring_load_exclusive(n) && n->as_LoadStore()->barrier_data() != 0);
353 match(Set res (CompareAndExchangeP mem (Binary oldval newval)));
354 effect(TEMP res, TEMP tmp1, TEMP tmp2, KILL cr);
355 ins_cost(2 * VOLATILE_REF_COST);
356 format %{ "cmpxchg $res = $mem, $oldval, $newval\t# ptr" %}
357 ins_encode %{
358 assert_different_registers($oldval$$Register, $mem$$Register);
359 assert_different_registers($newval$$Register, $mem$$Register);
360 // Pass $oldval to the pre-barrier (instead of loading from $mem), because
361 // $oldval is the only value that can be overwritten.
362 // The same holds for g1CompareAndSwapP and its Acq variant.
363 write_barrier_pre(masm, this,
364 noreg /* obj */,
365 $oldval$$Register /* pre_val */,
366 $tmp1$$Register /* tmp1 */,
367 $tmp2$$Register /* tmp2 */,
368 RegSet::of($mem$$Register, $oldval$$Register, $newval$$Register) /* preserve */,
369 RegSet::of($res$$Register) /* no_preserve */);
370 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register, Assembler::xword,
371 false /* acquire */, true /* release */, false /* weak */, $res$$Register);
372 write_barrier_post(masm, this,
373 $mem$$Register /* store_addr */,
374 $newval$$Register /* new_val */,
375 $tmp1$$Register /* tmp1 */,
376 $tmp2$$Register /* tmp2 */);
377 %}
378 ins_pipe(pipe_slow);
379 %}
380
381 // This pattern is generated automatically from g1_aarch64.m4.
382 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
383 instruct g1CompareAndExchangePAcq(iRegPNoSp res, indirect mem, iRegP oldval, iRegP newval, iRegPNoSp tmp1, iRegPNoSp tmp2, rFlagsReg cr)
384 %{
385 predicate(UseG1GC && needs_acquiring_load_exclusive(n) && n->as_LoadStore()->barrier_data() != 0);
386 match(Set res (CompareAndExchangeP mem (Binary oldval newval)));
387 effect(TEMP res, TEMP tmp1, TEMP tmp2, KILL cr);
388 ins_cost(VOLATILE_REF_COST);
389 format %{ "cmpxchg_acq $res = $mem, $oldval, $newval\t# ptr" %}
390 ins_encode %{
391 assert_different_registers($oldval$$Register, $mem$$Register);
392 assert_different_registers($newval$$Register, $mem$$Register);
393 // Pass $oldval to the pre-barrier (instead of loading from $mem), because
394 // $oldval is the only value that can be overwritten.
395 // The same holds for g1CompareAndSwapP and its Acq variant.
396 write_barrier_pre(masm, this,
397 noreg /* obj */,
398 $oldval$$Register /* pre_val */,
399 $tmp1$$Register /* tmp1 */,
400 $tmp2$$Register /* tmp2 */,
401 RegSet::of($mem$$Register, $oldval$$Register, $newval$$Register) /* preserve */,
402 RegSet::of($res$$Register) /* no_preserve */);
403 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register, Assembler::xword,
404 true /* acquire */, true /* release */, false /* weak */, $res$$Register);
405 write_barrier_post(masm, this,
406 $mem$$Register /* store_addr */,
407 $newval$$Register /* new_val */,
408 $tmp1$$Register /* tmp1 */,
409 $tmp2$$Register /* tmp2 */);
410 %}
411 ins_pipe(pipe_slow);
412 %}
413
414 // This pattern is generated automatically from g1_aarch64.m4.
415 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
416 instruct g1CompareAndExchangeN(iRegNNoSp res, indirect mem, iRegN oldval, iRegN newval, iRegPNoSp tmp1, iRegPNoSp tmp2, iRegPNoSp tmp3, rFlagsReg cr)
417 %{
418 predicate(UseG1GC && !needs_acquiring_load_exclusive(n) && n->as_LoadStore()->barrier_data() != 0);
419 match(Set res (CompareAndExchangeN mem (Binary oldval newval)));
420 effect(TEMP res, TEMP tmp1, TEMP tmp2, TEMP tmp3, KILL cr);
421 ins_cost(2 * VOLATILE_REF_COST);
422 format %{ "cmpxchg $res = $mem, $oldval, $newval\t# narrow oop" %}
423 ins_encode %{
424 assert_different_registers($oldval$$Register, $mem$$Register);
425 assert_different_registers($newval$$Register, $mem$$Register);
426 write_barrier_pre(masm, this,
427 $mem$$Register /* obj */,
428 $tmp1$$Register /* pre_val */,
429 $tmp2$$Register /* tmp1 */,
430 $tmp3$$Register /* tmp2 */,
431 RegSet::of($mem$$Register, $oldval$$Register, $newval$$Register) /* preserve */,
432 RegSet::of($res$$Register) /* no_preserve */);
433 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register, Assembler::word,
434 false /* acquire */, true /* release */, false /* weak */, $res$$Register);
435 __ decode_heap_oop($tmp1$$Register, $newval$$Register);
436 write_barrier_post(masm, this,
437 $mem$$Register /* store_addr */,
438 $tmp1$$Register /* new_val */,
439 $tmp2$$Register /* tmp1 */,
440 $tmp3$$Register /* tmp2 */);
441 %}
442 ins_pipe(pipe_slow);
443 %}
444
445 // This pattern is generated automatically from g1_aarch64.m4.
446 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
447 instruct g1CompareAndExchangeNAcq(iRegNNoSp res, indirect mem, iRegN oldval, iRegN newval, iRegPNoSp tmp1, iRegPNoSp tmp2, iRegPNoSp tmp3, rFlagsReg cr)
448 %{
449 predicate(UseG1GC && needs_acquiring_load_exclusive(n) && n->as_LoadStore()->barrier_data() != 0);
450 match(Set res (CompareAndExchangeN mem (Binary oldval newval)));
451 effect(TEMP res, TEMP tmp1, TEMP tmp2, TEMP tmp3, KILL cr);
452 ins_cost(VOLATILE_REF_COST);
453 format %{ "cmpxchg_acq $res = $mem, $oldval, $newval\t# narrow oop" %}
454 ins_encode %{
455 assert_different_registers($oldval$$Register, $mem$$Register);
456 assert_different_registers($newval$$Register, $mem$$Register);
457 write_barrier_pre(masm, this,
458 $mem$$Register /* obj */,
459 $tmp1$$Register /* pre_val */,
460 $tmp2$$Register /* tmp1 */,
461 $tmp3$$Register /* tmp2 */,
462 RegSet::of($mem$$Register, $oldval$$Register, $newval$$Register) /* preserve */,
463 RegSet::of($res$$Register) /* no_preserve */);
464 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register, Assembler::word,
465 true /* acquire */, true /* release */, false /* weak */, $res$$Register);
466 __ decode_heap_oop($tmp1$$Register, $newval$$Register);
467 write_barrier_post(masm, this,
468 $mem$$Register /* store_addr */,
469 $tmp1$$Register /* new_val */,
470 $tmp2$$Register /* tmp1 */,
471 $tmp3$$Register /* tmp2 */);
472 %}
473 ins_pipe(pipe_slow);
474 %}
475
476 // This pattern is generated automatically from g1_aarch64.m4.
477 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
478 instruct g1CompareAndSwapP(iRegINoSp res, indirect mem, iRegP newval, iRegPNoSp tmp1, iRegPNoSp tmp2, iRegP oldval, rFlagsReg cr)
479 %{
480 predicate(UseG1GC && !needs_acquiring_load_exclusive(n) && n->as_LoadStore()->barrier_data() != 0);
481 match(Set res (CompareAndSwapP mem (Binary oldval newval)));
482 match(Set res (WeakCompareAndSwapP mem (Binary oldval newval)));
483 effect(TEMP res, TEMP tmp1, TEMP tmp2, KILL cr);
484 ins_cost(2 * VOLATILE_REF_COST);
485 format %{ "cmpxchg $mem, $oldval, $newval\t# (ptr)\n\t"
486 "cset $res, EQ" %}
487 ins_encode %{
488 assert_different_registers($oldval$$Register, $mem$$Register);
489 assert_different_registers($newval$$Register, $mem$$Register);
490 write_barrier_pre(masm, this,
491 noreg /* obj */,
492 $oldval$$Register /* pre_val */,
493 $tmp1$$Register /* tmp1 */,
494 $tmp2$$Register /* tmp2 */,
495 RegSet::of($mem$$Register, $oldval$$Register, $newval$$Register) /* preserve */,
496 RegSet::of($res$$Register) /* no_preserve */);
497 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register, Assembler::xword,
498 false /* acquire */, true /* release */, false /* weak */, noreg);
499 __ cset($res$$Register, Assembler::EQ);
500 write_barrier_post(masm, this,
501 $mem$$Register /* store_addr */,
502 $newval$$Register /* new_val */,
503 $tmp1$$Register /* tmp1 */,
504 $tmp2$$Register /* tmp2 */);
505 %}
506 ins_pipe(pipe_slow);
507 %}
508
509 // This pattern is generated automatically from g1_aarch64.m4.
510 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
511 instruct g1CompareAndSwapPAcq(iRegINoSp res, indirect mem, iRegP newval, iRegPNoSp tmp1, iRegPNoSp tmp2, iRegP oldval, rFlagsReg cr)
512 %{
513 predicate(UseG1GC && needs_acquiring_load_exclusive(n) && n->as_LoadStore()->barrier_data() != 0);
514 match(Set res (CompareAndSwapP mem (Binary oldval newval)));
515 match(Set res (WeakCompareAndSwapP mem (Binary oldval newval)));
516 effect(TEMP res, TEMP tmp1, TEMP tmp2, KILL cr);
517 ins_cost(VOLATILE_REF_COST);
518 format %{ "cmpxchg_acq $mem, $oldval, $newval\t# (ptr)\n\t"
519 "cset $res, EQ" %}
520 ins_encode %{
521 assert_different_registers($oldval$$Register, $mem$$Register);
522 assert_different_registers($newval$$Register, $mem$$Register);
523 write_barrier_pre(masm, this,
524 noreg /* obj */,
525 $oldval$$Register /* pre_val */,
526 $tmp1$$Register /* tmp1 */,
527 $tmp2$$Register /* tmp2 */,
528 RegSet::of($mem$$Register, $oldval$$Register, $newval$$Register) /* preserve */,
529 RegSet::of($res$$Register) /* no_preserve */);
530 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register, Assembler::xword,
531 true /* acquire */, true /* release */, false /* weak */, noreg);
532 __ cset($res$$Register, Assembler::EQ);
533 write_barrier_post(masm, this,
534 $mem$$Register /* store_addr */,
535 $newval$$Register /* new_val */,
536 $tmp1$$Register /* tmp1 */,
537 $tmp2$$Register /* tmp2 */);
538 %}
539 ins_pipe(pipe_slow);
540 %}
541
542 // This pattern is generated automatically from g1_aarch64.m4.
543 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
544 instruct g1CompareAndSwapN(iRegINoSp res, indirect mem, iRegN newval, iRegPNoSp tmp1, iRegPNoSp tmp2, iRegPNoSp tmp3, iRegN oldval, rFlagsReg cr)
545 %{
546 predicate(UseG1GC && !needs_acquiring_load_exclusive(n) && n->as_LoadStore()->barrier_data() != 0);
547 match(Set res (CompareAndSwapN mem (Binary oldval newval)));
548 match(Set res (WeakCompareAndSwapN mem (Binary oldval newval)));
549 effect(TEMP res, TEMP tmp1, TEMP tmp2, TEMP tmp3, KILL cr);
550 ins_cost(2 * VOLATILE_REF_COST);
551 format %{ "cmpxchg $mem, $oldval, $newval\t# (narrow oop)\n\t"
552 "cset $res, EQ" %}
553 ins_encode %{
554 assert_different_registers($oldval$$Register, $mem$$Register);
555 assert_different_registers($newval$$Register, $mem$$Register);
556 write_barrier_pre(masm, this,
557 $mem$$Register /* obj */,
558 $tmp1$$Register /* pre_val */,
559 $tmp2$$Register /* tmp1 */,
560 $tmp3$$Register /* tmp2 */,
561 RegSet::of($mem$$Register, $oldval$$Register, $newval$$Register) /* preserve */,
562 RegSet::of($res$$Register) /* no_preserve */);
563 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register, Assembler::word,
564 false /* acquire */, true /* release */, false /* weak */, noreg);
565 __ cset($res$$Register, Assembler::EQ);
566 __ decode_heap_oop($tmp1$$Register, $newval$$Register);
567 write_barrier_post(masm, this,
568 $mem$$Register /* store_addr */,
569 $tmp1$$Register /* new_val */,
570 $tmp2$$Register /* tmp1 */,
571 $tmp3$$Register /* tmp2 */);
572 %}
573 ins_pipe(pipe_slow);
574 %}
575
576 // This pattern is generated automatically from g1_aarch64.m4.
577 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
578 instruct g1CompareAndSwapNAcq(iRegINoSp res, indirect mem, iRegN newval, iRegPNoSp tmp1, iRegPNoSp tmp2, iRegPNoSp tmp3, iRegN oldval, rFlagsReg cr)
579 %{
580 predicate(UseG1GC && needs_acquiring_load_exclusive(n) && n->as_LoadStore()->barrier_data() != 0);
581 match(Set res (CompareAndSwapN mem (Binary oldval newval)));
582 match(Set res (WeakCompareAndSwapN mem (Binary oldval newval)));
583 effect(TEMP res, TEMP tmp1, TEMP tmp2, TEMP tmp3, KILL cr);
584 ins_cost(VOLATILE_REF_COST);
585 format %{ "cmpxchg_acq $mem, $oldval, $newval\t# (narrow oop)\n\t"
586 "cset $res, EQ" %}
587 ins_encode %{
588 assert_different_registers($oldval$$Register, $mem$$Register);
589 assert_different_registers($newval$$Register, $mem$$Register);
590 write_barrier_pre(masm, this,
591 $mem$$Register /* obj */,
592 $tmp1$$Register /* pre_val */,
593 $tmp2$$Register /* tmp1 */,
594 $tmp3$$Register /* tmp2 */,
595 RegSet::of($mem$$Register, $oldval$$Register, $newval$$Register) /* preserve */,
596 RegSet::of($res$$Register) /* no_preserve */);
597 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register, Assembler::word,
598 true /* acquire */, true /* release */, false /* weak */, noreg);
599 __ cset($res$$Register, Assembler::EQ);
600 __ decode_heap_oop($tmp1$$Register, $newval$$Register);
601 write_barrier_post(masm, this,
602 $mem$$Register /* store_addr */,
603 $tmp1$$Register /* new_val */,
604 $tmp2$$Register /* tmp1 */,
605 $tmp3$$Register /* tmp2 */);
606 %}
607 ins_pipe(pipe_slow);
608 %}
609
610 // This pattern is generated automatically from g1_aarch64.m4.
611 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
612 instruct g1GetAndSetP(indirect mem, iRegP newval, iRegPNoSp tmp1, iRegPNoSp tmp2, iRegPNoSp preval, rFlagsReg cr)
613 %{
614 predicate(UseG1GC && !needs_acquiring_load_exclusive(n) && n->as_LoadStore()->barrier_data() != 0);
615 match(Set preval (GetAndSetP mem newval));
616 effect(TEMP preval, TEMP tmp1, TEMP tmp2, KILL cr);
617 ins_cost(2 * VOLATILE_REF_COST);
618 format %{ "atomic_xchg $preval, $newval, [$mem]" %}
619 ins_encode %{
620 assert_different_registers($mem$$Register, $newval$$Register);
621 write_barrier_pre(masm, this,
622 $mem$$Register /* obj */,
623 $preval$$Register /* pre_val (as a temporary register) */,
624 $tmp1$$Register /* tmp1 */,
625 $tmp2$$Register /* tmp2 */,
626 RegSet::of($mem$$Register, $preval$$Register, $newval$$Register) /* preserve */);
627 __ atomic_xchg($preval$$Register, $newval$$Register, $mem$$Register);
628 write_barrier_post(masm, this,
629 $mem$$Register /* store_addr */,
630 $newval$$Register /* new_val */,
631 $tmp1$$Register /* tmp1 */,
632 $tmp2$$Register /* tmp2 */);
633 %}
634 ins_pipe(pipe_serial);
635 %}
636
637 // This pattern is generated automatically from g1_aarch64.m4.
638 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
639 instruct g1GetAndSetPAcq(indirect mem, iRegP newval, iRegPNoSp tmp1, iRegPNoSp tmp2, iRegPNoSp preval, rFlagsReg cr)
640 %{
641 predicate(UseG1GC && needs_acquiring_load_exclusive(n) && n->as_LoadStore()->barrier_data() != 0);
642 match(Set preval (GetAndSetP mem newval));
643 effect(TEMP preval, TEMP tmp1, TEMP tmp2, KILL cr);
644 ins_cost(VOLATILE_REF_COST);
645 format %{ "atomic_xchg_acq $preval, $newval, [$mem]" %}
646 ins_encode %{
647 assert_different_registers($mem$$Register, $newval$$Register);
648 write_barrier_pre(masm, this,
649 $mem$$Register /* obj */,
650 $preval$$Register /* pre_val (as a temporary register) */,
651 $tmp1$$Register /* tmp1 */,
652 $tmp2$$Register /* tmp2 */,
653 RegSet::of($mem$$Register, $preval$$Register, $newval$$Register) /* preserve */);
654 __ atomic_xchgal($preval$$Register, $newval$$Register, $mem$$Register);
655 write_barrier_post(masm, this,
656 $mem$$Register /* store_addr */,
657 $newval$$Register /* new_val */,
658 $tmp1$$Register /* tmp1 */,
659 $tmp2$$Register /* tmp2 */);
660 %}
661 ins_pipe(pipe_serial);
662 %}
663
664 // This pattern is generated automatically from g1_aarch64.m4.
665 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
666 instruct g1GetAndSetN(indirect mem, iRegN newval, iRegPNoSp tmp1, iRegPNoSp tmp2, iRegPNoSp tmp3, iRegNNoSp preval, rFlagsReg cr)
667 %{
668 predicate(UseG1GC && !needs_acquiring_load_exclusive(n) && n->as_LoadStore()->barrier_data() != 0);
669 match(Set preval (GetAndSetN mem newval));
670 effect(TEMP preval, TEMP tmp1, TEMP tmp2, TEMP tmp3, KILL cr);
671 ins_cost(2 * VOLATILE_REF_COST);
672 format %{ "atomic_xchgw $preval, $newval, [$mem]" %}
673 ins_encode %{
674 assert_different_registers($mem$$Register, $newval$$Register);
675 write_barrier_pre(masm, this,
676 $mem$$Register /* obj */,
677 $tmp1$$Register /* pre_val */,
678 $tmp2$$Register /* tmp1 */,
679 $tmp3$$Register /* tmp2 */,
680 RegSet::of($mem$$Register, $preval$$Register, $newval$$Register) /* preserve */);
681 __ atomic_xchgw($preval$$Register, $newval$$Register, $mem$$Register);
682 __ decode_heap_oop($tmp1$$Register, $newval$$Register);
683 write_barrier_post(masm, this,
684 $mem$$Register /* store_addr */,
685 $tmp1$$Register /* new_val */,
686 $tmp2$$Register /* tmp1 */,
687 $tmp3$$Register /* tmp2 */);
688 %}
689 ins_pipe(pipe_serial);
690 %}
691
692 // This pattern is generated automatically from g1_aarch64.m4.
693 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
694 instruct g1GetAndSetNAcq(indirect mem, iRegN newval, iRegPNoSp tmp1, iRegPNoSp tmp2, iRegPNoSp tmp3, iRegNNoSp preval, rFlagsReg cr)
695 %{
696 predicate(UseG1GC && needs_acquiring_load_exclusive(n) && n->as_LoadStore()->barrier_data() != 0);
697 match(Set preval (GetAndSetN mem newval));
698 effect(TEMP preval, TEMP tmp1, TEMP tmp2, TEMP tmp3, KILL cr);
699 ins_cost(VOLATILE_REF_COST);
700 format %{ "atomic_xchgw_acq $preval, $newval, [$mem]" %}
701 ins_encode %{
702 assert_different_registers($mem$$Register, $newval$$Register);
703 write_barrier_pre(masm, this,
704 $mem$$Register /* obj */,
705 $tmp1$$Register /* pre_val */,
706 $tmp2$$Register /* tmp1 */,
707 $tmp3$$Register /* tmp2 */,
708 RegSet::of($mem$$Register, $preval$$Register, $newval$$Register) /* preserve */);
709 __ atomic_xchgalw($preval$$Register, $newval$$Register, $mem$$Register);
710 __ decode_heap_oop($tmp1$$Register, $newval$$Register);
711 write_barrier_post(masm, this,
712 $mem$$Register /* store_addr */,
713 $tmp1$$Register /* new_val */,
714 $tmp2$$Register /* tmp1 */,
715 $tmp3$$Register /* tmp2 */);
716 %}
717 ins_pipe(pipe_serial);
718 %}
719
720 // This pattern is generated automatically from g1_aarch64.m4.
721 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
722 instruct g1LoadP(iRegPNoSp dst, indirect mem, iRegPNoSp tmp1, iRegPNoSp tmp2, rFlagsReg cr)
723 %{
724 // This instruction does not need an acquiring counterpart because it is only
725 // used for reference loading (Reference::get()). The same holds for g1LoadN.
726 predicate(UseG1GC && !needs_acquiring_load(n) && n->as_Load()->barrier_data() != 0);
727 match(Set dst (LoadP mem));
728 effect(TEMP dst, TEMP tmp1, TEMP tmp2, KILL cr);
729 ins_cost(4 * INSN_COST);
730 format %{ "ldr $dst, $mem\t# ptr" %}
731 ins_encode %{
732 __ ldr($dst$$Register, $mem$$Register);
733 write_barrier_pre(masm, this,
734 noreg /* obj */,
735 $dst$$Register /* pre_val */,
736 $tmp1$$Register /* tmp1 */,
737 $tmp2$$Register /* tmp2 */);
738 %}
739 ins_pipe(iload_reg_mem);
740 %}
741
742 // This pattern is generated automatically from g1_aarch64.m4.
743 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
744 instruct g1LoadN(iRegNNoSp dst, indirect mem, iRegPNoSp tmp1, iRegPNoSp tmp2, iRegPNoSp tmp3, rFlagsReg cr)
745 %{
746 predicate(UseG1GC && !needs_acquiring_load(n) && n->as_Load()->barrier_data() != 0);
747 match(Set dst (LoadN mem));
748 effect(TEMP dst, TEMP tmp1, TEMP tmp2, TEMP tmp3, KILL cr);
749 ins_cost(4 * INSN_COST);
750 format %{ "ldrw $dst, $mem\t# compressed ptr" %}
751 ins_encode %{
752 __ ldrw($dst$$Register, $mem$$Register);
753 if ((barrier_data() & G1C2BarrierPre) != 0) {
754 __ decode_heap_oop($tmp1$$Register, $dst$$Register);
755 write_barrier_pre(masm, this,
756 noreg /* obj */,
757 $tmp1$$Register /* pre_val */,
758 $tmp2$$Register /* tmp1 */,
759 $tmp3$$Register /* tmp2 */);
760 }
761 %}
762 ins_pipe(iload_reg_mem);
763 %}
764
765 // END This section of the file is automatically generated. Do not edit --------------