1 //
  2 // Copyright (c) 2024, Oracle and/or its affiliates. All rights reserved.
  3 // DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  4 //
  5 // This code is free software; you can redistribute it and/or modify it
  6 // under the terms of the GNU General Public License version 2 only, as
  7 // published by the Free Software Foundation.
  8 //
  9 // This code is distributed in the hope that it will be useful, but WITHOUT
 10 // ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
 11 // FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
 12 // version 2 for more details (a copy is included in the LICENSE file that
 13 // accompanied this code).
 14 //
 15 // You should have received a copy of the GNU General Public License version
 16 // 2 along with this work; if not, write to the Free Software Foundation,
 17 // Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
 18 //
 19 // Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
 20 // or visit www.oracle.com if you need additional information or have any
 21 // questions.
 22 //
 23 
 24 source_hpp %{
 25 
 26 #include "gc/g1/c2/g1BarrierSetC2.hpp"
 27 #include "gc/shared/gc_globals.hpp"
 28 
 29 %}
 30 
 31 source %{
 32 
 33 #include "gc/g1/g1BarrierSetAssembler_aarch64.hpp"
 34 #include "gc/g1/g1BarrierSetRuntime.hpp"
 35 
 36 static void write_barrier_pre(MacroAssembler* masm,
 37                               const MachNode* node,
 38                               Register obj,
 39                               Register pre_val,
 40                               Register tmp1,
 41                               Register tmp2,
 42                               RegSet preserve = RegSet(),
 43                               RegSet no_preserve = RegSet()) {
 44   if (!G1PreBarrierStubC2::needs_barrier(node)) {
 45     return;
 46   }
 47   Assembler::InlineSkippedInstructionsCounter skip_counter(masm);
 48   G1BarrierSetAssembler* g1_asm = static_cast<G1BarrierSetAssembler*>(BarrierSet::barrier_set()->barrier_set_assembler());
 49   G1PreBarrierStubC2* const stub = G1PreBarrierStubC2::create(node);
 50   for (RegSetIterator<Register> reg = preserve.begin(); *reg != noreg; ++reg) {
 51     stub->preserve(*reg);
 52   }
 53   for (RegSetIterator<Register> reg = no_preserve.begin(); *reg != noreg; ++reg) {
 54     stub->dont_preserve(*reg);
 55   }
 56   g1_asm->g1_write_barrier_pre_c2(masm, obj, pre_val, rthread, tmp1, tmp2, stub);
 57 }
 58 
 59 static void write_barrier_post(MacroAssembler* masm,
 60                                const MachNode* node,
 61                                Register store_addr,
 62                                Register new_val,
 63                                Register tmp1,
 64                                Register tmp2) {
 65   if (!G1PostBarrierStubC2::needs_barrier(node)) {
 66     return;
 67   }
 68   Assembler::InlineSkippedInstructionsCounter skip_counter(masm);
 69   G1BarrierSetAssembler* g1_asm = static_cast<G1BarrierSetAssembler*>(BarrierSet::barrier_set()->barrier_set_assembler());
 70   G1PostBarrierStubC2* const stub = G1PostBarrierStubC2::create(node);
 71   g1_asm->g1_write_barrier_post_c2(masm, store_addr, new_val, rthread, tmp1, tmp2, stub);
 72 }
 73 
 74 %}
 75 
 76 // BEGIN This section of the file is automatically generated. Do not edit --------------
 77 
 78 // This section is generated from g1_aarch64.m4
 79 
 80 
 81 // This pattern is generated automatically from g1_aarch64.m4.
 82 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
 83 instruct g1StoreP(indirect mem, iRegP src, iRegPNoSp tmp1, iRegPNoSp tmp2, iRegPNoSp tmp3, rFlagsReg cr)
 84 %{
 85   predicate(UseG1GC && !needs_releasing_store(n) && n->as_Store()->barrier_data() != 0);
 86   match(Set mem (StoreP mem src));
 87   effect(TEMP tmp1, TEMP tmp2, TEMP tmp3, KILL cr);
 88   ins_cost(INSN_COST);
 89   format %{ "str  $src, $mem\t# ptr" %}
 90   ins_encode %{
 91     write_barrier_pre(masm, this,
 92                       $mem$$Register /* obj */,
 93                       $tmp1$$Register /* pre_val */,
 94                       $tmp2$$Register /* tmp1 */,
 95                       $tmp3$$Register /* tmp2 */,
 96                       RegSet::of($mem$$Register, $src$$Register) /* preserve */);
 97     __ str($src$$Register, $mem$$Register);
 98     write_barrier_post(masm, this,
 99                        $mem$$Register /* store_addr */,
100                        $src$$Register /* new_val */,
101                        $tmp2$$Register /* tmp1 */,
102                        $tmp3$$Register /* tmp2 */);
103   %}
104   ins_pipe(istore_reg_mem);
105 %}
106 
107 // This pattern is generated automatically from g1_aarch64.m4.
108 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
109 instruct g1StorePVolatile(indirect mem, iRegP src, iRegPNoSp tmp1, iRegPNoSp tmp2, iRegPNoSp tmp3, rFlagsReg cr)
110 %{
111   predicate(UseG1GC && needs_releasing_store(n) && n->as_Store()->barrier_data() != 0);
112   match(Set mem (StoreP mem src));
113   effect(TEMP tmp1, TEMP tmp2, TEMP tmp3, KILL cr);
114   ins_cost(VOLATILE_REF_COST);
115   format %{ "stlr  $src, $mem\t# ptr" %}
116   ins_encode %{
117     write_barrier_pre(masm, this,
118                       $mem$$Register /* obj */,
119                       $tmp1$$Register /* pre_val */,
120                       $tmp2$$Register /* tmp1 */,
121                       $tmp3$$Register /* tmp2 */,
122                       RegSet::of($mem$$Register, $src$$Register) /* preserve */);
123     __ stlr($src$$Register, $mem$$Register);
124     write_barrier_post(masm, this,
125                        $mem$$Register /* store_addr */,
126                        $src$$Register /* new_val */,
127                        $tmp2$$Register /* tmp1 */,
128                        $tmp3$$Register /* tmp2 */);
129   %}
130   ins_pipe(pipe_class_memory);
131 %}
132 
133 // This pattern is generated automatically from g1_aarch64.m4.
134 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
135 instruct g1StoreN(indirect mem, iRegN src, iRegPNoSp tmp1, iRegPNoSp tmp2, iRegPNoSp tmp3, rFlagsReg cr)
136 %{
137   predicate(UseG1GC && !needs_releasing_store(n) && n->as_Store()->barrier_data() != 0);
138   match(Set mem (StoreN mem src));
139   effect(TEMP tmp1, TEMP tmp2, TEMP tmp3, KILL cr);
140   ins_cost(INSN_COST);
141   format %{ "strw  $src, $mem\t# compressed ptr" %}
142   ins_encode %{
143     write_barrier_pre(masm, this,
144                       $mem$$Register /* obj */,
145                       $tmp1$$Register /* pre_val */,
146                       $tmp2$$Register /* tmp1 */,
147                       $tmp3$$Register /* tmp2 */,
148                       RegSet::of($mem$$Register, $src$$Register) /* preserve */);
149     __ strw($src$$Register, $mem$$Register);
150     if ((barrier_data() & G1C2BarrierPost) != 0) {
151       if ((barrier_data() & G1C2BarrierPostNotNull) == 0) {
152         __ decode_heap_oop($tmp1$$Register, $src$$Register);
153       } else {
154         __ decode_heap_oop_not_null($tmp1$$Register, $src$$Register);
155       }
156     }
157     write_barrier_post(masm, this,
158                        $mem$$Register /* store_addr */,
159                        $tmp1$$Register /* new_val */,
160                        $tmp2$$Register /* tmp1 */,
161                        $tmp3$$Register /* tmp2 */);
162   %}
163   ins_pipe(istore_reg_mem);
164 %}
165 
166 // This pattern is generated automatically from g1_aarch64.m4.
167 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
168 instruct g1StoreNVolatile(indirect mem, iRegN src, iRegPNoSp tmp1, iRegPNoSp tmp2, iRegPNoSp tmp3, rFlagsReg cr)
169 %{
170   predicate(UseG1GC && needs_releasing_store(n) && n->as_Store()->barrier_data() != 0);
171   match(Set mem (StoreN mem src));
172   effect(TEMP tmp1, TEMP tmp2, TEMP tmp3, KILL cr);
173   ins_cost(VOLATILE_REF_COST);
174   format %{ "stlrw  $src, $mem\t# compressed ptr" %}
175   ins_encode %{
176     write_barrier_pre(masm, this,
177                       $mem$$Register /* obj */,
178                       $tmp1$$Register /* pre_val */,
179                       $tmp2$$Register /* tmp1 */,
180                       $tmp3$$Register /* tmp2 */,
181                       RegSet::of($mem$$Register, $src$$Register) /* preserve */);
182     __ stlrw($src$$Register, $mem$$Register);
183     if ((barrier_data() & G1C2BarrierPost) != 0) {
184       if ((barrier_data() & G1C2BarrierPostNotNull) == 0) {
185         __ decode_heap_oop($tmp1$$Register, $src$$Register);
186       } else {
187         __ decode_heap_oop_not_null($tmp1$$Register, $src$$Register);
188       }
189     }
190     write_barrier_post(masm, this,
191                        $mem$$Register /* store_addr */,
192                        $tmp1$$Register /* new_val */,
193                        $tmp2$$Register /* tmp1 */,
194                        $tmp3$$Register /* tmp2 */);
195   %}
196   ins_pipe(pipe_class_memory);
197 %}
198 
199 // This pattern is generated automatically from g1_aarch64.m4.
200 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
201 instruct g1EncodePAndStoreN(indirect mem, iRegP src, iRegPNoSp tmp1, iRegPNoSp tmp2, iRegPNoSp tmp3, rFlagsReg cr)
202 %{
203   predicate(UseG1GC && !needs_releasing_store(n) && n->as_Store()->barrier_data() != 0);
204   match(Set mem (StoreN mem (EncodeP src)));
205   effect(TEMP tmp1, TEMP tmp2, TEMP tmp3, KILL cr);
206   ins_cost(INSN_COST);
207   format %{ "encode_heap_oop $tmp1, $src\n\t"
208             "strw  $tmp1, $mem\t# compressed ptr" %}
209   ins_encode %{
210     write_barrier_pre(masm, this,
211                       $mem$$Register /* obj */,
212                       $tmp1$$Register /* pre_val */,
213                       $tmp2$$Register /* tmp1 */,
214                       $tmp3$$Register /* tmp2 */,
215                       RegSet::of($mem$$Register, $src$$Register) /* preserve */);
216     if ((barrier_data() & G1C2BarrierPostNotNull) == 0) {
217       __ encode_heap_oop($tmp1$$Register, $src$$Register);
218     } else {
219       __ encode_heap_oop_not_null($tmp1$$Register, $src$$Register);
220     }
221     __ strw($tmp1$$Register, $mem$$Register);
222     write_barrier_post(masm, this,
223                        $mem$$Register /* store_addr */,
224                        $src$$Register /* new_val */,
225                        $tmp2$$Register /* tmp1 */,
226                        $tmp3$$Register /* tmp2 */);
227   %}
228   ins_pipe(istore_reg_mem);
229 %}
230 
231 // This pattern is generated automatically from g1_aarch64.m4.
232 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
233 instruct g1EncodePAndStoreNVolatile(indirect mem, iRegP src, iRegPNoSp tmp1, iRegPNoSp tmp2, iRegPNoSp tmp3, rFlagsReg cr)
234 %{
235   predicate(UseG1GC && needs_releasing_store(n) && n->as_Store()->barrier_data() != 0);
236   match(Set mem (StoreN mem (EncodeP src)));
237   effect(TEMP tmp1, TEMP tmp2, TEMP tmp3, KILL cr);
238   ins_cost(VOLATILE_REF_COST);
239   format %{ "encode_heap_oop $tmp1, $src\n\t"
240             "stlrw  $tmp1, $mem\t# compressed ptr" %}
241   ins_encode %{
242     write_barrier_pre(masm, this,
243                       $mem$$Register /* obj */,
244                       $tmp1$$Register /* pre_val */,
245                       $tmp2$$Register /* tmp1 */,
246                       $tmp3$$Register /* tmp2 */,
247                       RegSet::of($mem$$Register, $src$$Register) /* preserve */);
248     if ((barrier_data() & G1C2BarrierPostNotNull) == 0) {
249       __ encode_heap_oop($tmp1$$Register, $src$$Register);
250     } else {
251       __ encode_heap_oop_not_null($tmp1$$Register, $src$$Register);
252     }
253     __ stlrw($tmp1$$Register, $mem$$Register);
254     write_barrier_post(masm, this,
255                        $mem$$Register /* store_addr */,
256                        $src$$Register /* new_val */,
257                        $tmp2$$Register /* tmp1 */,
258                        $tmp3$$Register /* tmp2 */);
259   %}
260   ins_pipe(pipe_class_memory);
261 %}
262 
263 // This pattern is generated automatically from g1_aarch64.m4.
264 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
265 instruct g1CompareAndExchangeP(iRegPNoSp res, indirect mem, iRegP oldval, iRegP newval, iRegPNoSp tmp1, iRegPNoSp tmp2, rFlagsReg cr)
266 %{
267   predicate(UseG1GC && !needs_acquiring_load_exclusive(n) && n->as_LoadStore()->barrier_data() != 0);
268   match(Set res (CompareAndExchangeP mem (Binary oldval newval)));
269   effect(TEMP res, TEMP tmp1, TEMP tmp2, KILL cr);
270   ins_cost(2 * VOLATILE_REF_COST);
271   format %{ "cmpxchg $res = $mem, $oldval, $newval\t# ptr" %}
272   ins_encode %{
273     assert_different_registers($oldval$$Register, $mem$$Register);
274     assert_different_registers($newval$$Register, $mem$$Register);
275     // Pass $oldval to the pre-barrier (instead of loading from $mem), because
276     // $oldval is the only value that can be overwritten.
277     // The same holds for g1CompareAndSwapP and its Acq variant.
278     write_barrier_pre(masm, this,
279                       noreg /* obj */,
280                       $oldval$$Register /* pre_val */,
281                       $tmp1$$Register /* tmp1 */,
282                       $tmp2$$Register /* tmp2 */,
283                       RegSet::of($mem$$Register, $oldval$$Register, $newval$$Register) /* preserve */,
284                       RegSet::of($res$$Register) /* no_preserve */);
285     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register, Assembler::xword,
286                false /* acquire */, true /* release */, false /* weak */, $res$$Register);
287     write_barrier_post(masm, this,
288                        $mem$$Register /* store_addr */,
289                        $newval$$Register /* new_val */,
290                        $tmp1$$Register /* tmp1 */,
291                        $tmp2$$Register /* tmp2 */);
292   %}
293   ins_pipe(pipe_slow);
294 %}
295 
296 // This pattern is generated automatically from g1_aarch64.m4.
297 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
298 instruct g1CompareAndExchangePAcq(iRegPNoSp res, indirect mem, iRegP oldval, iRegP newval, iRegPNoSp tmp1, iRegPNoSp tmp2, rFlagsReg cr)
299 %{
300   predicate(UseG1GC && needs_acquiring_load_exclusive(n) && n->as_LoadStore()->barrier_data() != 0);
301   match(Set res (CompareAndExchangeP mem (Binary oldval newval)));
302   effect(TEMP res, TEMP tmp1, TEMP tmp2, KILL cr);
303   ins_cost(VOLATILE_REF_COST);
304   format %{ "cmpxchg_acq $res = $mem, $oldval, $newval\t# ptr" %}
305   ins_encode %{
306     assert_different_registers($oldval$$Register, $mem$$Register);
307     assert_different_registers($newval$$Register, $mem$$Register);
308     // Pass $oldval to the pre-barrier (instead of loading from $mem), because
309     // $oldval is the only value that can be overwritten.
310     // The same holds for g1CompareAndSwapP and its Acq variant.
311     write_barrier_pre(masm, this,
312                       noreg /* obj */,
313                       $oldval$$Register /* pre_val */,
314                       $tmp1$$Register /* tmp1 */,
315                       $tmp2$$Register /* tmp2 */,
316                       RegSet::of($mem$$Register, $oldval$$Register, $newval$$Register) /* preserve */,
317                       RegSet::of($res$$Register) /* no_preserve */);
318     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register, Assembler::xword,
319                true /* acquire */, true /* release */, false /* weak */, $res$$Register);
320     write_barrier_post(masm, this,
321                        $mem$$Register /* store_addr */,
322                        $newval$$Register /* new_val */,
323                        $tmp1$$Register /* tmp1 */,
324                        $tmp2$$Register /* tmp2 */);
325   %}
326   ins_pipe(pipe_slow);
327 %}
328 
329 // This pattern is generated automatically from g1_aarch64.m4.
330 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
331 instruct g1CompareAndExchangeN(iRegNNoSp res, indirect mem, iRegN oldval, iRegN newval, iRegPNoSp tmp1, iRegPNoSp tmp2, iRegPNoSp tmp3, rFlagsReg cr)
332 %{
333   predicate(UseG1GC && !needs_acquiring_load_exclusive(n) && n->as_LoadStore()->barrier_data() != 0);
334   match(Set res (CompareAndExchangeN mem (Binary oldval newval)));
335   effect(TEMP res, TEMP tmp1, TEMP tmp2, TEMP tmp3, KILL cr);
336   ins_cost(2 * VOLATILE_REF_COST);
337   format %{ "cmpxchg $res = $mem, $oldval, $newval\t# narrow oop" %}
338   ins_encode %{
339     assert_different_registers($oldval$$Register, $mem$$Register);
340     assert_different_registers($newval$$Register, $mem$$Register);
341     write_barrier_pre(masm, this,
342                       $mem$$Register /* obj */,
343                       $tmp1$$Register /* pre_val */,
344                       $tmp2$$Register /* tmp1 */,
345                       $tmp3$$Register /* tmp2 */,
346                       RegSet::of($mem$$Register, $oldval$$Register, $newval$$Register) /* preserve */,
347                       RegSet::of($res$$Register) /* no_preserve */);
348     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register, Assembler::word,
349                false /* acquire */, true /* release */, false /* weak */, $res$$Register);
350     __ decode_heap_oop($tmp1$$Register, $newval$$Register);
351     write_barrier_post(masm, this,
352                        $mem$$Register /* store_addr */,
353                        $tmp1$$Register /* new_val */,
354                        $tmp2$$Register /* tmp1 */,
355                        $tmp3$$Register /* tmp2 */);
356   %}
357   ins_pipe(pipe_slow);
358 %}
359 
360 // This pattern is generated automatically from g1_aarch64.m4.
361 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
362 instruct g1CompareAndExchangeNAcq(iRegNNoSp res, indirect mem, iRegN oldval, iRegN newval, iRegPNoSp tmp1, iRegPNoSp tmp2, iRegPNoSp tmp3, rFlagsReg cr)
363 %{
364   predicate(UseG1GC && needs_acquiring_load_exclusive(n) && n->as_LoadStore()->barrier_data() != 0);
365   match(Set res (CompareAndExchangeN mem (Binary oldval newval)));
366   effect(TEMP res, TEMP tmp1, TEMP tmp2, TEMP tmp3, KILL cr);
367   ins_cost(VOLATILE_REF_COST);
368   format %{ "cmpxchg_acq $res = $mem, $oldval, $newval\t# narrow oop" %}
369   ins_encode %{
370     assert_different_registers($oldval$$Register, $mem$$Register);
371     assert_different_registers($newval$$Register, $mem$$Register);
372     write_barrier_pre(masm, this,
373                       $mem$$Register /* obj */,
374                       $tmp1$$Register /* pre_val */,
375                       $tmp2$$Register /* tmp1 */,
376                       $tmp3$$Register /* tmp2 */,
377                       RegSet::of($mem$$Register, $oldval$$Register, $newval$$Register) /* preserve */,
378                       RegSet::of($res$$Register) /* no_preserve */);
379     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register, Assembler::word,
380                true /* acquire */, true /* release */, false /* weak */, $res$$Register);
381     __ decode_heap_oop($tmp1$$Register, $newval$$Register);
382     write_barrier_post(masm, this,
383                        $mem$$Register /* store_addr */,
384                        $tmp1$$Register /* new_val */,
385                        $tmp2$$Register /* tmp1 */,
386                        $tmp3$$Register /* tmp2 */);
387   %}
388   ins_pipe(pipe_slow);
389 %}
390 
391 // This pattern is generated automatically from g1_aarch64.m4.
392 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
393 instruct g1CompareAndSwapP(iRegINoSp res, indirect mem, iRegP newval, iRegPNoSp tmp1, iRegPNoSp tmp2, iRegP oldval, rFlagsReg cr)
394 %{
395   predicate(UseG1GC && !needs_acquiring_load_exclusive(n) && n->as_LoadStore()->barrier_data() != 0);
396   match(Set res (CompareAndSwapP mem (Binary oldval newval)));
397   match(Set res (WeakCompareAndSwapP mem (Binary oldval newval)));
398   effect(TEMP res, TEMP tmp1, TEMP tmp2, KILL cr);
399   ins_cost(2 * VOLATILE_REF_COST);
400   format %{ "cmpxchg $mem, $oldval, $newval\t# (ptr)\n\t"
401             "cset $res, EQ" %}
402   ins_encode %{
403     assert_different_registers($oldval$$Register, $mem$$Register);
404     assert_different_registers($newval$$Register, $mem$$Register);
405     write_barrier_pre(masm, this,
406                       noreg /* obj */,
407                       $oldval$$Register /* pre_val */,
408                       $tmp1$$Register /* tmp1 */,
409                       $tmp2$$Register /* tmp2 */,
410                       RegSet::of($mem$$Register, $oldval$$Register, $newval$$Register) /* preserve */,
411                       RegSet::of($res$$Register) /* no_preserve */);
412     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register, Assembler::xword,
413                false /* acquire */, true /* release */, false /* weak */, noreg);
414     __ cset($res$$Register, Assembler::EQ);
415     write_barrier_post(masm, this,
416                        $mem$$Register /* store_addr */,
417                        $newval$$Register /* new_val */,
418                        $tmp1$$Register /* tmp1 */,
419                        $tmp2$$Register /* tmp2 */);
420   %}
421   ins_pipe(pipe_slow);
422 %}
423 
424 // This pattern is generated automatically from g1_aarch64.m4.
425 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
426 instruct g1CompareAndSwapPAcq(iRegINoSp res, indirect mem, iRegP newval, iRegPNoSp tmp1, iRegPNoSp tmp2, iRegP oldval, rFlagsReg cr)
427 %{
428   predicate(UseG1GC && needs_acquiring_load_exclusive(n) && n->as_LoadStore()->barrier_data() != 0);
429   match(Set res (CompareAndSwapP mem (Binary oldval newval)));
430   match(Set res (WeakCompareAndSwapP mem (Binary oldval newval)));
431   effect(TEMP res, TEMP tmp1, TEMP tmp2, KILL cr);
432   ins_cost(VOLATILE_REF_COST);
433   format %{ "cmpxchg_acq $mem, $oldval, $newval\t# (ptr)\n\t"
434             "cset $res, EQ" %}
435   ins_encode %{
436     assert_different_registers($oldval$$Register, $mem$$Register);
437     assert_different_registers($newval$$Register, $mem$$Register);
438     write_barrier_pre(masm, this,
439                       noreg /* obj */,
440                       $oldval$$Register /* pre_val */,
441                       $tmp1$$Register /* tmp1 */,
442                       $tmp2$$Register /* tmp2 */,
443                       RegSet::of($mem$$Register, $oldval$$Register, $newval$$Register) /* preserve */,
444                       RegSet::of($res$$Register) /* no_preserve */);
445     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register, Assembler::xword,
446                true /* acquire */, true /* release */, false /* weak */, noreg);
447     __ cset($res$$Register, Assembler::EQ);
448     write_barrier_post(masm, this,
449                        $mem$$Register /* store_addr */,
450                        $newval$$Register /* new_val */,
451                        $tmp1$$Register /* tmp1 */,
452                        $tmp2$$Register /* tmp2 */);
453   %}
454   ins_pipe(pipe_slow);
455 %}
456 
457 // This pattern is generated automatically from g1_aarch64.m4.
458 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
459 instruct g1CompareAndSwapN(iRegINoSp res, indirect mem, iRegN newval, iRegPNoSp tmp1, iRegPNoSp tmp2, iRegPNoSp tmp3, iRegN oldval, rFlagsReg cr)
460 %{
461   predicate(UseG1GC && !needs_acquiring_load_exclusive(n) && n->as_LoadStore()->barrier_data() != 0);
462   match(Set res (CompareAndSwapN mem (Binary oldval newval)));
463   match(Set res (WeakCompareAndSwapN mem (Binary oldval newval)));
464   effect(TEMP res, TEMP tmp1, TEMP tmp2, TEMP tmp3, KILL cr);
465   ins_cost(2 * VOLATILE_REF_COST);
466   format %{ "cmpxchg $mem, $oldval, $newval\t# (narrow oop)\n\t"
467             "cset $res, EQ" %}
468   ins_encode %{
469     assert_different_registers($oldval$$Register, $mem$$Register);
470     assert_different_registers($newval$$Register, $mem$$Register);
471     write_barrier_pre(masm, this,
472                       $mem$$Register /* obj */,
473                       $tmp1$$Register /* pre_val */,
474                       $tmp2$$Register /* tmp1 */,
475                       $tmp3$$Register /* tmp2 */,
476                       RegSet::of($mem$$Register, $oldval$$Register, $newval$$Register) /* preserve */,
477                       RegSet::of($res$$Register) /* no_preserve */);
478     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register, Assembler::word,
479                false /* acquire */, true /* release */, false /* weak */, noreg);
480     __ cset($res$$Register, Assembler::EQ);
481     __ decode_heap_oop($tmp1$$Register, $newval$$Register);
482     write_barrier_post(masm, this,
483                        $mem$$Register /* store_addr */,
484                        $tmp1$$Register /* new_val */,
485                        $tmp2$$Register /* tmp1 */,
486                        $tmp3$$Register /* tmp2 */);
487   %}
488   ins_pipe(pipe_slow);
489 %}
490 
491 // This pattern is generated automatically from g1_aarch64.m4.
492 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
493 instruct g1CompareAndSwapNAcq(iRegINoSp res, indirect mem, iRegN newval, iRegPNoSp tmp1, iRegPNoSp tmp2, iRegPNoSp tmp3, iRegN oldval, rFlagsReg cr)
494 %{
495   predicate(UseG1GC && needs_acquiring_load_exclusive(n) && n->as_LoadStore()->barrier_data() != 0);
496   match(Set res (CompareAndSwapN mem (Binary oldval newval)));
497   match(Set res (WeakCompareAndSwapN mem (Binary oldval newval)));
498   effect(TEMP res, TEMP tmp1, TEMP tmp2, TEMP tmp3, KILL cr);
499   ins_cost(VOLATILE_REF_COST);
500   format %{ "cmpxchg_acq $mem, $oldval, $newval\t# (narrow oop)\n\t"
501             "cset $res, EQ" %}
502   ins_encode %{
503     assert_different_registers($oldval$$Register, $mem$$Register);
504     assert_different_registers($newval$$Register, $mem$$Register);
505     write_barrier_pre(masm, this,
506                       $mem$$Register /* obj */,
507                       $tmp1$$Register /* pre_val */,
508                       $tmp2$$Register /* tmp1 */,
509                       $tmp3$$Register /* tmp2 */,
510                       RegSet::of($mem$$Register, $oldval$$Register, $newval$$Register) /* preserve */,
511                       RegSet::of($res$$Register) /* no_preserve */);
512     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register, Assembler::word,
513                true /* acquire */, true /* release */, false /* weak */, noreg);
514     __ cset($res$$Register, Assembler::EQ);
515     __ decode_heap_oop($tmp1$$Register, $newval$$Register);
516     write_barrier_post(masm, this,
517                        $mem$$Register /* store_addr */,
518                        $tmp1$$Register /* new_val */,
519                        $tmp2$$Register /* tmp1 */,
520                        $tmp3$$Register /* tmp2 */);
521   %}
522   ins_pipe(pipe_slow);
523 %}
524 
525 // This pattern is generated automatically from g1_aarch64.m4.
526 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
527 instruct g1GetAndSetP(indirect mem, iRegP newval, iRegPNoSp tmp1, iRegPNoSp tmp2, iRegPNoSp preval, rFlagsReg cr)
528 %{
529   predicate(UseG1GC && !needs_acquiring_load_exclusive(n) && n->as_LoadStore()->barrier_data() != 0);
530   match(Set preval (GetAndSetP mem newval));
531   effect(TEMP preval, TEMP tmp1, TEMP tmp2, KILL cr);
532   ins_cost(2 * VOLATILE_REF_COST);
533   format %{ "atomic_xchg  $preval, $newval, [$mem]" %}
534   ins_encode %{
535     assert_different_registers($mem$$Register, $newval$$Register);
536     write_barrier_pre(masm, this,
537                       $mem$$Register /* obj */,
538                       $preval$$Register /* pre_val (as a temporary register) */,
539                       $tmp1$$Register /* tmp1 */,
540                       $tmp2$$Register /* tmp2 */,
541                       RegSet::of($mem$$Register, $preval$$Register, $newval$$Register) /* preserve */);
542     __ atomic_xchg($preval$$Register, $newval$$Register, $mem$$Register);
543     write_barrier_post(masm, this,
544                        $mem$$Register /* store_addr */,
545                        $newval$$Register /* new_val */,
546                        $tmp1$$Register /* tmp1 */,
547                        $tmp2$$Register /* tmp2 */);
548   %}
549   ins_pipe(pipe_serial);
550 %}
551 
552 // This pattern is generated automatically from g1_aarch64.m4.
553 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
554 instruct g1GetAndSetPAcq(indirect mem, iRegP newval, iRegPNoSp tmp1, iRegPNoSp tmp2, iRegPNoSp preval, rFlagsReg cr)
555 %{
556   predicate(UseG1GC && needs_acquiring_load_exclusive(n) && n->as_LoadStore()->barrier_data() != 0);
557   match(Set preval (GetAndSetP mem newval));
558   effect(TEMP preval, TEMP tmp1, TEMP tmp2, KILL cr);
559   ins_cost(VOLATILE_REF_COST);
560   format %{ "atomic_xchg_acq  $preval, $newval, [$mem]" %}
561   ins_encode %{
562     assert_different_registers($mem$$Register, $newval$$Register);
563     write_barrier_pre(masm, this,
564                       $mem$$Register /* obj */,
565                       $preval$$Register /* pre_val (as a temporary register) */,
566                       $tmp1$$Register /* tmp1 */,
567                       $tmp2$$Register /* tmp2 */,
568                       RegSet::of($mem$$Register, $preval$$Register, $newval$$Register) /* preserve */);
569     __ atomic_xchgal($preval$$Register, $newval$$Register, $mem$$Register);
570     write_barrier_post(masm, this,
571                        $mem$$Register /* store_addr */,
572                        $newval$$Register /* new_val */,
573                        $tmp1$$Register /* tmp1 */,
574                        $tmp2$$Register /* tmp2 */);
575   %}
576   ins_pipe(pipe_serial);
577 %}
578 
579 // This pattern is generated automatically from g1_aarch64.m4.
580 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
581 instruct g1GetAndSetN(indirect mem, iRegN newval, iRegPNoSp tmp1, iRegPNoSp tmp2, iRegPNoSp tmp3, iRegNNoSp preval, rFlagsReg cr)
582 %{
583   predicate(UseG1GC && !needs_acquiring_load_exclusive(n) && n->as_LoadStore()->barrier_data() != 0);
584   match(Set preval (GetAndSetN mem newval));
585   effect(TEMP preval, TEMP tmp1, TEMP tmp2, TEMP tmp3, KILL cr);
586   ins_cost(2 * VOLATILE_REF_COST);
587   format %{ "atomic_xchgw $preval, $newval, [$mem]" %}
588   ins_encode %{
589     assert_different_registers($mem$$Register, $newval$$Register);
590     write_barrier_pre(masm, this,
591                       $mem$$Register /* obj */,
592                       $tmp1$$Register /* pre_val */,
593                       $tmp2$$Register /* tmp1 */,
594                       $tmp3$$Register /* tmp2 */,
595                       RegSet::of($mem$$Register, $preval$$Register, $newval$$Register) /* preserve */);
596     __ atomic_xchgw($preval$$Register, $newval$$Register, $mem$$Register);
597     __ decode_heap_oop($tmp1$$Register, $newval$$Register);
598     write_barrier_post(masm, this,
599                        $mem$$Register /* store_addr */,
600                        $tmp1$$Register /* new_val */,
601                        $tmp2$$Register /* tmp1 */,
602                        $tmp3$$Register /* tmp2 */);
603   %}
604   ins_pipe(pipe_serial);
605 %}
606 
607 // This pattern is generated automatically from g1_aarch64.m4.
608 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
609 instruct g1GetAndSetNAcq(indirect mem, iRegN newval, iRegPNoSp tmp1, iRegPNoSp tmp2, iRegPNoSp tmp3, iRegNNoSp preval, rFlagsReg cr)
610 %{
611   predicate(UseG1GC && needs_acquiring_load_exclusive(n) && n->as_LoadStore()->barrier_data() != 0);
612   match(Set preval (GetAndSetN mem newval));
613   effect(TEMP preval, TEMP tmp1, TEMP tmp2, TEMP tmp3, KILL cr);
614   ins_cost(VOLATILE_REF_COST);
615   format %{ "atomic_xchgw_acq $preval, $newval, [$mem]" %}
616   ins_encode %{
617     assert_different_registers($mem$$Register, $newval$$Register);
618     write_barrier_pre(masm, this,
619                       $mem$$Register /* obj */,
620                       $tmp1$$Register /* pre_val */,
621                       $tmp2$$Register /* tmp1 */,
622                       $tmp3$$Register /* tmp2 */,
623                       RegSet::of($mem$$Register, $preval$$Register, $newval$$Register) /* preserve */);
624     __ atomic_xchgalw($preval$$Register, $newval$$Register, $mem$$Register);
625     __ decode_heap_oop($tmp1$$Register, $newval$$Register);
626     write_barrier_post(masm, this,
627                        $mem$$Register /* store_addr */,
628                        $tmp1$$Register /* new_val */,
629                        $tmp2$$Register /* tmp1 */,
630                        $tmp3$$Register /* tmp2 */);
631   %}
632   ins_pipe(pipe_serial);
633 %}
634 
635 // This pattern is generated automatically from g1_aarch64.m4.
636 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
637 instruct g1LoadP(iRegPNoSp dst, indirect mem, iRegPNoSp tmp1, iRegPNoSp tmp2, rFlagsReg cr)
638 %{
639   // This instruction does not need an acquiring counterpart because it is only
640   // used for reference loading (Reference::get()). The same holds for g1LoadN.
641   predicate(UseG1GC && !needs_acquiring_load(n) && n->as_Load()->barrier_data() != 0);
642   match(Set dst (LoadP mem));
643   effect(TEMP dst, TEMP tmp1, TEMP tmp2, KILL cr);
644   ins_cost(4 * INSN_COST);
645   format %{ "ldr  $dst, $mem\t# ptr" %}
646   ins_encode %{
647     __ ldr($dst$$Register, $mem$$Register);
648     write_barrier_pre(masm, this,
649                       noreg /* obj */,
650                       $dst$$Register /* pre_val */,
651                       $tmp1$$Register /* tmp1 */,
652                       $tmp2$$Register /* tmp2 */);
653   %}
654   ins_pipe(iload_reg_mem);
655 %}
656 
657 // This pattern is generated automatically from g1_aarch64.m4.
658 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
659 instruct g1LoadN(iRegNNoSp dst, indirect mem, iRegPNoSp tmp1, iRegPNoSp tmp2, iRegPNoSp tmp3, rFlagsReg cr)
660 %{
661   predicate(UseG1GC && !needs_acquiring_load(n) && n->as_Load()->barrier_data() != 0);
662   match(Set dst (LoadN mem));
663   effect(TEMP dst, TEMP tmp1, TEMP tmp2, TEMP tmp3, KILL cr);
664   ins_cost(4 * INSN_COST);
665   format %{ "ldrw  $dst, $mem\t# compressed ptr" %}
666   ins_encode %{
667     __ ldrw($dst$$Register, $mem$$Register);
668     if ((barrier_data() & G1C2BarrierPre) != 0) {
669       __ decode_heap_oop($tmp1$$Register, $dst$$Register);
670       write_barrier_pre(masm, this,
671                         noreg /* obj */,
672                         $tmp1$$Register /* pre_val */,
673                         $tmp2$$Register /* tmp1 */,
674                         $tmp3$$Register /* tmp2 */);
675     }
676   %}
677   ins_pipe(iload_reg_mem);
678 %}
679 
680 // END This section of the file is automatically generated. Do not edit --------------