1 //
  2 // Copyright (c) 2024, Oracle and/or its affiliates. All rights reserved.
  3 // DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  4 //
  5 // This code is free software; you can redistribute it and/or modify it
  6 // under the terms of the GNU General Public License version 2 only, as
  7 // published by the Free Software Foundation.
  8 //
  9 // This code is distributed in the hope that it will be useful, but WITHOUT
 10 // ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
 11 // FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
 12 // version 2 for more details (a copy is included in the LICENSE file that
 13 // accompanied this code).
 14 //
 15 // You should have received a copy of the GNU General Public License version
 16 // 2 along with this work; if not, write to the Free Software Foundation,
 17 // Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
 18 //
 19 // Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
 20 // or visit www.oracle.com if you need additional information or have any
 21 // questions.
 22 //
 23 
 24 source_hpp %{
 25 
 26 #include "gc/g1/c2/g1BarrierSetC2.hpp"
 27 #include "gc/shared/gc_globals.hpp"
 28 
 29 %}
 30 
 31 source %{
 32 
 33 #include "gc/g1/g1BarrierSetAssembler_aarch64.hpp"
 34 #include "gc/g1/g1BarrierSetRuntime.hpp"
 35 
 36 static void write_barrier_pre(MacroAssembler* masm,
 37                               const MachNode* node,
 38                               Register obj,
 39                               Register pre_val,
 40                               Register tmp1,
 41                               Register tmp2,
 42                               RegSet preserve = RegSet(),
 43                               RegSet no_preserve = RegSet()) {
 44   if (!G1PreBarrierStubC2::needs_barrier(node)) {
 45     return;
 46   }
 47   Assembler::InlineSkippedInstructionsCounter skip_counter(masm);
 48   G1BarrierSetAssembler* g1_asm = static_cast<G1BarrierSetAssembler*>(BarrierSet::barrier_set()->barrier_set_assembler());
 49   G1PreBarrierStubC2* const stub = G1PreBarrierStubC2::create(node);
 50   for (RegSetIterator<Register> reg = preserve.begin(); *reg != noreg; ++reg) {
 51     stub->preserve(*reg);
 52   }
 53   for (RegSetIterator<Register> reg = no_preserve.begin(); *reg != noreg; ++reg) {
 54     stub->dont_preserve(*reg);
 55   }
 56   g1_asm->g1_write_barrier_pre_c2(masm, obj, pre_val, rthread, tmp1, tmp2, stub);
 57 }
 58 
 59 static void write_barrier_post(MacroAssembler* masm,
 60                                const MachNode* node,
 61                                Register store_addr,
 62                                Register new_val,
 63                                Register tmp1,
 64                                Register tmp2,
 65                                RegSet preserve = RegSet()) {
 66   if (!G1PostBarrierStubC2::needs_barrier(node)) {
 67     return;
 68   }
 69   Assembler::InlineSkippedInstructionsCounter skip_counter(masm);
 70   G1BarrierSetAssembler* g1_asm = static_cast<G1BarrierSetAssembler*>(BarrierSet::barrier_set()->barrier_set_assembler());
 71   G1PostBarrierStubC2* const stub = G1PostBarrierStubC2::create(node);
 72   for (RegSetIterator<Register> reg = preserve.begin(); *reg != noreg; ++reg) {
 73     stub->preserve(*reg);
 74   }
 75   g1_asm->g1_write_barrier_post_c2(masm, store_addr, new_val, rthread, tmp1, tmp2, stub);
 76 }
 77 
 78 %}
 79 
 80 // TODO 8350865 (same applies to g1StoreLSpecialTwoOops)
 81 // - Can we use an unbound register for src?
 82 // - Do no set/overwrite barrier data here, also handle G1C2BarrierPostNotNull
 83 // - Is the zero-extend really required in all the places?
 84 // - Move this into the .m4?
 85 instruct g1StoreLSpecialOneOop(indirect mem, iRegL_R11 src, immI off, iRegPNoSp tmp1, iRegPNoSp tmp2, iRegPNoSp tmp3, iRegPNoSp tmp4, rFlagsReg cr)
 86 %{
 87   predicate(UseG1GC);
 88   match(Set mem (StoreLSpecial mem (Binary src off)));
 89   effect(TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, USE_KILL src, KILL cr);
 90   ins_cost(INSN_COST);
 91   format %{ "str  $src, $mem\t# g1StoreLSpecialOneOop" %}
 92   ins_encode %{
 93     ((MachNode*)this)->set_barrier_data(G1C2BarrierPre | G1C2BarrierPost);
 94 
 95     // Adjust address to point to narrow oop
 96     __ add($tmp4$$Register, $mem$$Register, $off$$constant);
 97     write_barrier_pre(masm, this,
 98                       $tmp4$$Register /* obj */,
 99                       $tmp1$$Register /* pre_val */,
100                       $tmp2$$Register /* tmp1 */,
101                       $tmp3$$Register /* tmp2 */,
102                       RegSet::of($mem$$Register, $src$$Register, $tmp4$$Register) /* preserve */);
103 
104     __ str($src$$Register, $mem$$Register);
105 
106     // Shift long value to extract the narrow oop field value and zero-extend it
107     __ lsr($src$$Register, $src$$Register, $off$$constant << LogBitsPerByte);
108     __ ubfm($src$$Register, $src$$Register, 0, 31);
109 
110     write_barrier_post(masm, this,
111                        $tmp4$$Register /* store_addr */,
112                        $src$$Register /* new_val */,
113                        $tmp2$$Register /* tmp1 */,
114                        $tmp3$$Register /* tmp2 */);
115   %}
116   ins_pipe(istore_reg_mem);
117 %}
118 
119 instruct g1StoreLSpecialTwoOops(indirect mem, iRegL_R11 src, iRegPNoSp tmp1, iRegPNoSp tmp2, iRegPNoSp tmp3, iRegPNoSp tmp4, rFlagsReg cr)
120 %{
121   predicate(UseG1GC);
122   match(Set mem (StoreLSpecial mem src));
123   effect(TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, USE_KILL src, KILL cr);
124   ins_cost(INSN_COST);
125   format %{ "str  $src, $mem\t# g1StoreLSpecialTwoOops" %}
126   ins_encode %{
127     ((MachNode*)this)->set_barrier_data(G1C2BarrierPre | G1C2BarrierPost);
128 
129     write_barrier_pre(masm, this,
130                       $mem$$Register /* obj */,
131                       $tmp1$$Register /* pre_val */,
132                       $tmp2$$Register /* tmp1 */,
133                       $tmp3$$Register /* tmp2 */,
134                       RegSet::of($mem$$Register, $src$$Register) /* preserve */);
135     // Adjust address to point to the second narrow oop in the long value
136     __ add($tmp4$$Register, $mem$$Register, 4);
137     write_barrier_pre(masm, this,
138                       $tmp4$$Register /* obj */,
139                       $tmp1$$Register /* pre_val */,
140                       $tmp2$$Register /* tmp1 */,
141                       $tmp3$$Register /* tmp2 */,
142                       RegSet::of($mem$$Register, $src$$Register, $tmp4$$Register) /* preserve */);
143 
144     __ str($src$$Register, $mem$$Register);
145 
146     // Zero-extend first narrow oop to long
147     __ ubfm($tmp1$$Register, $src$$Register, 0, 31);
148 
149     // Shift long value to extract the second narrow oop field value
150     __ lsr($src$$Register, $src$$Register, 32);
151 
152     write_barrier_post(masm, this,
153                        $mem$$Register /* store_addr */,
154                        $tmp1$$Register /* new_val */,
155                        $tmp2$$Register /* tmp1 */,
156                        $tmp3$$Register /* tmp2 */,
157                        RegSet::of($tmp1$$Register, $tmp4$$Register) /* preserve */);
158     write_barrier_post(masm, this,
159                        $tmp4$$Register /* store_addr */,
160                        $src$$Register /* new_val */,
161                        $tmp2$$Register /* tmp1 */,
162                        $tmp3$$Register /* tmp2 */);
163   %}
164   ins_pipe(istore_reg_mem);
165 %}
166 
167 
168 // BEGIN This section of the file is automatically generated. Do not edit --------------
169 
170 // This section is generated from g1_aarch64.m4
171 
172 
173 // This pattern is generated automatically from g1_aarch64.m4.
174 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
175 instruct g1StoreP(indirect mem, iRegP src, iRegPNoSp tmp1, iRegPNoSp tmp2, iRegPNoSp tmp3, rFlagsReg cr)
176 %{
177   predicate(UseG1GC && !needs_releasing_store(n) && n->as_Store()->barrier_data() != 0);
178   match(Set mem (StoreP mem src));
179   effect(TEMP tmp1, TEMP tmp2, TEMP tmp3, KILL cr);
180   ins_cost(INSN_COST);
181   format %{ "str  $src, $mem\t# ptr" %}
182   ins_encode %{
183     write_barrier_pre(masm, this,
184                       $mem$$Register /* obj */,
185                       $tmp1$$Register /* pre_val */,
186                       $tmp2$$Register /* tmp1 */,
187                       $tmp3$$Register /* tmp2 */,
188                       RegSet::of($mem$$Register, $src$$Register) /* preserve */);
189     __ str($src$$Register, $mem$$Register);
190     write_barrier_post(masm, this,
191                        $mem$$Register /* store_addr */,
192                        $src$$Register /* new_val */,
193                        $tmp2$$Register /* tmp1 */,
194                        $tmp3$$Register /* tmp2 */);
195   %}
196   ins_pipe(istore_reg_mem);
197 %}
198 
199 // This pattern is generated automatically from g1_aarch64.m4.
200 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
201 instruct g1StorePVolatile(indirect mem, iRegP src, iRegPNoSp tmp1, iRegPNoSp tmp2, iRegPNoSp tmp3, rFlagsReg cr)
202 %{
203   predicate(UseG1GC && needs_releasing_store(n) && n->as_Store()->barrier_data() != 0);
204   match(Set mem (StoreP mem src));
205   effect(TEMP tmp1, TEMP tmp2, TEMP tmp3, KILL cr);
206   ins_cost(VOLATILE_REF_COST);
207   format %{ "stlr  $src, $mem\t# ptr" %}
208   ins_encode %{
209     write_barrier_pre(masm, this,
210                       $mem$$Register /* obj */,
211                       $tmp1$$Register /* pre_val */,
212                       $tmp2$$Register /* tmp1 */,
213                       $tmp3$$Register /* tmp2 */,
214                       RegSet::of($mem$$Register, $src$$Register) /* preserve */);
215     __ stlr($src$$Register, $mem$$Register);
216     write_barrier_post(masm, this,
217                        $mem$$Register /* store_addr */,
218                        $src$$Register /* new_val */,
219                        $tmp2$$Register /* tmp1 */,
220                        $tmp3$$Register /* tmp2 */);
221   %}
222   ins_pipe(pipe_class_memory);
223 %}
224 
225 // This pattern is generated automatically from g1_aarch64.m4.
226 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
227 instruct g1StoreN(indirect mem, iRegN src, iRegPNoSp tmp1, iRegPNoSp tmp2, iRegPNoSp tmp3, rFlagsReg cr)
228 %{
229   predicate(UseG1GC && !needs_releasing_store(n) && n->as_Store()->barrier_data() != 0);
230   match(Set mem (StoreN mem src));
231   effect(TEMP tmp1, TEMP tmp2, TEMP tmp3, KILL cr);
232   ins_cost(INSN_COST);
233   format %{ "strw  $src, $mem\t# compressed ptr" %}
234   ins_encode %{
235     write_barrier_pre(masm, this,
236                       $mem$$Register /* obj */,
237                       $tmp1$$Register /* pre_val */,
238                       $tmp2$$Register /* tmp1 */,
239                       $tmp3$$Register /* tmp2 */,
240                       RegSet::of($mem$$Register, $src$$Register) /* preserve */);
241     __ strw($src$$Register, $mem$$Register);
242     if ((barrier_data() & G1C2BarrierPost) != 0) {
243       if ((barrier_data() & G1C2BarrierPostNotNull) == 0) {
244         __ decode_heap_oop($tmp1$$Register, $src$$Register);
245       } else {
246         __ decode_heap_oop_not_null($tmp1$$Register, $src$$Register);
247       }
248     }
249     write_barrier_post(masm, this,
250                        $mem$$Register /* store_addr */,
251                        $tmp1$$Register /* new_val */,
252                        $tmp2$$Register /* tmp1 */,
253                        $tmp3$$Register /* tmp2 */);
254   %}
255   ins_pipe(istore_reg_mem);
256 %}
257 
258 // This pattern is generated automatically from g1_aarch64.m4.
259 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
260 instruct g1StoreNVolatile(indirect mem, iRegN src, iRegPNoSp tmp1, iRegPNoSp tmp2, iRegPNoSp tmp3, rFlagsReg cr)
261 %{
262   predicate(UseG1GC && needs_releasing_store(n) && n->as_Store()->barrier_data() != 0);
263   match(Set mem (StoreN mem src));
264   effect(TEMP tmp1, TEMP tmp2, TEMP tmp3, KILL cr);
265   ins_cost(VOLATILE_REF_COST);
266   format %{ "stlrw  $src, $mem\t# compressed ptr" %}
267   ins_encode %{
268     write_barrier_pre(masm, this,
269                       $mem$$Register /* obj */,
270                       $tmp1$$Register /* pre_val */,
271                       $tmp2$$Register /* tmp1 */,
272                       $tmp3$$Register /* tmp2 */,
273                       RegSet::of($mem$$Register, $src$$Register) /* preserve */);
274     __ stlrw($src$$Register, $mem$$Register);
275     if ((barrier_data() & G1C2BarrierPost) != 0) {
276       if ((barrier_data() & G1C2BarrierPostNotNull) == 0) {
277         __ decode_heap_oop($tmp1$$Register, $src$$Register);
278       } else {
279         __ decode_heap_oop_not_null($tmp1$$Register, $src$$Register);
280       }
281     }
282     write_barrier_post(masm, this,
283                        $mem$$Register /* store_addr */,
284                        $tmp1$$Register /* new_val */,
285                        $tmp2$$Register /* tmp1 */,
286                        $tmp3$$Register /* tmp2 */);
287   %}
288   ins_pipe(pipe_class_memory);
289 %}
290 
291 // This pattern is generated automatically from g1_aarch64.m4.
292 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
293 instruct g1EncodePAndStoreN(indirect mem, iRegP src, iRegPNoSp tmp1, iRegPNoSp tmp2, iRegPNoSp tmp3, rFlagsReg cr)
294 %{
295   predicate(UseG1GC && !needs_releasing_store(n) && n->as_Store()->barrier_data() != 0);
296   match(Set mem (StoreN mem (EncodeP src)));
297   effect(TEMP tmp1, TEMP tmp2, TEMP tmp3, KILL cr);
298   ins_cost(INSN_COST);
299   format %{ "encode_heap_oop $tmp1, $src\n\t"
300             "strw  $tmp1, $mem\t# compressed ptr" %}
301   ins_encode %{
302     write_barrier_pre(masm, this,
303                       $mem$$Register /* obj */,
304                       $tmp1$$Register /* pre_val */,
305                       $tmp2$$Register /* tmp1 */,
306                       $tmp3$$Register /* tmp2 */,
307                       RegSet::of($mem$$Register, $src$$Register) /* preserve */);
308     if ((barrier_data() & G1C2BarrierPostNotNull) == 0) {
309       __ encode_heap_oop($tmp1$$Register, $src$$Register);
310     } else {
311       __ encode_heap_oop_not_null($tmp1$$Register, $src$$Register);
312     }
313     __ strw($tmp1$$Register, $mem$$Register);
314     write_barrier_post(masm, this,
315                        $mem$$Register /* store_addr */,
316                        $src$$Register /* new_val */,
317                        $tmp2$$Register /* tmp1 */,
318                        $tmp3$$Register /* tmp2 */);
319   %}
320   ins_pipe(istore_reg_mem);
321 %}
322 
323 // This pattern is generated automatically from g1_aarch64.m4.
324 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
325 instruct g1EncodePAndStoreNVolatile(indirect mem, iRegP src, iRegPNoSp tmp1, iRegPNoSp tmp2, iRegPNoSp tmp3, rFlagsReg cr)
326 %{
327   predicate(UseG1GC && needs_releasing_store(n) && n->as_Store()->barrier_data() != 0);
328   match(Set mem (StoreN mem (EncodeP src)));
329   effect(TEMP tmp1, TEMP tmp2, TEMP tmp3, KILL cr);
330   ins_cost(VOLATILE_REF_COST);
331   format %{ "encode_heap_oop $tmp1, $src\n\t"
332             "stlrw  $tmp1, $mem\t# compressed ptr" %}
333   ins_encode %{
334     write_barrier_pre(masm, this,
335                       $mem$$Register /* obj */,
336                       $tmp1$$Register /* pre_val */,
337                       $tmp2$$Register /* tmp1 */,
338                       $tmp3$$Register /* tmp2 */,
339                       RegSet::of($mem$$Register, $src$$Register) /* preserve */);
340     if ((barrier_data() & G1C2BarrierPostNotNull) == 0) {
341       __ encode_heap_oop($tmp1$$Register, $src$$Register);
342     } else {
343       __ encode_heap_oop_not_null($tmp1$$Register, $src$$Register);
344     }
345     __ stlrw($tmp1$$Register, $mem$$Register);
346     write_barrier_post(masm, this,
347                        $mem$$Register /* store_addr */,
348                        $src$$Register /* new_val */,
349                        $tmp2$$Register /* tmp1 */,
350                        $tmp3$$Register /* tmp2 */);
351   %}
352   ins_pipe(pipe_class_memory);
353 %}
354 
355 // This pattern is generated automatically from g1_aarch64.m4.
356 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
357 instruct g1CompareAndExchangeP(iRegPNoSp res, indirect mem, iRegP oldval, iRegP newval, iRegPNoSp tmp1, iRegPNoSp tmp2, rFlagsReg cr)
358 %{
359   predicate(UseG1GC && !needs_acquiring_load_exclusive(n) && n->as_LoadStore()->barrier_data() != 0);
360   match(Set res (CompareAndExchangeP mem (Binary oldval newval)));
361   effect(TEMP res, TEMP tmp1, TEMP tmp2, KILL cr);
362   ins_cost(2 * VOLATILE_REF_COST);
363   format %{ "cmpxchg $res = $mem, $oldval, $newval\t# ptr" %}
364   ins_encode %{
365     assert_different_registers($oldval$$Register, $mem$$Register);
366     assert_different_registers($newval$$Register, $mem$$Register);
367     // Pass $oldval to the pre-barrier (instead of loading from $mem), because
368     // $oldval is the only value that can be overwritten.
369     // The same holds for g1CompareAndSwapP and its Acq variant.
370     write_barrier_pre(masm, this,
371                       noreg /* obj */,
372                       $oldval$$Register /* pre_val */,
373                       $tmp1$$Register /* tmp1 */,
374                       $tmp2$$Register /* tmp2 */,
375                       RegSet::of($mem$$Register, $oldval$$Register, $newval$$Register) /* preserve */,
376                       RegSet::of($res$$Register) /* no_preserve */);
377     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register, Assembler::xword,
378                false /* acquire */, true /* release */, false /* weak */, $res$$Register);
379     write_barrier_post(masm, this,
380                        $mem$$Register /* store_addr */,
381                        $newval$$Register /* new_val */,
382                        $tmp1$$Register /* tmp1 */,
383                        $tmp2$$Register /* tmp2 */);
384   %}
385   ins_pipe(pipe_slow);
386 %}
387 
388 // This pattern is generated automatically from g1_aarch64.m4.
389 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
390 instruct g1CompareAndExchangePAcq(iRegPNoSp res, indirect mem, iRegP oldval, iRegP newval, iRegPNoSp tmp1, iRegPNoSp tmp2, rFlagsReg cr)
391 %{
392   predicate(UseG1GC && needs_acquiring_load_exclusive(n) && n->as_LoadStore()->barrier_data() != 0);
393   match(Set res (CompareAndExchangeP mem (Binary oldval newval)));
394   effect(TEMP res, TEMP tmp1, TEMP tmp2, KILL cr);
395   ins_cost(VOLATILE_REF_COST);
396   format %{ "cmpxchg_acq $res = $mem, $oldval, $newval\t# ptr" %}
397   ins_encode %{
398     assert_different_registers($oldval$$Register, $mem$$Register);
399     assert_different_registers($newval$$Register, $mem$$Register);
400     // Pass $oldval to the pre-barrier (instead of loading from $mem), because
401     // $oldval is the only value that can be overwritten.
402     // The same holds for g1CompareAndSwapP and its Acq variant.
403     write_barrier_pre(masm, this,
404                       noreg /* obj */,
405                       $oldval$$Register /* pre_val */,
406                       $tmp1$$Register /* tmp1 */,
407                       $tmp2$$Register /* tmp2 */,
408                       RegSet::of($mem$$Register, $oldval$$Register, $newval$$Register) /* preserve */,
409                       RegSet::of($res$$Register) /* no_preserve */);
410     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register, Assembler::xword,
411                true /* acquire */, true /* release */, false /* weak */, $res$$Register);
412     write_barrier_post(masm, this,
413                        $mem$$Register /* store_addr */,
414                        $newval$$Register /* new_val */,
415                        $tmp1$$Register /* tmp1 */,
416                        $tmp2$$Register /* tmp2 */);
417   %}
418   ins_pipe(pipe_slow);
419 %}
420 
421 // This pattern is generated automatically from g1_aarch64.m4.
422 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
423 instruct g1CompareAndExchangeN(iRegNNoSp res, indirect mem, iRegN oldval, iRegN newval, iRegPNoSp tmp1, iRegPNoSp tmp2, iRegPNoSp tmp3, rFlagsReg cr)
424 %{
425   predicate(UseG1GC && !needs_acquiring_load_exclusive(n) && n->as_LoadStore()->barrier_data() != 0);
426   match(Set res (CompareAndExchangeN mem (Binary oldval newval)));
427   effect(TEMP res, TEMP tmp1, TEMP tmp2, TEMP tmp3, KILL cr);
428   ins_cost(2 * VOLATILE_REF_COST);
429   format %{ "cmpxchg $res = $mem, $oldval, $newval\t# narrow oop" %}
430   ins_encode %{
431     assert_different_registers($oldval$$Register, $mem$$Register);
432     assert_different_registers($newval$$Register, $mem$$Register);
433     write_barrier_pre(masm, this,
434                       $mem$$Register /* obj */,
435                       $tmp1$$Register /* pre_val */,
436                       $tmp2$$Register /* tmp1 */,
437                       $tmp3$$Register /* tmp2 */,
438                       RegSet::of($mem$$Register, $oldval$$Register, $newval$$Register) /* preserve */,
439                       RegSet::of($res$$Register) /* no_preserve */);
440     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register, Assembler::word,
441                false /* acquire */, true /* release */, false /* weak */, $res$$Register);
442     __ decode_heap_oop($tmp1$$Register, $newval$$Register);
443     write_barrier_post(masm, this,
444                        $mem$$Register /* store_addr */,
445                        $tmp1$$Register /* new_val */,
446                        $tmp2$$Register /* tmp1 */,
447                        $tmp3$$Register /* tmp2 */);
448   %}
449   ins_pipe(pipe_slow);
450 %}
451 
452 // This pattern is generated automatically from g1_aarch64.m4.
453 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
454 instruct g1CompareAndExchangeNAcq(iRegNNoSp res, indirect mem, iRegN oldval, iRegN newval, iRegPNoSp tmp1, iRegPNoSp tmp2, iRegPNoSp tmp3, rFlagsReg cr)
455 %{
456   predicate(UseG1GC && needs_acquiring_load_exclusive(n) && n->as_LoadStore()->barrier_data() != 0);
457   match(Set res (CompareAndExchangeN mem (Binary oldval newval)));
458   effect(TEMP res, TEMP tmp1, TEMP tmp2, TEMP tmp3, KILL cr);
459   ins_cost(VOLATILE_REF_COST);
460   format %{ "cmpxchg_acq $res = $mem, $oldval, $newval\t# narrow oop" %}
461   ins_encode %{
462     assert_different_registers($oldval$$Register, $mem$$Register);
463     assert_different_registers($newval$$Register, $mem$$Register);
464     write_barrier_pre(masm, this,
465                       $mem$$Register /* obj */,
466                       $tmp1$$Register /* pre_val */,
467                       $tmp2$$Register /* tmp1 */,
468                       $tmp3$$Register /* tmp2 */,
469                       RegSet::of($mem$$Register, $oldval$$Register, $newval$$Register) /* preserve */,
470                       RegSet::of($res$$Register) /* no_preserve */);
471     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register, Assembler::word,
472                true /* acquire */, true /* release */, false /* weak */, $res$$Register);
473     __ decode_heap_oop($tmp1$$Register, $newval$$Register);
474     write_barrier_post(masm, this,
475                        $mem$$Register /* store_addr */,
476                        $tmp1$$Register /* new_val */,
477                        $tmp2$$Register /* tmp1 */,
478                        $tmp3$$Register /* tmp2 */);
479   %}
480   ins_pipe(pipe_slow);
481 %}
482 
483 // This pattern is generated automatically from g1_aarch64.m4.
484 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
485 instruct g1CompareAndSwapP(iRegINoSp res, indirect mem, iRegP newval, iRegPNoSp tmp1, iRegPNoSp tmp2, iRegP oldval, rFlagsReg cr)
486 %{
487   predicate(UseG1GC && !needs_acquiring_load_exclusive(n) && n->as_LoadStore()->barrier_data() != 0);
488   match(Set res (CompareAndSwapP mem (Binary oldval newval)));
489   match(Set res (WeakCompareAndSwapP mem (Binary oldval newval)));
490   effect(TEMP res, TEMP tmp1, TEMP tmp2, KILL cr);
491   ins_cost(2 * VOLATILE_REF_COST);
492   format %{ "cmpxchg $mem, $oldval, $newval\t# (ptr)\n\t"
493             "cset $res, EQ" %}
494   ins_encode %{
495     assert_different_registers($oldval$$Register, $mem$$Register);
496     assert_different_registers($newval$$Register, $mem$$Register);
497     write_barrier_pre(masm, this,
498                       noreg /* obj */,
499                       $oldval$$Register /* pre_val */,
500                       $tmp1$$Register /* tmp1 */,
501                       $tmp2$$Register /* tmp2 */,
502                       RegSet::of($mem$$Register, $oldval$$Register, $newval$$Register) /* preserve */,
503                       RegSet::of($res$$Register) /* no_preserve */);
504     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register, Assembler::xword,
505                false /* acquire */, true /* release */, false /* weak */, noreg);
506     __ cset($res$$Register, Assembler::EQ);
507     write_barrier_post(masm, this,
508                        $mem$$Register /* store_addr */,
509                        $newval$$Register /* new_val */,
510                        $tmp1$$Register /* tmp1 */,
511                        $tmp2$$Register /* tmp2 */);
512   %}
513   ins_pipe(pipe_slow);
514 %}
515 
516 // This pattern is generated automatically from g1_aarch64.m4.
517 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
518 instruct g1CompareAndSwapPAcq(iRegINoSp res, indirect mem, iRegP newval, iRegPNoSp tmp1, iRegPNoSp tmp2, iRegP oldval, rFlagsReg cr)
519 %{
520   predicate(UseG1GC && needs_acquiring_load_exclusive(n) && n->as_LoadStore()->barrier_data() != 0);
521   match(Set res (CompareAndSwapP mem (Binary oldval newval)));
522   match(Set res (WeakCompareAndSwapP mem (Binary oldval newval)));
523   effect(TEMP res, TEMP tmp1, TEMP tmp2, KILL cr);
524   ins_cost(VOLATILE_REF_COST);
525   format %{ "cmpxchg_acq $mem, $oldval, $newval\t# (ptr)\n\t"
526             "cset $res, EQ" %}
527   ins_encode %{
528     assert_different_registers($oldval$$Register, $mem$$Register);
529     assert_different_registers($newval$$Register, $mem$$Register);
530     write_barrier_pre(masm, this,
531                       noreg /* obj */,
532                       $oldval$$Register /* pre_val */,
533                       $tmp1$$Register /* tmp1 */,
534                       $tmp2$$Register /* tmp2 */,
535                       RegSet::of($mem$$Register, $oldval$$Register, $newval$$Register) /* preserve */,
536                       RegSet::of($res$$Register) /* no_preserve */);
537     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register, Assembler::xword,
538                true /* acquire */, true /* release */, false /* weak */, noreg);
539     __ cset($res$$Register, Assembler::EQ);
540     write_barrier_post(masm, this,
541                        $mem$$Register /* store_addr */,
542                        $newval$$Register /* new_val */,
543                        $tmp1$$Register /* tmp1 */,
544                        $tmp2$$Register /* tmp2 */);
545   %}
546   ins_pipe(pipe_slow);
547 %}
548 
549 // This pattern is generated automatically from g1_aarch64.m4.
550 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
551 instruct g1CompareAndSwapN(iRegINoSp res, indirect mem, iRegN newval, iRegPNoSp tmp1, iRegPNoSp tmp2, iRegPNoSp tmp3, iRegN oldval, rFlagsReg cr)
552 %{
553   predicate(UseG1GC && !needs_acquiring_load_exclusive(n) && n->as_LoadStore()->barrier_data() != 0);
554   match(Set res (CompareAndSwapN mem (Binary oldval newval)));
555   match(Set res (WeakCompareAndSwapN mem (Binary oldval newval)));
556   effect(TEMP res, TEMP tmp1, TEMP tmp2, TEMP tmp3, KILL cr);
557   ins_cost(2 * VOLATILE_REF_COST);
558   format %{ "cmpxchg $mem, $oldval, $newval\t# (narrow oop)\n\t"
559             "cset $res, EQ" %}
560   ins_encode %{
561     assert_different_registers($oldval$$Register, $mem$$Register);
562     assert_different_registers($newval$$Register, $mem$$Register);
563     write_barrier_pre(masm, this,
564                       $mem$$Register /* obj */,
565                       $tmp1$$Register /* pre_val */,
566                       $tmp2$$Register /* tmp1 */,
567                       $tmp3$$Register /* tmp2 */,
568                       RegSet::of($mem$$Register, $oldval$$Register, $newval$$Register) /* preserve */,
569                       RegSet::of($res$$Register) /* no_preserve */);
570     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register, Assembler::word,
571                false /* acquire */, true /* release */, false /* weak */, noreg);
572     __ cset($res$$Register, Assembler::EQ);
573     __ decode_heap_oop($tmp1$$Register, $newval$$Register);
574     write_barrier_post(masm, this,
575                        $mem$$Register /* store_addr */,
576                        $tmp1$$Register /* new_val */,
577                        $tmp2$$Register /* tmp1 */,
578                        $tmp3$$Register /* tmp2 */);
579   %}
580   ins_pipe(pipe_slow);
581 %}
582 
583 // This pattern is generated automatically from g1_aarch64.m4.
584 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
585 instruct g1CompareAndSwapNAcq(iRegINoSp res, indirect mem, iRegN newval, iRegPNoSp tmp1, iRegPNoSp tmp2, iRegPNoSp tmp3, iRegN oldval, rFlagsReg cr)
586 %{
587   predicate(UseG1GC && needs_acquiring_load_exclusive(n) && n->as_LoadStore()->barrier_data() != 0);
588   match(Set res (CompareAndSwapN mem (Binary oldval newval)));
589   match(Set res (WeakCompareAndSwapN mem (Binary oldval newval)));
590   effect(TEMP res, TEMP tmp1, TEMP tmp2, TEMP tmp3, KILL cr);
591   ins_cost(VOLATILE_REF_COST);
592   format %{ "cmpxchg_acq $mem, $oldval, $newval\t# (narrow oop)\n\t"
593             "cset $res, EQ" %}
594   ins_encode %{
595     assert_different_registers($oldval$$Register, $mem$$Register);
596     assert_different_registers($newval$$Register, $mem$$Register);
597     write_barrier_pre(masm, this,
598                       $mem$$Register /* obj */,
599                       $tmp1$$Register /* pre_val */,
600                       $tmp2$$Register /* tmp1 */,
601                       $tmp3$$Register /* tmp2 */,
602                       RegSet::of($mem$$Register, $oldval$$Register, $newval$$Register) /* preserve */,
603                       RegSet::of($res$$Register) /* no_preserve */);
604     __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register, Assembler::word,
605                true /* acquire */, true /* release */, false /* weak */, noreg);
606     __ cset($res$$Register, Assembler::EQ);
607     __ decode_heap_oop($tmp1$$Register, $newval$$Register);
608     write_barrier_post(masm, this,
609                        $mem$$Register /* store_addr */,
610                        $tmp1$$Register /* new_val */,
611                        $tmp2$$Register /* tmp1 */,
612                        $tmp3$$Register /* tmp2 */);
613   %}
614   ins_pipe(pipe_slow);
615 %}
616 
617 // This pattern is generated automatically from g1_aarch64.m4.
618 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
619 instruct g1GetAndSetP(indirect mem, iRegP newval, iRegPNoSp tmp1, iRegPNoSp tmp2, iRegPNoSp preval, rFlagsReg cr)
620 %{
621   predicate(UseG1GC && !needs_acquiring_load_exclusive(n) && n->as_LoadStore()->barrier_data() != 0);
622   match(Set preval (GetAndSetP mem newval));
623   effect(TEMP preval, TEMP tmp1, TEMP tmp2, KILL cr);
624   ins_cost(2 * VOLATILE_REF_COST);
625   format %{ "atomic_xchg  $preval, $newval, [$mem]" %}
626   ins_encode %{
627     assert_different_registers($mem$$Register, $newval$$Register);
628     write_barrier_pre(masm, this,
629                       $mem$$Register /* obj */,
630                       $preval$$Register /* pre_val (as a temporary register) */,
631                       $tmp1$$Register /* tmp1 */,
632                       $tmp2$$Register /* tmp2 */,
633                       RegSet::of($mem$$Register, $preval$$Register, $newval$$Register) /* preserve */);
634     __ atomic_xchg($preval$$Register, $newval$$Register, $mem$$Register);
635     write_barrier_post(masm, this,
636                        $mem$$Register /* store_addr */,
637                        $newval$$Register /* new_val */,
638                        $tmp1$$Register /* tmp1 */,
639                        $tmp2$$Register /* tmp2 */);
640   %}
641   ins_pipe(pipe_serial);
642 %}
643 
644 // This pattern is generated automatically from g1_aarch64.m4.
645 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
646 instruct g1GetAndSetPAcq(indirect mem, iRegP newval, iRegPNoSp tmp1, iRegPNoSp tmp2, iRegPNoSp preval, rFlagsReg cr)
647 %{
648   predicate(UseG1GC && needs_acquiring_load_exclusive(n) && n->as_LoadStore()->barrier_data() != 0);
649   match(Set preval (GetAndSetP mem newval));
650   effect(TEMP preval, TEMP tmp1, TEMP tmp2, KILL cr);
651   ins_cost(VOLATILE_REF_COST);
652   format %{ "atomic_xchg_acq  $preval, $newval, [$mem]" %}
653   ins_encode %{
654     assert_different_registers($mem$$Register, $newval$$Register);
655     write_barrier_pre(masm, this,
656                       $mem$$Register /* obj */,
657                       $preval$$Register /* pre_val (as a temporary register) */,
658                       $tmp1$$Register /* tmp1 */,
659                       $tmp2$$Register /* tmp2 */,
660                       RegSet::of($mem$$Register, $preval$$Register, $newval$$Register) /* preserve */);
661     __ atomic_xchgal($preval$$Register, $newval$$Register, $mem$$Register);
662     write_barrier_post(masm, this,
663                        $mem$$Register /* store_addr */,
664                        $newval$$Register /* new_val */,
665                        $tmp1$$Register /* tmp1 */,
666                        $tmp2$$Register /* tmp2 */);
667   %}
668   ins_pipe(pipe_serial);
669 %}
670 
671 // This pattern is generated automatically from g1_aarch64.m4.
672 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
673 instruct g1GetAndSetN(indirect mem, iRegN newval, iRegPNoSp tmp1, iRegPNoSp tmp2, iRegPNoSp tmp3, iRegNNoSp preval, rFlagsReg cr)
674 %{
675   predicate(UseG1GC && !needs_acquiring_load_exclusive(n) && n->as_LoadStore()->barrier_data() != 0);
676   match(Set preval (GetAndSetN mem newval));
677   effect(TEMP preval, TEMP tmp1, TEMP tmp2, TEMP tmp3, KILL cr);
678   ins_cost(2 * VOLATILE_REF_COST);
679   format %{ "atomic_xchgw $preval, $newval, [$mem]" %}
680   ins_encode %{
681     assert_different_registers($mem$$Register, $newval$$Register);
682     write_barrier_pre(masm, this,
683                       $mem$$Register /* obj */,
684                       $tmp1$$Register /* pre_val */,
685                       $tmp2$$Register /* tmp1 */,
686                       $tmp3$$Register /* tmp2 */,
687                       RegSet::of($mem$$Register, $preval$$Register, $newval$$Register) /* preserve */);
688     __ atomic_xchgw($preval$$Register, $newval$$Register, $mem$$Register);
689     __ decode_heap_oop($tmp1$$Register, $newval$$Register);
690     write_barrier_post(masm, this,
691                        $mem$$Register /* store_addr */,
692                        $tmp1$$Register /* new_val */,
693                        $tmp2$$Register /* tmp1 */,
694                        $tmp3$$Register /* tmp2 */);
695   %}
696   ins_pipe(pipe_serial);
697 %}
698 
699 // This pattern is generated automatically from g1_aarch64.m4.
700 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
701 instruct g1GetAndSetNAcq(indirect mem, iRegN newval, iRegPNoSp tmp1, iRegPNoSp tmp2, iRegPNoSp tmp3, iRegNNoSp preval, rFlagsReg cr)
702 %{
703   predicate(UseG1GC && needs_acquiring_load_exclusive(n) && n->as_LoadStore()->barrier_data() != 0);
704   match(Set preval (GetAndSetN mem newval));
705   effect(TEMP preval, TEMP tmp1, TEMP tmp2, TEMP tmp3, KILL cr);
706   ins_cost(VOLATILE_REF_COST);
707   format %{ "atomic_xchgw_acq $preval, $newval, [$mem]" %}
708   ins_encode %{
709     assert_different_registers($mem$$Register, $newval$$Register);
710     write_barrier_pre(masm, this,
711                       $mem$$Register /* obj */,
712                       $tmp1$$Register /* pre_val */,
713                       $tmp2$$Register /* tmp1 */,
714                       $tmp3$$Register /* tmp2 */,
715                       RegSet::of($mem$$Register, $preval$$Register, $newval$$Register) /* preserve */);
716     __ atomic_xchgalw($preval$$Register, $newval$$Register, $mem$$Register);
717     __ decode_heap_oop($tmp1$$Register, $newval$$Register);
718     write_barrier_post(masm, this,
719                        $mem$$Register /* store_addr */,
720                        $tmp1$$Register /* new_val */,
721                        $tmp2$$Register /* tmp1 */,
722                        $tmp3$$Register /* tmp2 */);
723   %}
724   ins_pipe(pipe_serial);
725 %}
726 
727 // This pattern is generated automatically from g1_aarch64.m4.
728 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
729 instruct g1LoadP(iRegPNoSp dst, indirect mem, iRegPNoSp tmp1, iRegPNoSp tmp2, rFlagsReg cr)
730 %{
731   // This instruction does not need an acquiring counterpart because it is only
732   // used for reference loading (Reference::get()). The same holds for g1LoadN.
733   predicate(UseG1GC && !needs_acquiring_load(n) && n->as_Load()->barrier_data() != 0);
734   match(Set dst (LoadP mem));
735   effect(TEMP dst, TEMP tmp1, TEMP tmp2, KILL cr);
736   ins_cost(4 * INSN_COST);
737   format %{ "ldr  $dst, $mem\t# ptr" %}
738   ins_encode %{
739     __ ldr($dst$$Register, $mem$$Register);
740     write_barrier_pre(masm, this,
741                       noreg /* obj */,
742                       $dst$$Register /* pre_val */,
743                       $tmp1$$Register /* tmp1 */,
744                       $tmp2$$Register /* tmp2 */);
745   %}
746   ins_pipe(iload_reg_mem);
747 %}
748 
749 // This pattern is generated automatically from g1_aarch64.m4.
750 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
751 instruct g1LoadN(iRegNNoSp dst, indirect mem, iRegPNoSp tmp1, iRegPNoSp tmp2, iRegPNoSp tmp3, rFlagsReg cr)
752 %{
753   predicate(UseG1GC && !needs_acquiring_load(n) && n->as_Load()->barrier_data() != 0);
754   match(Set dst (LoadN mem));
755   effect(TEMP dst, TEMP tmp1, TEMP tmp2, TEMP tmp3, KILL cr);
756   ins_cost(4 * INSN_COST);
757   format %{ "ldrw  $dst, $mem\t# compressed ptr" %}
758   ins_encode %{
759     __ ldrw($dst$$Register, $mem$$Register);
760     if ((barrier_data() & G1C2BarrierPre) != 0) {
761       __ decode_heap_oop($tmp1$$Register, $dst$$Register);
762       write_barrier_pre(masm, this,
763                         noreg /* obj */,
764                         $tmp1$$Register /* pre_val */,
765                         $tmp2$$Register /* tmp1 */,
766                         $tmp3$$Register /* tmp2 */);
767     }
768   %}
769   ins_pipe(iload_reg_mem);
770 %}
771 
772 // END This section of the file is automatically generated. Do not edit --------------