1 //
2 // Copyright (c) 2024, 2025, Oracle and/or its affiliates. All rights reserved.
3 // Copyright (c) 2024, Huawei Technologies Co., Ltd. All rights reserved.
4 // DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
5 //
6 // This code is free software; you can redistribute it and/or modify it
7 // under the terms of the GNU General Public License version 2 only, as
8 // published by the Free Software Foundation.
9 //
10 // This code is distributed in the hope that it will be useful, but WITHOUT
11 // ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
12 // FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
13 // version 2 for more details (a copy is included in the LICENSE file that
14 // accompanied this code).
15 //
16 // You should have received a copy of the GNU General Public License version
17 // 2 along with this work; if not, write to the Free Software Foundation,
18 // Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
19 //
20 // Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
21 // or visit www.oracle.com if you need additional information or have any
22 // questions.
23 //
24
25 source_hpp %{
26
27 #include "gc/g1/c2/g1BarrierSetC2.hpp"
28 #include "gc/shared/gc_globals.hpp"
29
30 %}
31
32 source %{
33
34 #include "gc/g1/g1BarrierSetAssembler_riscv.hpp"
35 #include "gc/g1/g1BarrierSetRuntime.hpp"
36
37 static void write_barrier_pre(MacroAssembler* masm,
38 const MachNode* node,
39 Register obj,
40 Register pre_val,
41 Register tmp1,
42 Register tmp2,
43 RegSet preserve = RegSet(),
44 RegSet no_preserve = RegSet()) {
45 if (!G1PreBarrierStubC2::needs_barrier(node)) {
46 return;
47 }
48 Assembler::InlineSkippedInstructionsCounter skip_counter(masm);
49 G1BarrierSetAssembler* g1_asm = static_cast<G1BarrierSetAssembler*>(BarrierSet::barrier_set()->barrier_set_assembler());
50 G1PreBarrierStubC2* const stub = G1PreBarrierStubC2::create(node);
51 for (RegSetIterator<Register> reg = preserve.begin(); *reg != noreg; ++reg) {
52 stub->preserve(*reg);
53 }
54 for (RegSetIterator<Register> reg = no_preserve.begin(); *reg != noreg; ++reg) {
55 stub->dont_preserve(*reg);
56 }
57 g1_asm->g1_write_barrier_pre_c2(masm, obj, pre_val, xthread, tmp1, tmp2, stub);
58 }
59
60 static void write_barrier_post(MacroAssembler* masm,
61 const MachNode* node,
62 Register store_addr,
63 Register new_val,
64 Register tmp1,
65 Register tmp2) {
66 if (!G1BarrierStubC2::needs_post_barrier(node)) {
67 return;
68 }
69 Assembler::InlineSkippedInstructionsCounter skip_counter(masm);
70 G1BarrierSetAssembler* g1_asm = static_cast<G1BarrierSetAssembler*>(BarrierSet::barrier_set()->barrier_set_assembler());
71 bool new_val_may_be_null = G1BarrierStubC2::post_new_val_may_be_null(node);
72 g1_asm->g1_write_barrier_post_c2(masm, store_addr, new_val, xthread, tmp1, tmp2, new_val_may_be_null);
73 }
74
75 %}
76
77 instruct g1StoreP(indirect mem, iRegP src, iRegPNoSp tmp1, iRegPNoSp tmp2, iRegPNoSp tmp3, rFlagsReg cr)
78 %{
79 predicate(UseG1GC && n->as_Store()->barrier_data() != 0);
80 match(Set mem (StoreP mem src));
81 effect(TEMP tmp1, TEMP tmp2, TEMP tmp3, KILL cr);
82 ins_cost(STORE_COST);
83 format %{ "sd $src, $mem\t# ptr" %}
84 ins_encode %{
85 guarantee($mem$$disp == 0, "impossible encoding");
86 write_barrier_pre(masm, this,
87 $mem$$Register /* obj */,
88 $tmp1$$Register /* pre_val */,
89 $tmp2$$Register /* tmp1 */,
90 $tmp3$$Register /* tmp2 */,
91 RegSet::of($mem$$Register, $src$$Register) /* preserve */);
92 __ sd($src$$Register, Address($mem$$Register));
93 write_barrier_post(masm, this,
94 $mem$$Register /* store_addr */,
95 $src$$Register /* new_val */,
96 $tmp2$$Register /* tmp1 */,
97 $tmp3$$Register /* tmp2 */);
98 %}
99 ins_pipe(istore_reg_mem);
100 %}
101
102 instruct g1StoreN(indirect mem, iRegN src, iRegPNoSp tmp1, iRegPNoSp tmp2, iRegPNoSp tmp3, rFlagsReg cr)
103 %{
104 predicate(UseG1GC && n->as_Store()->barrier_data() != 0);
105 match(Set mem (StoreN mem src));
106 effect(TEMP tmp1, TEMP tmp2, TEMP tmp3, KILL cr);
107 ins_cost(STORE_COST);
108 format %{ "sw $src, $mem\t# compressed ptr" %}
109 ins_encode %{
110 guarantee($mem$$disp == 0, "impossible encoding");
111 write_barrier_pre(masm, this,
112 $mem$$Register /* obj */,
113 $tmp1$$Register /* pre_val */,
114 $tmp2$$Register /* tmp1 */,
115 $tmp3$$Register /* tmp2 */,
116 RegSet::of($mem$$Register, $src$$Register) /* preserve */);
117 __ sw($src$$Register, Address($mem$$Register));
118 if ((barrier_data() & G1C2BarrierPost) != 0) {
119 if ((barrier_data() & G1C2BarrierPostNotNull) == 0) {
120 __ decode_heap_oop($tmp1$$Register, $src$$Register);
121 } else {
122 __ decode_heap_oop_not_null($tmp1$$Register, $src$$Register);
123 }
124 }
125 write_barrier_post(masm, this,
126 $mem$$Register /* store_addr */,
127 $tmp1$$Register /* new_val */,
128 $tmp2$$Register /* tmp1 */,
129 $tmp3$$Register /* tmp2 */);
130 %}
131 ins_pipe(istore_reg_mem);
132 %}
133
134 instruct g1EncodePAndStoreN(indirect mem, iRegP src, iRegPNoSp tmp1, iRegPNoSp tmp2, iRegPNoSp tmp3, rFlagsReg cr)
135 %{
136 predicate(UseG1GC && n->as_Store()->barrier_data() != 0);
137 match(Set mem (StoreN mem (EncodeP src)));
138 effect(TEMP tmp1, TEMP tmp2, TEMP tmp3, KILL cr);
139 ins_cost(STORE_COST);
140 format %{ "encode_heap_oop $tmp1, $src\n\t"
141 "sw $tmp1, $mem\t# compressed ptr" %}
142 ins_encode %{
143 guarantee($mem$$disp == 0, "impossible encoding");
144 write_barrier_pre(masm, this,
145 $mem$$Register /* obj */,
146 $tmp1$$Register /* pre_val */,
147 $tmp2$$Register /* tmp1 */,
148 $tmp3$$Register /* tmp2 */,
149 RegSet::of($mem$$Register, $src$$Register) /* preserve */);
150 if ((barrier_data() & G1C2BarrierPostNotNull) == 0) {
151 __ encode_heap_oop($tmp1$$Register, $src$$Register);
152 } else {
153 __ encode_heap_oop_not_null($tmp1$$Register, $src$$Register);
154 }
155 __ sw($tmp1$$Register, Address($mem$$Register));
156 write_barrier_post(masm, this,
157 $mem$$Register /* store_addr */,
158 $src$$Register /* new_val */,
159 $tmp2$$Register /* tmp1 */,
160 $tmp3$$Register /* tmp2 */);
161 %}
162 ins_pipe(istore_reg_mem);
163 %}
164
165 instruct g1CompareAndExchangeP(iRegPNoSp res, indirect mem, iRegP oldval, iRegP newval, iRegPNoSp tmp1, iRegPNoSp tmp2, rFlagsReg cr)
166 %{
167 predicate(UseG1GC && n->as_LoadStore()->barrier_data() != 0);
168 match(Set res (CompareAndExchangeP mem (Binary oldval newval)));
169 effect(TEMP res, TEMP tmp1, TEMP tmp2, KILL cr);
170 ins_cost(2 * VOLATILE_REF_COST);
171 format %{ "cmpxchg $res = $mem, $oldval, $newval\t# ptr" %}
172 ins_encode %{
173 guarantee($mem$$disp == 0, "impossible encoding");
174 assert_different_registers($oldval$$Register, $mem$$Register);
175 assert_different_registers($newval$$Register, $mem$$Register);
176 // Pass $oldval to the pre-barrier (instead of loading from $mem), because
177 // $oldval is the only value that can be overwritten.
178 // The same holds for g1CompareAndSwapP and its Acq variant.
179 write_barrier_pre(masm, this,
180 noreg /* obj */,
181 $oldval$$Register /* pre_val */,
182 $tmp1$$Register /* tmp1 */,
183 $tmp2$$Register /* tmp2 */,
184 RegSet::of($mem$$Register, $oldval$$Register, $newval$$Register) /* preserve */,
185 RegSet::of($res$$Register) /* no_preserve */);
186 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register, Assembler::int64,
187 /*acquire*/ Assembler::relaxed, /*release*/ Assembler::rl, $res$$Register);
188 write_barrier_post(masm, this,
189 $mem$$Register /* store_addr */,
190 $newval$$Register /* new_val */,
191 $tmp1$$Register /* tmp1 */,
192 $tmp2$$Register /* tmp2 */);
193 %}
194 ins_pipe(pipe_slow);
195 %}
196
197 instruct g1CompareAndExchangePAcq(iRegPNoSp res, indirect mem, iRegP oldval, iRegP newval, iRegPNoSp tmp1, iRegPNoSp tmp2, rFlagsReg cr)
198 %{
199 predicate(UseG1GC && needs_acquiring_load_reserved(n) && n->as_LoadStore()->barrier_data() != 0);
200 match(Set res (CompareAndExchangeP mem (Binary oldval newval)));
201 effect(TEMP res, TEMP tmp1, TEMP tmp2, KILL cr);
202 ins_cost(VOLATILE_REF_COST);
203 format %{ "cmpxchg_acq $res = $mem, $oldval, $newval\t# ptr" %}
204 ins_encode %{
205 guarantee($mem$$disp == 0, "impossible encoding");
206 assert_different_registers($oldval$$Register, $mem$$Register);
207 assert_different_registers($newval$$Register, $mem$$Register);
208 // Pass $oldval to the pre-barrier (instead of loading from $mem), because
209 // $oldval is the only value that can be overwritten.
210 // The same holds for g1CompareAndSwapP and its Acq variant.
211 write_barrier_pre(masm, this,
212 noreg /* obj */,
213 $oldval$$Register /* pre_val */,
214 $tmp1$$Register /* tmp1 */,
215 $tmp2$$Register /* tmp2 */,
216 RegSet::of($mem$$Register, $oldval$$Register, $newval$$Register) /* preserve */,
217 RegSet::of($res$$Register) /* no_preserve */);
218 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register, Assembler::int64,
219 /*acquire*/ Assembler::aq, /*release*/ Assembler::rl, $res$$Register);
220 write_barrier_post(masm, this,
221 $mem$$Register /* store_addr */,
222 $newval$$Register /* new_val */,
223 $tmp1$$Register /* tmp1 */,
224 $tmp2$$Register /* tmp2 */);
225 %}
226 ins_pipe(pipe_slow);
227 %}
228
229 instruct g1CompareAndExchangeN(iRegNNoSp res, indirect mem, iRegN oldval, iRegN newval, iRegPNoSp tmp1, iRegPNoSp tmp2, iRegPNoSp tmp3, rFlagsReg cr)
230 %{
231 predicate(UseG1GC && n->as_LoadStore()->barrier_data() != 0);
232 match(Set res (CompareAndExchangeN mem (Binary oldval newval)));
233 effect(TEMP res, TEMP tmp1, TEMP tmp2, TEMP tmp3, KILL cr);
234 ins_cost(2 * VOLATILE_REF_COST);
235 format %{ "cmpxchg $res = $mem, $oldval, $newval\t# narrow oop" %}
236 ins_encode %{
237 guarantee($mem$$disp == 0, "impossible encoding");
238 assert_different_registers($oldval$$Register, $mem$$Register);
239 assert_different_registers($newval$$Register, $mem$$Register);
240 write_barrier_pre(masm, this,
241 $mem$$Register /* obj */,
242 $tmp1$$Register /* pre_val */,
243 $tmp2$$Register /* tmp1 */,
244 $tmp3$$Register /* tmp2 */,
245 RegSet::of($mem$$Register, $oldval$$Register, $newval$$Register) /* preserve */,
246 RegSet::of($res$$Register) /* no_preserve */);
247 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register, Assembler::uint32,
248 /*acquire*/ Assembler::relaxed, /*release*/ Assembler::rl, $res$$Register);
249 __ decode_heap_oop($tmp1$$Register, $newval$$Register);
250 write_barrier_post(masm, this,
251 $mem$$Register /* store_addr */,
252 $tmp1$$Register /* new_val */,
253 $tmp2$$Register /* tmp1 */,
254 $tmp3$$Register /* tmp2 */);
255 %}
256 ins_pipe(pipe_slow);
257 %}
258
259 instruct g1CompareAndExchangeNAcq(iRegNNoSp res, indirect mem, iRegN oldval, iRegN newval, iRegPNoSp tmp1, iRegPNoSp tmp2, iRegPNoSp tmp3, rFlagsReg cr)
260 %{
261 predicate(UseG1GC && needs_acquiring_load_reserved(n) && n->as_LoadStore()->barrier_data() != 0);
262 match(Set res (CompareAndExchangeN mem (Binary oldval newval)));
263 effect(TEMP res, TEMP tmp1, TEMP tmp2, TEMP tmp3, KILL cr);
264 ins_cost(VOLATILE_REF_COST);
265 format %{ "cmpxchg_acq $res = $mem, $oldval, $newval\t# narrow oop" %}
266 ins_encode %{
267 guarantee($mem$$disp == 0, "impossible encoding");
268 assert_different_registers($oldval$$Register, $mem$$Register);
269 assert_different_registers($newval$$Register, $mem$$Register);
270 write_barrier_pre(masm, this,
271 $mem$$Register /* obj */,
272 $tmp1$$Register /* pre_val */,
273 $tmp2$$Register /* tmp1 */,
274 $tmp3$$Register /* tmp2 */,
275 RegSet::of($mem$$Register, $oldval$$Register, $newval$$Register) /* preserve */,
276 RegSet::of($res$$Register) /* no_preserve */);
277 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register, Assembler::uint32,
278 /*acquire*/ Assembler::aq, /*release*/ Assembler::rl, $res$$Register);
279 __ decode_heap_oop($tmp1$$Register, $newval$$Register);
280 write_barrier_post(masm, this,
281 $mem$$Register /* store_addr */,
282 $tmp1$$Register /* new_val */,
283 $tmp2$$Register /* tmp1 */,
284 $tmp3$$Register /* tmp2 */);
285 %}
286 ins_pipe(pipe_slow);
287 %}
288
289 instruct g1CompareAndSwapP(iRegINoSp res, indirect mem, iRegP newval, iRegPNoSp tmp1, iRegPNoSp tmp2, iRegP oldval, rFlagsReg cr)
290 %{
291 predicate(UseG1GC && n->as_LoadStore()->barrier_data() != 0);
292 match(Set res (CompareAndSwapP mem (Binary oldval newval)));
293 match(Set res (WeakCompareAndSwapP mem (Binary oldval newval)));
294 effect(TEMP res, TEMP tmp1, TEMP tmp2, KILL cr);
295 ins_cost(2 * VOLATILE_REF_COST);
296 format %{ "cmpxchg $mem, $oldval, $newval\t# (ptr)\n\t"
297 "mv $res, $res == $oldval" %}
298 ins_encode %{
299 guarantee($mem$$disp == 0, "impossible encoding");
300 assert_different_registers($oldval$$Register, $mem$$Register);
301 assert_different_registers($newval$$Register, $mem$$Register);
302 write_barrier_pre(masm, this,
303 noreg /* obj */,
304 $oldval$$Register /* pre_val */,
305 $tmp1$$Register /* tmp1 */,
306 $tmp2$$Register /* tmp2 */,
307 RegSet::of($mem$$Register, $oldval$$Register, $newval$$Register) /* preserve */,
308 RegSet::of($res$$Register) /* no_preserve */);
309 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register, Assembler::int64,
310 /*acquire*/ Assembler::relaxed, /*release*/ Assembler::rl, $res$$Register,
311 /*result as bool*/ true);
312 write_barrier_post(masm, this,
313 $mem$$Register /* store_addr */,
314 $newval$$Register /* new_val */,
315 $tmp1$$Register /* tmp1 */,
316 $tmp2$$Register /* tmp2 */);
317 %}
318 ins_pipe(pipe_slow);
319 %}
320
321 instruct g1CompareAndSwapPAcq(iRegINoSp res, indirect mem, iRegP newval, iRegPNoSp tmp1, iRegPNoSp tmp2, iRegP oldval, rFlagsReg cr)
322 %{
323 predicate(UseG1GC && needs_acquiring_load_reserved(n) && n->as_LoadStore()->barrier_data() != 0);
324 match(Set res (CompareAndSwapP mem (Binary oldval newval)));
325 match(Set res (WeakCompareAndSwapP mem (Binary oldval newval)));
326 effect(TEMP res, TEMP tmp1, TEMP tmp2, KILL cr);
327 ins_cost(VOLATILE_REF_COST);
328 format %{ "cmpxchg_acq $mem, $oldval, $newval\t# (ptr)\n\t"
329 "mv $res, $res == $oldval" %}
330 ins_encode %{
331 guarantee($mem$$disp == 0, "impossible encoding");
332 assert_different_registers($oldval$$Register, $mem$$Register);
333 assert_different_registers($newval$$Register, $mem$$Register);
334 write_barrier_pre(masm, this,
335 noreg /* obj */,
336 $oldval$$Register /* pre_val */,
337 $tmp1$$Register /* tmp1 */,
338 $tmp2$$Register /* tmp2 */,
339 RegSet::of($mem$$Register, $oldval$$Register, $newval$$Register) /* preserve */,
340 RegSet::of($res$$Register) /* no_preserve */);
341 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register, Assembler::int64,
342 /*acquire*/ Assembler::aq, /*release*/ Assembler::rl, $res$$Register,
343 /*result as bool*/ true);
344 write_barrier_post(masm, this,
345 $mem$$Register /* store_addr */,
346 $newval$$Register /* new_val */,
347 $tmp1$$Register /* tmp1 */,
348 $tmp2$$Register /* tmp2 */);
349 %}
350 ins_pipe(pipe_slow);
351 %}
352
353 instruct g1CompareAndSwapN(iRegINoSp res, indirect mem, iRegN newval, iRegPNoSp tmp1, iRegPNoSp tmp2, iRegPNoSp tmp3, iRegN oldval, rFlagsReg cr)
354 %{
355 predicate(UseG1GC && n->as_LoadStore()->barrier_data() != 0);
356 match(Set res (CompareAndSwapN mem (Binary oldval newval)));
357 match(Set res (WeakCompareAndSwapN mem (Binary oldval newval)));
358 effect(TEMP res, TEMP tmp1, TEMP tmp2, TEMP tmp3, KILL cr);
359 ins_cost(2 * VOLATILE_REF_COST);
360 format %{ "cmpxchg $mem, $oldval, $newval\t# (narrow oop)\n\t"
361 "mv $res, $res == $oldval" %}
362 ins_encode %{
363 guarantee($mem$$disp == 0, "impossible encoding");
364 assert_different_registers($oldval$$Register, $mem$$Register);
365 assert_different_registers($newval$$Register, $mem$$Register);
366 write_barrier_pre(masm, this,
367 $mem$$Register /* obj */,
368 $tmp1$$Register /* pre_val */,
369 $tmp2$$Register /* tmp1 */,
370 $tmp3$$Register /* tmp2 */,
371 RegSet::of($mem$$Register, $oldval$$Register, $newval$$Register) /* preserve */,
372 RegSet::of($res$$Register) /* no_preserve */);
373 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register, Assembler::uint32,
374 /*acquire*/ Assembler::relaxed, /*release*/ Assembler::rl, $res$$Register,
375 /*result as bool*/ true);
376 __ decode_heap_oop($tmp1$$Register, $newval$$Register);
377 write_barrier_post(masm, this,
378 $mem$$Register /* store_addr */,
379 $tmp1$$Register /* new_val */,
380 $tmp2$$Register /* tmp1 */,
381 $tmp3$$Register /* tmp2 */);
382 %}
383 ins_pipe(pipe_slow);
384 %}
385
386 instruct g1CompareAndSwapNAcq(iRegINoSp res, indirect mem, iRegN newval, iRegPNoSp tmp1, iRegPNoSp tmp2, iRegPNoSp tmp3, iRegN oldval, rFlagsReg cr)
387 %{
388 predicate(UseG1GC && needs_acquiring_load_reserved(n) && n->as_LoadStore()->barrier_data() != 0);
389 match(Set res (CompareAndSwapN mem (Binary oldval newval)));
390 match(Set res (WeakCompareAndSwapN mem (Binary oldval newval)));
391 effect(TEMP res, TEMP tmp1, TEMP tmp2, TEMP tmp3, KILL cr);
392 ins_cost(VOLATILE_REF_COST);
393 format %{ "cmpxchg_acq $mem, $oldval, $newval\t# (narrow oop)\n\t"
394 "mv $res, $res == $oldval" %}
395 ins_encode %{
396 guarantee($mem$$disp == 0, "impossible encoding");
397 assert_different_registers($oldval$$Register, $mem$$Register);
398 assert_different_registers($newval$$Register, $mem$$Register);
399 write_barrier_pre(masm, this,
400 $mem$$Register /* obj */,
401 $tmp1$$Register /* pre_val */,
402 $tmp2$$Register /* tmp1 */,
403 $tmp3$$Register /* tmp2 */,
404 RegSet::of($mem$$Register, $oldval$$Register, $newval$$Register) /* preserve */,
405 RegSet::of($res$$Register) /* no_preserve */);
406 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register, Assembler::uint32,
407 /*acquire*/ Assembler::aq, /*release*/ Assembler::rl, $res$$Register,
408 /*result as bool*/ true);
409 __ decode_heap_oop($tmp1$$Register, $newval$$Register);
410 write_barrier_post(masm, this,
411 $mem$$Register /* store_addr */,
412 $tmp1$$Register /* new_val */,
413 $tmp2$$Register /* tmp1 */,
414 $tmp3$$Register /* tmp2 */);
415 %}
416 ins_pipe(pipe_slow);
417 %}
418
419 instruct g1GetAndSetP(indirect mem, iRegP newval, iRegPNoSp tmp1, iRegPNoSp tmp2, iRegPNoSp preval, rFlagsReg cr)
420 %{
421 predicate(UseG1GC && n->as_LoadStore()->barrier_data() != 0);
422 match(Set preval (GetAndSetP mem newval));
423 effect(TEMP preval, TEMP tmp1, TEMP tmp2, KILL cr);
424 ins_cost(2 * VOLATILE_REF_COST);
425 format %{ "atomic_xchg $preval, $newval, [$mem]" %}
426 ins_encode %{
427 guarantee($mem$$disp == 0, "impossible encoding");
428 assert_different_registers($mem$$Register, $newval$$Register);
429 write_barrier_pre(masm, this,
430 $mem$$Register /* obj */,
431 $preval$$Register /* pre_val (as a temporary register) */,
432 $tmp1$$Register /* tmp1 */,
433 $tmp2$$Register /* tmp2 */,
434 RegSet::of($mem$$Register, $preval$$Register, $newval$$Register) /* preserve */);
435 __ atomic_xchg($preval$$Register, $newval$$Register, $mem$$Register);
436 write_barrier_post(masm, this,
437 $mem$$Register /* store_addr */,
438 $newval$$Register /* new_val */,
439 $tmp1$$Register /* tmp1 */,
440 $tmp2$$Register /* tmp2 */);
441 %}
442 ins_pipe(pipe_serial);
443 %}
444
445 instruct g1GetAndSetPAcq(indirect mem, iRegP newval, iRegPNoSp tmp1, iRegPNoSp tmp2, iRegPNoSp preval, rFlagsReg cr)
446 %{
447 predicate(UseG1GC && needs_acquiring_load_reserved(n) && n->as_LoadStore()->barrier_data() != 0);
448 match(Set preval (GetAndSetP mem newval));
449 effect(TEMP preval, TEMP tmp1, TEMP tmp2, KILL cr);
450 ins_cost(VOLATILE_REF_COST);
451 format %{ "atomic_xchg_acq $preval, $newval, [$mem]" %}
452 ins_encode %{
453 guarantee($mem$$disp == 0, "impossible encoding");
454 assert_different_registers($mem$$Register, $newval$$Register);
455 write_barrier_pre(masm, this,
456 $mem$$Register /* obj */,
457 $preval$$Register /* pre_val (as a temporary register) */,
458 $tmp1$$Register /* tmp1 */,
459 $tmp2$$Register /* tmp2 */,
460 RegSet::of($mem$$Register, $preval$$Register, $newval$$Register) /* preserve */);
461 __ atomic_xchgal($preval$$Register, $newval$$Register, $mem$$Register);
462 write_barrier_post(masm, this,
463 $mem$$Register /* store_addr */,
464 $newval$$Register /* new_val */,
465 $tmp1$$Register /* tmp1 */,
466 $tmp2$$Register /* tmp2 */);
467 %}
468 ins_pipe(pipe_serial);
469 %}
470
471 instruct g1GetAndSetN(indirect mem, iRegN newval, iRegPNoSp tmp1, iRegPNoSp tmp2, iRegPNoSp tmp3, iRegNNoSp preval, rFlagsReg cr)
472 %{
473 predicate(UseG1GC && n->as_LoadStore()->barrier_data() != 0);
474 match(Set preval (GetAndSetN mem newval));
475 effect(TEMP preval, TEMP tmp1, TEMP tmp2, TEMP tmp3, KILL cr);
476 ins_cost(2 * VOLATILE_REF_COST);
477 format %{ "atomic_xchgwu $preval, $newval, [$mem]" %}
478 ins_encode %{
479 guarantee($mem$$disp == 0, "impossible encoding");
480 assert_different_registers($mem$$Register, $newval$$Register);
481 write_barrier_pre(masm, this,
482 $mem$$Register /* obj */,
483 $tmp1$$Register /* pre_val */,
484 $tmp2$$Register /* tmp1 */,
485 $tmp3$$Register /* tmp2 */,
486 RegSet::of($mem$$Register, $preval$$Register, $newval$$Register) /* preserve */);
487 __ atomic_xchgwu($preval$$Register, $newval$$Register, $mem$$Register);
488 __ decode_heap_oop($tmp1$$Register, $newval$$Register);
489 write_barrier_post(masm, this,
490 $mem$$Register /* store_addr */,
491 $tmp1$$Register /* new_val */,
492 $tmp2$$Register /* tmp1 */,
493 $tmp3$$Register /* tmp2 */);
494 %}
495 ins_pipe(pipe_serial);
496 %}
497
498 instruct g1GetAndSetNAcq(indirect mem, iRegN newval, iRegPNoSp tmp1, iRegPNoSp tmp2, iRegPNoSp tmp3, iRegNNoSp preval, rFlagsReg cr)
499 %{
500 predicate(UseG1GC && needs_acquiring_load_reserved(n) && n->as_LoadStore()->barrier_data() != 0);
501 match(Set preval (GetAndSetN mem newval));
502 effect(TEMP preval, TEMP tmp1, TEMP tmp2, TEMP tmp3, KILL cr);
503 ins_cost(VOLATILE_REF_COST);
504 format %{ "atomic_xchgwu_acq $preval, $newval, [$mem]" %}
505 ins_encode %{
506 guarantee($mem$$disp == 0, "impossible encoding");
507 assert_different_registers($mem$$Register, $newval$$Register);
508 write_barrier_pre(masm, this,
509 $mem$$Register /* obj */,
510 $tmp1$$Register /* pre_val */,
511 $tmp2$$Register /* tmp1 */,
512 $tmp3$$Register /* tmp2 */,
513 RegSet::of($mem$$Register, $preval$$Register, $newval$$Register) /* preserve */);
514 __ atomic_xchgalwu($preval$$Register, $newval$$Register, $mem$$Register);
515 __ decode_heap_oop($tmp1$$Register, $newval$$Register);
516 write_barrier_post(masm, this,
517 $mem$$Register /* store_addr */,
518 $tmp1$$Register /* new_val */,
519 $tmp2$$Register /* tmp1 */,
520 $tmp3$$Register /* tmp2 */);
521 %}
522 ins_pipe(pipe_serial);
523 %}
524
525 instruct g1LoadP(iRegPNoSp dst, indirect mem, iRegPNoSp tmp1, iRegPNoSp tmp2, rFlagsReg cr)
526 %{
527 predicate(UseG1GC && n->as_Load()->barrier_data() != 0);
528 match(Set dst (LoadP mem));
529 effect(TEMP dst, TEMP tmp1, TEMP tmp2, KILL cr);
530 ins_cost(LOAD_COST + BRANCH_COST);
531 format %{ "ld $dst, $mem\t# ptr" %}
532 ins_encode %{
533 guarantee($mem$$disp == 0, "impossible encoding");
534 __ ld($dst$$Register, Address($mem$$Register));
535 write_barrier_pre(masm, this,
536 noreg /* obj */,
537 $dst$$Register /* pre_val */,
538 $tmp1$$Register /* tmp1 */,
539 $tmp2$$Register /* tmp2 */);
540 %}
541 ins_pipe(iload_reg_mem);
542 %}
543
544 instruct g1LoadN(iRegNNoSp dst, indirect mem, iRegPNoSp tmp1, iRegPNoSp tmp2, iRegPNoSp tmp3, rFlagsReg cr)
545 %{
546 predicate(UseG1GC && n->as_Load()->barrier_data() != 0);
547 match(Set dst (LoadN mem));
548 effect(TEMP dst, TEMP tmp1, TEMP tmp2, TEMP tmp3, KILL cr);
549 ins_cost(LOAD_COST + BRANCH_COST);
550 format %{ "lwu $dst, $mem\t# compressed ptr" %}
551 ins_encode %{
552 guarantee($mem$$disp == 0, "impossible encoding");
553 __ lwu($dst$$Register, Address($mem$$Register));
554 if ((barrier_data() & G1C2BarrierPre) != 0) {
555 __ decode_heap_oop($tmp1$$Register, $dst$$Register);
556 write_barrier_pre(masm, this,
557 noreg /* obj */,
558 $tmp1$$Register /* pre_val */,
559 $tmp2$$Register /* tmp1 */,
560 $tmp3$$Register /* tmp2 */);
561 }
562 %}
563 ins_pipe(iload_reg_mem);
564 %}