1 //
2 // Copyright (c) 2024, 2026, Oracle and/or its affiliates. All rights reserved.
3 // Copyright (c) 2024, Huawei Technologies Co., Ltd. All rights reserved.
4 // DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
5 //
6 // This code is free software; you can redistribute it and/or modify it
7 // under the terms of the GNU General Public License version 2 only, as
8 // published by the Free Software Foundation.
9 //
10 // This code is distributed in the hope that it will be useful, but WITHOUT
11 // ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
12 // FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
13 // version 2 for more details (a copy is included in the LICENSE file that
14 // accompanied this code).
15 //
16 // You should have received a copy of the GNU General Public License version
17 // 2 along with this work; if not, write to the Free Software Foundation,
18 // Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
19 //
20 // Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
21 // or visit www.oracle.com if you need additional information or have any
22 // questions.
23 //
24
25 source_hpp %{
26
27 #include "gc/g1/c2/g1BarrierSetC2.hpp"
28 #include "gc/shared/gc_globals.hpp"
29
30 %}
31
32 source %{
33
34 #include "gc/g1/g1BarrierSetAssembler_riscv.hpp"
35 #include "gc/g1/g1BarrierSetRuntime.hpp"
36
37 static void write_barrier_pre(MacroAssembler* masm,
38 const MachNode* node,
39 Register obj,
40 Register pre_val,
41 Register tmp1,
42 Register tmp2,
43 RegSet preserve = RegSet(),
44 RegSet no_preserve = RegSet()) {
45 if (!G1PreBarrierStubC2::needs_barrier(node)) {
46 return;
47 }
48 Assembler::InlineSkippedInstructionsCounter skip_counter(masm);
49 G1BarrierSetAssembler* g1_asm = static_cast<G1BarrierSetAssembler*>(BarrierSet::barrier_set()->barrier_set_assembler());
50 G1PreBarrierStubC2* const stub = G1PreBarrierStubC2::create(node);
51 for (RegSetIterator<Register> reg = preserve.begin(); *reg != noreg; ++reg) {
52 stub->preserve(*reg);
53 }
54 for (RegSetIterator<Register> reg = no_preserve.begin(); *reg != noreg; ++reg) {
55 stub->dont_preserve(*reg);
56 }
57 g1_asm->g1_write_barrier_pre_c2(masm, obj, pre_val, xthread, tmp1, tmp2, stub);
58 }
59
60 static void write_barrier_post(MacroAssembler* masm,
61 const MachNode* node,
62 Register store_addr,
63 Register new_val,
64 Register tmp1,
65 Register tmp2) {
66 if (!G1BarrierStubC2::needs_post_barrier(node)) {
67 return;
68 }
69 Assembler::InlineSkippedInstructionsCounter skip_counter(masm);
70 G1BarrierSetAssembler* g1_asm = static_cast<G1BarrierSetAssembler*>(BarrierSet::barrier_set()->barrier_set_assembler());
71 bool new_val_may_be_null = G1BarrierStubC2::post_new_val_may_be_null(node);
72 g1_asm->g1_write_barrier_post_c2(masm, store_addr, new_val, xthread, tmp1, tmp2, new_val_may_be_null);
73 }
74
75 %}
76
77
78 // TODO 8350865 (same applies to g1StoreLSpecialTwoOops)
79 // - Do not set/overwrite barrier data here, also handle G1C2BarrierPostNotNull
80
81 instruct g1StoreLSpecialOneOopOff0(indirect mem, iRegLNoSp src, immI0 off, iRegPNoSp tmp1, iRegPNoSp tmp2, iRegPNoSp tmp3, rFlagsReg cr)
82 %{
83 predicate(UseG1GC);
84 match(Set mem (StoreLSpecial mem (Binary src off)));
85 effect(TEMP tmp1, TEMP tmp2, TEMP tmp3, KILL cr);
86 ins_cost(STORE_COST);
87 format %{ "sd $src, $mem\t# g1StoreLSpecialOneOopOff0" %}
88 ins_encode %{
89 guarantee($mem$$disp == 0, "impossible encoding");
90 ((MachNode*)this)->set_barrier_data(G1C2BarrierPre | G1C2BarrierPost);
91
92 write_barrier_pre(masm, this,
93 $mem$$Register /* obj */,
94 $tmp1$$Register /* pre_val */,
95 $tmp2$$Register /* tmp1 */,
96 $tmp3$$Register /* tmp2 */,
97 RegSet::of($mem$$Register, $src$$Register) /* preserve */);
98
99 __ sd($src$$Register, Address($mem$$Register));
100
101 // Extract the narrow oop field value
102 __ zext($tmp1$$Register, $src$$Register, 32);
103 __ decode_heap_oop($tmp1$$Register, $tmp1$$Register);
104 write_barrier_post(masm, this,
105 $mem$$Register /* store_addr */,
106 $tmp1$$Register /* new_val */,
107 $tmp2$$Register /* tmp1 */,
108 $tmp3$$Register /* tmp2 */);
109 %}
110 ins_pipe(istore_reg_mem);
111 %}
112
113 instruct g1StoreLSpecialOneOopOff4(indirect mem, iRegLNoSp src, immI_4 off, iRegPNoSp tmp1, iRegPNoSp tmp2, iRegPNoSp tmp3, iRegPNoSp tmp4, rFlagsReg cr)
114 %{
115 predicate(UseG1GC);
116 match(Set mem (StoreLSpecial mem (Binary src off)));
117 effect(TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, KILL cr);
118 ins_cost(STORE_COST);
119 format %{ "sd $src, $mem\t# g1StoreLSpecialOneOopOff4" %}
120 ins_encode %{
121 guarantee($mem$$disp == 0, "impossible encoding");
122 ((MachNode*)this)->set_barrier_data(G1C2BarrierPre | G1C2BarrierPost);
123
124 // Adjust address to point to narrow oop
125 __ add($tmp4$$Register, $mem$$Register, 4);
126 write_barrier_pre(masm, this,
127 $tmp4$$Register /* obj */,
128 $tmp1$$Register /* pre_val */,
129 $tmp2$$Register /* tmp1 */,
130 $tmp3$$Register /* tmp2 */,
131 RegSet::of($mem$$Register, $src$$Register, $tmp4$$Register) /* preserve */);
132
133 __ sd($src$$Register, Address($mem$$Register));
134
135 // Shift long value to extract the narrow oop field value and zero-extend it
136 __ srli($tmp1$$Register, $src$$Register, 32);
137 __ decode_heap_oop($tmp1$$Register, $tmp1$$Register);
138 write_barrier_post(masm, this,
139 $tmp4$$Register /* store_addr */,
140 $tmp1$$Register /* new_val */,
141 $tmp2$$Register /* tmp1 */,
142 $tmp3$$Register /* tmp2 */);
143 %}
144 ins_pipe(istore_reg_mem);
145 %}
146
147 instruct g1StoreLSpecialTwoOops(indirect mem, iRegLNoSp src, iRegPNoSp tmp1, iRegPNoSp tmp2, iRegPNoSp tmp3, iRegPNoSp tmp4, rFlagsReg cr)
148 %{
149 predicate(UseG1GC);
150 match(Set mem (StoreLSpecial mem src));
151 effect(TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, KILL cr);
152 ins_cost(STORE_COST);
153 format %{ "sd $src, $mem\t# g1StoreLSpecialTwoOops" %}
154 ins_encode %{
155 guarantee($mem$$disp == 0, "impossible encoding");
156 ((MachNode*)this)->set_barrier_data(G1C2BarrierPre | G1C2BarrierPost);
157
158 write_barrier_pre(masm, this,
159 $mem$$Register /* obj */,
160 $tmp1$$Register /* pre_val */,
161 $tmp2$$Register /* tmp1 */,
162 $tmp3$$Register /* tmp2 */,
163 RegSet::of($mem$$Register, $src$$Register) /* preserve */);
164 // Adjust address to point to the second narrow oop in the long value
165 __ add($tmp4$$Register, $mem$$Register, 4);
166 write_barrier_pre(masm, this,
167 $tmp4$$Register /* obj */,
168 $tmp1$$Register /* pre_val */,
169 $tmp2$$Register /* tmp1 */,
170 $tmp3$$Register /* tmp2 */,
171 RegSet::of($mem$$Register, $src$$Register, $tmp4$$Register) /* preserve */);
172
173 __ sd($src$$Register, Address($mem$$Register));
174
175 // Zero-extend first narrow oop to long
176 __ zext($tmp1$$Register, $src$$Register, 32);
177 __ decode_heap_oop($tmp1$$Register, $tmp1$$Register);
178 write_barrier_post(masm, this,
179 $mem$$Register /* store_addr */,
180 $tmp1$$Register /* new_val */,
181 $tmp2$$Register /* tmp1 */,
182 $tmp3$$Register /* tmp2 */);
183
184 // Shift long value to extract the second narrow oop field value
185 __ srli($tmp1$$Register, $src$$Register, 32);
186 __ decode_heap_oop($tmp1$$Register, $tmp1$$Register);
187 write_barrier_post(masm, this,
188 $tmp4$$Register /* store_addr */,
189 $tmp1$$Register /* new_val */,
190 $tmp2$$Register /* tmp1 */,
191 $tmp3$$Register /* tmp2 */);
192 %}
193 ins_pipe(istore_reg_mem);
194 %}
195
196
197 instruct g1StoreP(indirect mem, iRegP src, iRegPNoSp tmp1, iRegPNoSp tmp2, iRegPNoSp tmp3, rFlagsReg cr)
198 %{
199 predicate(UseG1GC && n->as_Store()->barrier_data() != 0);
200 match(Set mem (StoreP mem src));
201 effect(TEMP tmp1, TEMP tmp2, TEMP tmp3, KILL cr);
202 ins_cost(STORE_COST);
203 format %{ "sd $src, $mem\t# ptr" %}
204 ins_encode %{
205 guarantee($mem$$disp == 0, "impossible encoding");
206 write_barrier_pre(masm, this,
207 $mem$$Register /* obj */,
208 $tmp1$$Register /* pre_val */,
209 $tmp2$$Register /* tmp1 */,
210 $tmp3$$Register /* tmp2 */,
211 RegSet::of($mem$$Register, $src$$Register) /* preserve */);
212 __ sd($src$$Register, Address($mem$$Register));
213 write_barrier_post(masm, this,
214 $mem$$Register /* store_addr */,
215 $src$$Register /* new_val */,
216 $tmp2$$Register /* tmp1 */,
217 $tmp3$$Register /* tmp2 */);
218 %}
219 ins_pipe(istore_reg_mem);
220 %}
221
222 instruct g1StoreN(indirect mem, iRegN src, iRegPNoSp tmp1, iRegPNoSp tmp2, iRegPNoSp tmp3, rFlagsReg cr)
223 %{
224 predicate(UseG1GC && n->as_Store()->barrier_data() != 0);
225 match(Set mem (StoreN mem src));
226 effect(TEMP tmp1, TEMP tmp2, TEMP tmp3, KILL cr);
227 ins_cost(STORE_COST);
228 format %{ "sw $src, $mem\t# compressed ptr" %}
229 ins_encode %{
230 guarantee($mem$$disp == 0, "impossible encoding");
231 write_barrier_pre(masm, this,
232 $mem$$Register /* obj */,
233 $tmp1$$Register /* pre_val */,
234 $tmp2$$Register /* tmp1 */,
235 $tmp3$$Register /* tmp2 */,
236 RegSet::of($mem$$Register, $src$$Register) /* preserve */);
237 __ sw($src$$Register, Address($mem$$Register));
238 if ((barrier_data() & G1C2BarrierPost) != 0) {
239 if ((barrier_data() & G1C2BarrierPostNotNull) == 0) {
240 __ decode_heap_oop($tmp1$$Register, $src$$Register);
241 } else {
242 __ decode_heap_oop_not_null($tmp1$$Register, $src$$Register);
243 }
244 }
245 write_barrier_post(masm, this,
246 $mem$$Register /* store_addr */,
247 $tmp1$$Register /* new_val */,
248 $tmp2$$Register /* tmp1 */,
249 $tmp3$$Register /* tmp2 */);
250 %}
251 ins_pipe(istore_reg_mem);
252 %}
253
254 instruct g1EncodePAndStoreN(indirect mem, iRegP src, iRegPNoSp tmp1, iRegPNoSp tmp2, iRegPNoSp tmp3, rFlagsReg cr)
255 %{
256 predicate(UseG1GC && n->as_Store()->barrier_data() != 0);
257 match(Set mem (StoreN mem (EncodeP src)));
258 effect(TEMP tmp1, TEMP tmp2, TEMP tmp3, KILL cr);
259 ins_cost(STORE_COST);
260 format %{ "encode_heap_oop $tmp1, $src\n\t"
261 "sw $tmp1, $mem\t# compressed ptr" %}
262 ins_encode %{
263 guarantee($mem$$disp == 0, "impossible encoding");
264 write_barrier_pre(masm, this,
265 $mem$$Register /* obj */,
266 $tmp1$$Register /* pre_val */,
267 $tmp2$$Register /* tmp1 */,
268 $tmp3$$Register /* tmp2 */,
269 RegSet::of($mem$$Register, $src$$Register) /* preserve */);
270 if ((barrier_data() & G1C2BarrierPostNotNull) == 0) {
271 __ encode_heap_oop($tmp1$$Register, $src$$Register);
272 } else {
273 __ encode_heap_oop_not_null($tmp1$$Register, $src$$Register);
274 }
275 __ sw($tmp1$$Register, Address($mem$$Register));
276 write_barrier_post(masm, this,
277 $mem$$Register /* store_addr */,
278 $src$$Register /* new_val */,
279 $tmp2$$Register /* tmp1 */,
280 $tmp3$$Register /* tmp2 */);
281 %}
282 ins_pipe(istore_reg_mem);
283 %}
284
285 instruct g1CompareAndExchangeP(iRegPNoSp res, indirect mem, iRegP oldval, iRegP newval, iRegPNoSp tmp1, iRegPNoSp tmp2, rFlagsReg cr)
286 %{
287 predicate(UseG1GC && n->as_LoadStore()->barrier_data() != 0);
288 match(Set res (CompareAndExchangeP mem (Binary oldval newval)));
289 effect(TEMP res, TEMP tmp1, TEMP tmp2, KILL cr);
290 ins_cost(2 * VOLATILE_REF_COST);
291 format %{ "cmpxchg $res = $mem, $oldval, $newval\t# ptr" %}
292 ins_encode %{
293 guarantee($mem$$disp == 0, "impossible encoding");
294 assert_different_registers($oldval$$Register, $mem$$Register);
295 assert_different_registers($newval$$Register, $mem$$Register);
296 // Pass $oldval to the pre-barrier (instead of loading from $mem), because
297 // $oldval is the only value that can be overwritten.
298 // The same holds for g1CompareAndSwapP and its Acq variant.
299 write_barrier_pre(masm, this,
300 noreg /* obj */,
301 $oldval$$Register /* pre_val */,
302 $tmp1$$Register /* tmp1 */,
303 $tmp2$$Register /* tmp2 */,
304 RegSet::of($mem$$Register, $oldval$$Register, $newval$$Register) /* preserve */,
305 RegSet::of($res$$Register) /* no_preserve */);
306 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register, Assembler::int64,
307 /*acquire*/ Assembler::relaxed, /*release*/ Assembler::rl, $res$$Register);
308 write_barrier_post(masm, this,
309 $mem$$Register /* store_addr */,
310 $newval$$Register /* new_val */,
311 $tmp1$$Register /* tmp1 */,
312 $tmp2$$Register /* tmp2 */);
313 %}
314 ins_pipe(pipe_slow);
315 %}
316
317 instruct g1CompareAndExchangePAcq(iRegPNoSp res, indirect mem, iRegP oldval, iRegP newval, iRegPNoSp tmp1, iRegPNoSp tmp2, rFlagsReg cr)
318 %{
319 predicate(UseG1GC && needs_acquiring_load_reserved(n) && n->as_LoadStore()->barrier_data() != 0);
320 match(Set res (CompareAndExchangeP mem (Binary oldval newval)));
321 effect(TEMP res, TEMP tmp1, TEMP tmp2, KILL cr);
322 ins_cost(VOLATILE_REF_COST);
323 format %{ "cmpxchg_acq $res = $mem, $oldval, $newval\t# ptr" %}
324 ins_encode %{
325 guarantee($mem$$disp == 0, "impossible encoding");
326 assert_different_registers($oldval$$Register, $mem$$Register);
327 assert_different_registers($newval$$Register, $mem$$Register);
328 // Pass $oldval to the pre-barrier (instead of loading from $mem), because
329 // $oldval is the only value that can be overwritten.
330 // The same holds for g1CompareAndSwapP and its Acq variant.
331 write_barrier_pre(masm, this,
332 noreg /* obj */,
333 $oldval$$Register /* pre_val */,
334 $tmp1$$Register /* tmp1 */,
335 $tmp2$$Register /* tmp2 */,
336 RegSet::of($mem$$Register, $oldval$$Register, $newval$$Register) /* preserve */,
337 RegSet::of($res$$Register) /* no_preserve */);
338 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register, Assembler::int64,
339 /*acquire*/ Assembler::aq, /*release*/ Assembler::rl, $res$$Register);
340 write_barrier_post(masm, this,
341 $mem$$Register /* store_addr */,
342 $newval$$Register /* new_val */,
343 $tmp1$$Register /* tmp1 */,
344 $tmp2$$Register /* tmp2 */);
345 %}
346 ins_pipe(pipe_slow);
347 %}
348
349 instruct g1CompareAndExchangeN(iRegNNoSp res, indirect mem, iRegN oldval, iRegN newval, iRegPNoSp tmp1, iRegPNoSp tmp2, iRegPNoSp tmp3, rFlagsReg cr)
350 %{
351 predicate(UseG1GC && n->as_LoadStore()->barrier_data() != 0);
352 match(Set res (CompareAndExchangeN mem (Binary oldval newval)));
353 effect(TEMP res, TEMP tmp1, TEMP tmp2, TEMP tmp3, KILL cr);
354 ins_cost(2 * VOLATILE_REF_COST);
355 format %{ "cmpxchg $res = $mem, $oldval, $newval\t# narrow oop" %}
356 ins_encode %{
357 guarantee($mem$$disp == 0, "impossible encoding");
358 assert_different_registers($oldval$$Register, $mem$$Register);
359 assert_different_registers($newval$$Register, $mem$$Register);
360 write_barrier_pre(masm, this,
361 $mem$$Register /* obj */,
362 $tmp1$$Register /* pre_val */,
363 $tmp2$$Register /* tmp1 */,
364 $tmp3$$Register /* tmp2 */,
365 RegSet::of($mem$$Register, $oldval$$Register, $newval$$Register) /* preserve */,
366 RegSet::of($res$$Register) /* no_preserve */);
367 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register, Assembler::uint32,
368 /*acquire*/ Assembler::relaxed, /*release*/ Assembler::rl, $res$$Register);
369 __ decode_heap_oop($tmp1$$Register, $newval$$Register);
370 write_barrier_post(masm, this,
371 $mem$$Register /* store_addr */,
372 $tmp1$$Register /* new_val */,
373 $tmp2$$Register /* tmp1 */,
374 $tmp3$$Register /* tmp2 */);
375 %}
376 ins_pipe(pipe_slow);
377 %}
378
379 instruct g1CompareAndExchangeNAcq(iRegNNoSp res, indirect mem, iRegN oldval, iRegN newval, iRegPNoSp tmp1, iRegPNoSp tmp2, iRegPNoSp tmp3, rFlagsReg cr)
380 %{
381 predicate(UseG1GC && needs_acquiring_load_reserved(n) && n->as_LoadStore()->barrier_data() != 0);
382 match(Set res (CompareAndExchangeN mem (Binary oldval newval)));
383 effect(TEMP res, TEMP tmp1, TEMP tmp2, TEMP tmp3, KILL cr);
384 ins_cost(VOLATILE_REF_COST);
385 format %{ "cmpxchg_acq $res = $mem, $oldval, $newval\t# narrow oop" %}
386 ins_encode %{
387 guarantee($mem$$disp == 0, "impossible encoding");
388 assert_different_registers($oldval$$Register, $mem$$Register);
389 assert_different_registers($newval$$Register, $mem$$Register);
390 write_barrier_pre(masm, this,
391 $mem$$Register /* obj */,
392 $tmp1$$Register /* pre_val */,
393 $tmp2$$Register /* tmp1 */,
394 $tmp3$$Register /* tmp2 */,
395 RegSet::of($mem$$Register, $oldval$$Register, $newval$$Register) /* preserve */,
396 RegSet::of($res$$Register) /* no_preserve */);
397 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register, Assembler::uint32,
398 /*acquire*/ Assembler::aq, /*release*/ Assembler::rl, $res$$Register);
399 __ decode_heap_oop($tmp1$$Register, $newval$$Register);
400 write_barrier_post(masm, this,
401 $mem$$Register /* store_addr */,
402 $tmp1$$Register /* new_val */,
403 $tmp2$$Register /* tmp1 */,
404 $tmp3$$Register /* tmp2 */);
405 %}
406 ins_pipe(pipe_slow);
407 %}
408
409 instruct g1CompareAndSwapP(iRegINoSp res, indirect mem, iRegP newval, iRegPNoSp tmp1, iRegPNoSp tmp2, iRegP oldval, rFlagsReg cr)
410 %{
411 predicate(UseG1GC && n->as_LoadStore()->barrier_data() != 0);
412 match(Set res (CompareAndSwapP mem (Binary oldval newval)));
413 match(Set res (WeakCompareAndSwapP mem (Binary oldval newval)));
414 effect(TEMP res, TEMP tmp1, TEMP tmp2, KILL cr);
415 ins_cost(2 * VOLATILE_REF_COST);
416 format %{ "cmpxchg $mem, $oldval, $newval\t# (ptr)\n\t"
417 "mv $res, $res == $oldval" %}
418 ins_encode %{
419 guarantee($mem$$disp == 0, "impossible encoding");
420 assert_different_registers($oldval$$Register, $mem$$Register);
421 assert_different_registers($newval$$Register, $mem$$Register);
422 write_barrier_pre(masm, this,
423 noreg /* obj */,
424 $oldval$$Register /* pre_val */,
425 $tmp1$$Register /* tmp1 */,
426 $tmp2$$Register /* tmp2 */,
427 RegSet::of($mem$$Register, $oldval$$Register, $newval$$Register) /* preserve */,
428 RegSet::of($res$$Register) /* no_preserve */);
429 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register, Assembler::int64,
430 /*acquire*/ Assembler::relaxed, /*release*/ Assembler::rl, $res$$Register,
431 /*result as bool*/ true);
432 write_barrier_post(masm, this,
433 $mem$$Register /* store_addr */,
434 $newval$$Register /* new_val */,
435 $tmp1$$Register /* tmp1 */,
436 $tmp2$$Register /* tmp2 */);
437 %}
438 ins_pipe(pipe_slow);
439 %}
440
441 instruct g1CompareAndSwapPAcq(iRegINoSp res, indirect mem, iRegP newval, iRegPNoSp tmp1, iRegPNoSp tmp2, iRegP oldval, rFlagsReg cr)
442 %{
443 predicate(UseG1GC && needs_acquiring_load_reserved(n) && n->as_LoadStore()->barrier_data() != 0);
444 match(Set res (CompareAndSwapP mem (Binary oldval newval)));
445 match(Set res (WeakCompareAndSwapP mem (Binary oldval newval)));
446 effect(TEMP res, TEMP tmp1, TEMP tmp2, KILL cr);
447 ins_cost(VOLATILE_REF_COST);
448 format %{ "cmpxchg_acq $mem, $oldval, $newval\t# (ptr)\n\t"
449 "mv $res, $res == $oldval" %}
450 ins_encode %{
451 guarantee($mem$$disp == 0, "impossible encoding");
452 assert_different_registers($oldval$$Register, $mem$$Register);
453 assert_different_registers($newval$$Register, $mem$$Register);
454 write_barrier_pre(masm, this,
455 noreg /* obj */,
456 $oldval$$Register /* pre_val */,
457 $tmp1$$Register /* tmp1 */,
458 $tmp2$$Register /* tmp2 */,
459 RegSet::of($mem$$Register, $oldval$$Register, $newval$$Register) /* preserve */,
460 RegSet::of($res$$Register) /* no_preserve */);
461 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register, Assembler::int64,
462 /*acquire*/ Assembler::aq, /*release*/ Assembler::rl, $res$$Register,
463 /*result as bool*/ true);
464 write_barrier_post(masm, this,
465 $mem$$Register /* store_addr */,
466 $newval$$Register /* new_val */,
467 $tmp1$$Register /* tmp1 */,
468 $tmp2$$Register /* tmp2 */);
469 %}
470 ins_pipe(pipe_slow);
471 %}
472
473 instruct g1CompareAndSwapN(iRegINoSp res, indirect mem, iRegN newval, iRegPNoSp tmp1, iRegPNoSp tmp2, iRegPNoSp tmp3, iRegN oldval, rFlagsReg cr)
474 %{
475 predicate(UseG1GC && n->as_LoadStore()->barrier_data() != 0);
476 match(Set res (CompareAndSwapN mem (Binary oldval newval)));
477 match(Set res (WeakCompareAndSwapN mem (Binary oldval newval)));
478 effect(TEMP res, TEMP tmp1, TEMP tmp2, TEMP tmp3, KILL cr);
479 ins_cost(2 * VOLATILE_REF_COST);
480 format %{ "cmpxchg $mem, $oldval, $newval\t# (narrow oop)\n\t"
481 "mv $res, $res == $oldval" %}
482 ins_encode %{
483 guarantee($mem$$disp == 0, "impossible encoding");
484 assert_different_registers($oldval$$Register, $mem$$Register);
485 assert_different_registers($newval$$Register, $mem$$Register);
486 write_barrier_pre(masm, this,
487 $mem$$Register /* obj */,
488 $tmp1$$Register /* pre_val */,
489 $tmp2$$Register /* tmp1 */,
490 $tmp3$$Register /* tmp2 */,
491 RegSet::of($mem$$Register, $oldval$$Register, $newval$$Register) /* preserve */,
492 RegSet::of($res$$Register) /* no_preserve */);
493 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register, Assembler::uint32,
494 /*acquire*/ Assembler::relaxed, /*release*/ Assembler::rl, $res$$Register,
495 /*result as bool*/ true);
496 __ decode_heap_oop($tmp1$$Register, $newval$$Register);
497 write_barrier_post(masm, this,
498 $mem$$Register /* store_addr */,
499 $tmp1$$Register /* new_val */,
500 $tmp2$$Register /* tmp1 */,
501 $tmp3$$Register /* tmp2 */);
502 %}
503 ins_pipe(pipe_slow);
504 %}
505
506 instruct g1CompareAndSwapNAcq(iRegINoSp res, indirect mem, iRegN newval, iRegPNoSp tmp1, iRegPNoSp tmp2, iRegPNoSp tmp3, iRegN oldval, rFlagsReg cr)
507 %{
508 predicate(UseG1GC && needs_acquiring_load_reserved(n) && n->as_LoadStore()->barrier_data() != 0);
509 match(Set res (CompareAndSwapN mem (Binary oldval newval)));
510 match(Set res (WeakCompareAndSwapN mem (Binary oldval newval)));
511 effect(TEMP res, TEMP tmp1, TEMP tmp2, TEMP tmp3, KILL cr);
512 ins_cost(VOLATILE_REF_COST);
513 format %{ "cmpxchg_acq $mem, $oldval, $newval\t# (narrow oop)\n\t"
514 "mv $res, $res == $oldval" %}
515 ins_encode %{
516 guarantee($mem$$disp == 0, "impossible encoding");
517 assert_different_registers($oldval$$Register, $mem$$Register);
518 assert_different_registers($newval$$Register, $mem$$Register);
519 write_barrier_pre(masm, this,
520 $mem$$Register /* obj */,
521 $tmp1$$Register /* pre_val */,
522 $tmp2$$Register /* tmp1 */,
523 $tmp3$$Register /* tmp2 */,
524 RegSet::of($mem$$Register, $oldval$$Register, $newval$$Register) /* preserve */,
525 RegSet::of($res$$Register) /* no_preserve */);
526 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register, Assembler::uint32,
527 /*acquire*/ Assembler::aq, /*release*/ Assembler::rl, $res$$Register,
528 /*result as bool*/ true);
529 __ decode_heap_oop($tmp1$$Register, $newval$$Register);
530 write_barrier_post(masm, this,
531 $mem$$Register /* store_addr */,
532 $tmp1$$Register /* new_val */,
533 $tmp2$$Register /* tmp1 */,
534 $tmp3$$Register /* tmp2 */);
535 %}
536 ins_pipe(pipe_slow);
537 %}
538
539 instruct g1GetAndSetP(indirect mem, iRegP newval, iRegPNoSp tmp1, iRegPNoSp tmp2, iRegPNoSp preval, rFlagsReg cr)
540 %{
541 predicate(UseG1GC && n->as_LoadStore()->barrier_data() != 0);
542 match(Set preval (GetAndSetP mem newval));
543 effect(TEMP preval, TEMP tmp1, TEMP tmp2, KILL cr);
544 ins_cost(2 * VOLATILE_REF_COST);
545 format %{ "atomic_xchg $preval, $newval, [$mem]" %}
546 ins_encode %{
547 guarantee($mem$$disp == 0, "impossible encoding");
548 assert_different_registers($mem$$Register, $newval$$Register);
549 write_barrier_pre(masm, this,
550 $mem$$Register /* obj */,
551 $preval$$Register /* pre_val (as a temporary register) */,
552 $tmp1$$Register /* tmp1 */,
553 $tmp2$$Register /* tmp2 */,
554 RegSet::of($mem$$Register, $preval$$Register, $newval$$Register) /* preserve */);
555 __ atomic_xchg($preval$$Register, $newval$$Register, $mem$$Register);
556 write_barrier_post(masm, this,
557 $mem$$Register /* store_addr */,
558 $newval$$Register /* new_val */,
559 $tmp1$$Register /* tmp1 */,
560 $tmp2$$Register /* tmp2 */);
561 %}
562 ins_pipe(pipe_serial);
563 %}
564
565 instruct g1GetAndSetPAcq(indirect mem, iRegP newval, iRegPNoSp tmp1, iRegPNoSp tmp2, iRegPNoSp preval, rFlagsReg cr)
566 %{
567 predicate(UseG1GC && needs_acquiring_load_reserved(n) && n->as_LoadStore()->barrier_data() != 0);
568 match(Set preval (GetAndSetP mem newval));
569 effect(TEMP preval, TEMP tmp1, TEMP tmp2, KILL cr);
570 ins_cost(VOLATILE_REF_COST);
571 format %{ "atomic_xchg_acq $preval, $newval, [$mem]" %}
572 ins_encode %{
573 guarantee($mem$$disp == 0, "impossible encoding");
574 assert_different_registers($mem$$Register, $newval$$Register);
575 write_barrier_pre(masm, this,
576 $mem$$Register /* obj */,
577 $preval$$Register /* pre_val (as a temporary register) */,
578 $tmp1$$Register /* tmp1 */,
579 $tmp2$$Register /* tmp2 */,
580 RegSet::of($mem$$Register, $preval$$Register, $newval$$Register) /* preserve */);
581 __ atomic_xchgal($preval$$Register, $newval$$Register, $mem$$Register);
582 write_barrier_post(masm, this,
583 $mem$$Register /* store_addr */,
584 $newval$$Register /* new_val */,
585 $tmp1$$Register /* tmp1 */,
586 $tmp2$$Register /* tmp2 */);
587 %}
588 ins_pipe(pipe_serial);
589 %}
590
591 instruct g1GetAndSetN(indirect mem, iRegN newval, iRegPNoSp tmp1, iRegPNoSp tmp2, iRegPNoSp tmp3, iRegNNoSp preval, rFlagsReg cr)
592 %{
593 predicate(UseG1GC && n->as_LoadStore()->barrier_data() != 0);
594 match(Set preval (GetAndSetN mem newval));
595 effect(TEMP preval, TEMP tmp1, TEMP tmp2, TEMP tmp3, KILL cr);
596 ins_cost(2 * VOLATILE_REF_COST);
597 format %{ "atomic_xchgwu $preval, $newval, [$mem]" %}
598 ins_encode %{
599 guarantee($mem$$disp == 0, "impossible encoding");
600 assert_different_registers($mem$$Register, $newval$$Register);
601 write_barrier_pre(masm, this,
602 $mem$$Register /* obj */,
603 $tmp1$$Register /* pre_val */,
604 $tmp2$$Register /* tmp1 */,
605 $tmp3$$Register /* tmp2 */,
606 RegSet::of($mem$$Register, $preval$$Register, $newval$$Register) /* preserve */);
607 __ atomic_xchgwu($preval$$Register, $newval$$Register, $mem$$Register);
608 __ decode_heap_oop($tmp1$$Register, $newval$$Register);
609 write_barrier_post(masm, this,
610 $mem$$Register /* store_addr */,
611 $tmp1$$Register /* new_val */,
612 $tmp2$$Register /* tmp1 */,
613 $tmp3$$Register /* tmp2 */);
614 %}
615 ins_pipe(pipe_serial);
616 %}
617
618 instruct g1GetAndSetNAcq(indirect mem, iRegN newval, iRegPNoSp tmp1, iRegPNoSp tmp2, iRegPNoSp tmp3, iRegNNoSp preval, rFlagsReg cr)
619 %{
620 predicate(UseG1GC && needs_acquiring_load_reserved(n) && n->as_LoadStore()->barrier_data() != 0);
621 match(Set preval (GetAndSetN mem newval));
622 effect(TEMP preval, TEMP tmp1, TEMP tmp2, TEMP tmp3, KILL cr);
623 ins_cost(VOLATILE_REF_COST);
624 format %{ "atomic_xchgwu_acq $preval, $newval, [$mem]" %}
625 ins_encode %{
626 guarantee($mem$$disp == 0, "impossible encoding");
627 assert_different_registers($mem$$Register, $newval$$Register);
628 write_barrier_pre(masm, this,
629 $mem$$Register /* obj */,
630 $tmp1$$Register /* pre_val */,
631 $tmp2$$Register /* tmp1 */,
632 $tmp3$$Register /* tmp2 */,
633 RegSet::of($mem$$Register, $preval$$Register, $newval$$Register) /* preserve */);
634 __ atomic_xchgalwu($preval$$Register, $newval$$Register, $mem$$Register);
635 __ decode_heap_oop($tmp1$$Register, $newval$$Register);
636 write_barrier_post(masm, this,
637 $mem$$Register /* store_addr */,
638 $tmp1$$Register /* new_val */,
639 $tmp2$$Register /* tmp1 */,
640 $tmp3$$Register /* tmp2 */);
641 %}
642 ins_pipe(pipe_serial);
643 %}
644
645 instruct g1LoadP(iRegPNoSp dst, indirect mem, iRegPNoSp tmp1, iRegPNoSp tmp2, rFlagsReg cr)
646 %{
647 predicate(UseG1GC && n->as_Load()->barrier_data() != 0);
648 match(Set dst (LoadP mem));
649 effect(TEMP dst, TEMP tmp1, TEMP tmp2, KILL cr);
650 ins_cost(LOAD_COST + BRANCH_COST);
651 format %{ "ld $dst, $mem\t# ptr" %}
652 ins_encode %{
653 guarantee($mem$$disp == 0, "impossible encoding");
654 __ ld($dst$$Register, Address($mem$$Register));
655 write_barrier_pre(masm, this,
656 noreg /* obj */,
657 $dst$$Register /* pre_val */,
658 $tmp1$$Register /* tmp1 */,
659 $tmp2$$Register /* tmp2 */);
660 %}
661 ins_pipe(iload_reg_mem);
662 %}
663
664 instruct g1LoadN(iRegNNoSp dst, indirect mem, iRegPNoSp tmp1, iRegPNoSp tmp2, iRegPNoSp tmp3, rFlagsReg cr)
665 %{
666 predicate(UseG1GC && n->as_Load()->barrier_data() != 0);
667 match(Set dst (LoadN mem));
668 effect(TEMP dst, TEMP tmp1, TEMP tmp2, TEMP tmp3, KILL cr);
669 ins_cost(LOAD_COST + BRANCH_COST);
670 format %{ "lwu $dst, $mem\t# compressed ptr" %}
671 ins_encode %{
672 guarantee($mem$$disp == 0, "impossible encoding");
673 __ lwu($dst$$Register, Address($mem$$Register));
674 if ((barrier_data() & G1C2BarrierPre) != 0) {
675 __ decode_heap_oop($tmp1$$Register, $dst$$Register);
676 write_barrier_pre(masm, this,
677 noreg /* obj */,
678 $tmp1$$Register /* pre_val */,
679 $tmp2$$Register /* tmp1 */,
680 $tmp3$$Register /* tmp2 */);
681 }
682 %}
683 ins_pipe(iload_reg_mem);
684 %}