1 /*
2 * Copyright (c) 2026, Oracle and/or its affiliates. All rights reserved.
3 * Copyright (c) 2018, 2020, Red Hat, Inc. All rights reserved.
4 * Copyright (c) 2020, 2021, Huawei Technologies Co., Ltd. All rights reserved.
5 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
6 *
7 * This code is free software; you can redistribute it and/or modify it
8 * under the terms of the GNU General Public License version 2 only, as
9 * published by the Free Software Foundation.
10 *
11 * This code is distributed in the hope that it will be useful, but WITHOUT
12 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
13 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
14 * version 2 for more details (a copy is included in the LICENSE file that
15 * accompanied this code).
16 *
17 * You should have received a copy of the GNU General Public License version
18 * 2 along with this work; if not, write to the Free Software Foundation,
19 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
20 *
21 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
22 * or visit www.oracle.com if you need additional information or have any
23 * questions.
24 *
25 */
26
27 #include "gc/shenandoah/heuristics/shenandoahHeuristics.hpp"
28 #include "gc/shenandoah/mode/shenandoahMode.hpp"
29 #include "gc/shenandoah/shenandoahBarrierSet.hpp"
30 #include "gc/shenandoah/shenandoahBarrierSetAssembler.hpp"
31 #include "gc/shenandoah/shenandoahHeap.inline.hpp"
32 #include "gc/shenandoah/shenandoahHeapRegion.hpp"
33 #include "gc/shenandoah/shenandoahNMethod.inline.hpp"
34 #include "gc/shenandoah/shenandoahRuntime.hpp"
35 #include "gc/shenandoah/shenandoahThreadLocalData.hpp"
36 #include "interpreter/interp_masm.hpp"
37 #include "interpreter/interpreter.hpp"
38 #include "nativeInst_riscv.hpp"
39 #include "runtime/javaThread.hpp"
40 #include "runtime/sharedRuntime.hpp"
41 #ifdef COMPILER1
42 #include "c1/c1_LIRAssembler.hpp"
43 #include "c1/c1_MacroAssembler.hpp"
44 #include "gc/shenandoah/c1/shenandoahBarrierSetC1.hpp"
45 #endif
46 #ifdef COMPILER2
47 #include "gc/shenandoah/c2/shenandoahBarrierSetC2.hpp"
48 #include "opto/output.hpp"
49 #endif
50
51 #define __ masm->
52
53 void ShenandoahBarrierSetAssembler::arraycopy_prologue(MacroAssembler* masm, DecoratorSet decorators, bool is_oop,
54 Register src, Register dst, Register count, RegSet saved_regs) {
55 if (is_oop) {
56 bool dest_uninitialized = (decorators & IS_DEST_UNINITIALIZED) != 0;
57 if ((ShenandoahSATBBarrier && !dest_uninitialized) || ShenandoahLoadRefBarrier) {
58
59 Label done;
60
61 // Avoid calling runtime if count == 0
62 __ beqz(count, done);
63
64 // Is GC active?
65 Address gc_state(xthread, in_bytes(ShenandoahThreadLocalData::gc_state_offset()));
66 assert_different_registers(src, dst, count, t0);
67
68 assert(!saved_regs.contains(t0), "Sanity: about to clobber t0");
69
70 __ lbu(t0, gc_state);
71 if (ShenandoahSATBBarrier && dest_uninitialized) {
72 __ test_bit(t0, t0, ShenandoahHeap::HAS_FORWARDED_BITPOS);
73 __ beqz(t0, done);
74 } else {
75 __ andi(t0, t0, ShenandoahHeap::HAS_FORWARDED | ShenandoahHeap::MARKING);
76 __ beqz(t0, done);
77 }
78
79 __ push_call_clobbered_registers();
80 // If arguments are not in proper places, shuffle them.
81 // Doing this via the stack is the most straight-forward way to avoid
82 // accidentally smashing any register.
83 if (c_rarg0 != src || c_rarg1 != dst || c_rarg2 != count) {
84 __ push_reg(RegSet::of(src), sp);
85 __ push_reg(RegSet::of(dst), sp);
86 __ push_reg(RegSet::of(count), sp);
87 __ pop_reg(RegSet::of(c_rarg2), sp);
88 __ pop_reg(RegSet::of(c_rarg1), sp);
89 __ pop_reg(RegSet::of(c_rarg0), sp);
90 }
91 address target = nullptr;
92 if (UseCompressedOops) {
93 target = CAST_FROM_FN_PTR(address, ShenandoahRuntime::arraycopy_barrier_narrow_oop);
94 } else {
95 target = CAST_FROM_FN_PTR(address, ShenandoahRuntime::arraycopy_barrier_oop);
96 }
97 __ call_VM_leaf(target, 3);
98 __ pop_call_clobbered_registers();
99 __ bind(done);
100 }
101 }
102 }
103
104 void ShenandoahBarrierSetAssembler::arraycopy_epilogue(MacroAssembler* masm, DecoratorSet decorators, bool is_oop,
105 Register start, Register count, Register tmp) {
106 if (ShenandoahCardBarrier && is_oop) {
107 gen_write_ref_array_post_barrier(masm, decorators, start, count, tmp);
108 }
109 }
110
111 void ShenandoahBarrierSetAssembler::satb_barrier(MacroAssembler* masm,
112 Register obj,
113 Register pre_val,
114 Register thread,
115 Register tmp1,
116 Register tmp2) {
117 assert(ShenandoahSATBBarrier, "Should be checked by caller");
118 assert(thread == xthread, "must be");
119
120 Label done;
121 Label runtime;
122
123 assert_different_registers(obj, pre_val, tmp1, tmp2);
124 assert(pre_val != noreg && tmp1 != noreg && tmp2 != noreg, "expecting a register");
125
126 Address index(thread, in_bytes(ShenandoahThreadLocalData::satb_mark_queue_index_offset()));
127 Address buffer(thread, in_bytes(ShenandoahThreadLocalData::satb_mark_queue_buffer_offset()));
128
129 // Is marking active?
130 Address gc_state(xthread, in_bytes(ShenandoahThreadLocalData::gc_state_offset()));
131 __ lbu(t1, gc_state);
132 __ test_bit(t1, t1, ShenandoahHeap::MARKING_BITPOS);
133 __ beqz(t1, done);
134
135 // Do we need to load the previous value?
136 if (obj != noreg) {
137 if (UseCompressedOops) {
138 __ lwu(pre_val, Address(obj, 0));
139 __ decode_heap_oop(pre_val);
140 } else {
141 __ ld(pre_val, Address(obj, 0));
142 }
143 }
144
145 // Is the previous value null?
146 __ beqz(pre_val, done);
147
148 // Can we store original value in the thread's buffer?
149 // Is index == 0?
150 // (The index field is typed as size_t.)
151 __ ld(tmp1, index); // tmp := *index_adr
152 __ beqz(tmp1, runtime); // tmp == 0? If yes, goto runtime
153
154 __ subi(tmp1, tmp1, wordSize); // tmp := tmp - wordSize
155 __ sd(tmp1, index); // *index_adr := tmp
156 __ ld(tmp2, buffer);
157 __ add(tmp1, tmp1, tmp2); // tmp := tmp + *buffer_adr
158
159 // Record the previous value
160 __ sd(pre_val, Address(tmp1, 0));
161 __ j(done);
162
163 // Slow-path call.
164 __ bind(runtime);
165 __ enter();
166 __ push_call_clobbered_registers();
167 if (c_rarg0 != pre_val) {
168 __ mv(c_rarg0, pre_val);
169 }
170 // Calling with super_call_VM_leaf with c_rarg0 bypasses interpreter checks and avoids any moves.
171 __ super_call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::write_barrier_pre), c_rarg0);
172 __ pop_call_clobbered_registers();
173 __ leave();
174
175 __ bind(done);
176 }
177
178 void ShenandoahBarrierSetAssembler::load_reference_barrier(MacroAssembler* masm,
179 Register dst,
180 Address load_addr,
181 DecoratorSet decorators) {
182 assert(ShenandoahLoadRefBarrier, "Should be enabled");
183 assert(dst != t1 && load_addr.base() != t1, "need t1");
184 assert_different_registers(load_addr.base(), t0, t1);
185
186 bool is_strong = ShenandoahBarrierSet::is_strong_access(decorators);
187 bool is_weak = ShenandoahBarrierSet::is_weak_access(decorators);
188 bool is_phantom = ShenandoahBarrierSet::is_phantom_access(decorators);
189 bool is_native = ShenandoahBarrierSet::is_native_access(decorators);
190 bool is_narrow = UseCompressedOops && !is_native;
191
192 Label heap_stable, not_cset;
193 Address gc_state(xthread, in_bytes(ShenandoahThreadLocalData::gc_state_offset()));
194 __ lbu(t1, gc_state);
195
196 // Check for heap stability
197 if (is_strong) {
198 __ test_bit(t1, t1, ShenandoahHeap::HAS_FORWARDED_BITPOS);
199 __ beqz(t1, heap_stable);
200 } else {
201 Label lrb;
202 __ test_bit(t0, t1, ShenandoahHeap::WEAK_ROOTS_BITPOS);
203 __ bnez(t0, lrb);
204 __ test_bit(t0, t1, ShenandoahHeap::HAS_FORWARDED_BITPOS);
205 __ beqz(t0, heap_stable);
206 __ bind(lrb);
207 }
208
209 // use x11 for load address
210 Register result_dst = dst;
211 if (dst == x11) {
212 __ mv(t1, dst);
213 dst = t1;
214 }
215
216 // Save x10 and x11, unless it is an output register
217 RegSet saved_regs = RegSet::of(x10, x11) - result_dst;
218 __ push_reg(saved_regs, sp);
219 __ la(x11, load_addr);
220 __ mv(x10, dst);
221
222 // Test for in-cset
223 if (is_strong) {
224 __ mv(t1, ShenandoahHeap::in_cset_fast_test_addr());
225 __ srli(t0, x10, ShenandoahHeapRegion::region_size_bytes_shift_jint());
226 __ add(t1, t1, t0);
227 __ lbu(t1, Address(t1));
228 __ test_bit(t0, t1, 0);
229 __ beqz(t0, not_cset);
230 }
231
232 // Slow-path call
233 __ enter();
234 __ push_call_clobbered_registers();
235 address target = nullptr;
236 if (is_strong) {
237 if (is_narrow) {
238 target = CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_strong_narrow);
239 } else {
240 target = CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_strong);
241 }
242 } else if (is_weak) {
243 if (is_narrow) {
244 target = CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_weak_narrow);
245 } else {
246 target = CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_weak);
247 }
248 } else {
249 assert(is_phantom, "only remaining strength");
250 assert(!is_narrow, "phantom access cannot be narrow");
251 target = CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_phantom);
252 }
253 // Calling with super_call_VM_leaf with c_rarg0/1 bypasses interpreter checks and avoids any moves.
254 __ super_call_VM_leaf(target, c_rarg0, c_rarg1);
255 __ mv(t0, x10);
256 __ pop_call_clobbered_registers();
257 __ mv(x10, t0);
258 __ leave();
259
260 __ bind(not_cset);
261 __ mv(result_dst, x10);
262 __ pop_reg(saved_regs, sp);
263
264 __ bind(heap_stable);
265 }
266
267 //
268 // Arguments:
269 //
270 // Inputs:
271 // src: oop location to load from, might be clobbered
272 //
273 // Output:
274 // dst: oop loaded from src location
275 //
276 // Kill:
277 // x30 (tmp reg)
278 //
279 // Alias:
280 // dst: x30 (might use x30 as temporary output register to avoid clobbering src)
281 //
282 void ShenandoahBarrierSetAssembler::load_at(MacroAssembler* masm,
283 DecoratorSet decorators,
284 BasicType type,
285 Register dst,
286 Address src,
287 Register tmp1,
288 Register tmp2) {
289 // 1: non-reference load, no additional barrier is needed
290 if (!is_reference_type(type)) {
291 BarrierSetAssembler::load_at(masm, decorators, type, dst, src, tmp1, tmp2);
292 return;
293 }
294
295 // 2: load a reference from src location and apply LRB if needed
296 if (ShenandoahBarrierSet::need_load_reference_barrier(decorators, type)) {
297 Register result_dst = dst;
298
299 // Preserve src location for LRB
300 RegSet saved_regs;
301 if (dst == src.base()) {
302 dst = (src.base() == x28) ? x29 : x28;
303 saved_regs = RegSet::of(dst);
304 __ push_reg(saved_regs, sp);
305 }
306 assert_different_registers(dst, src.base());
307
308 BarrierSetAssembler::load_at(masm, decorators, type, dst, src, tmp1, tmp2);
309
310 load_reference_barrier(masm, dst, src, decorators);
311
312 if (dst != result_dst) {
313 __ mv(result_dst, dst);
314 dst = result_dst;
315 }
316
317 if (saved_regs.bits() != 0) {
318 __ pop_reg(saved_regs, sp);
319 }
320 } else {
321 BarrierSetAssembler::load_at(masm, decorators, type, dst, src, tmp1, tmp2);
322 }
323
324 // 3: apply keep-alive barrier if needed
325 if (ShenandoahBarrierSet::need_keep_alive_barrier(decorators, type)) {
326 satb_barrier(masm /* masm */,
327 noreg /* obj */,
328 dst /* pre_val */,
329 xthread /* thread */,
330 tmp1 /* tmp1 */,
331 tmp2 /* tmp2 */);
332 }
333 }
334
335 void ShenandoahBarrierSetAssembler::card_barrier(MacroAssembler* masm, Register obj) {
336 assert(ShenandoahCardBarrier, "Should have been checked by caller");
337
338 __ srli(obj, obj, CardTable::card_shift());
339
340 assert(CardTable::dirty_card_val() == 0, "must be");
341
342 Address curr_ct_holder_addr(xthread, in_bytes(ShenandoahThreadLocalData::card_table_offset()));
343 __ ld(t1, curr_ct_holder_addr);
344 __ add(t1, obj, t1);
345
346 if (UseCondCardMark) {
347 Label L_already_dirty;
348 __ lbu(t0, Address(t1));
349 __ beqz(t0, L_already_dirty);
350 __ sb(zr, Address(t1));
351 __ bind(L_already_dirty);
352 } else {
353 __ sb(zr, Address(t1));
354 }
355 }
356
357 void ShenandoahBarrierSetAssembler::store_at(MacroAssembler* masm, DecoratorSet decorators, BasicType type,
358 Address dst, Register val, Register tmp1, Register tmp2, Register tmp3) {
359 // 1: non-reference types require no barriers
360 if (!is_reference_type(type)) {
361 BarrierSetAssembler::store_at(masm, decorators, type, dst, val, tmp1, tmp2, tmp3);
362 return;
363 }
364
365 // Flatten object address right away for simplicity: likely needed by barriers
366 if (dst.offset() == 0) {
367 if (dst.base() != tmp3) {
368 __ mv(tmp3, dst.base());
369 }
370 } else {
371 __ la(tmp3, dst);
372 }
373
374 // 2: pre-barrier: SATB needs the previous value
375 if (ShenandoahBarrierSet::need_satb_barrier(decorators, type)) {
376 satb_barrier(masm,
377 tmp3 /* obj */,
378 tmp2 /* pre_val */,
379 xthread /* thread */,
380 tmp1 /* tmp */,
381 t0 /* tmp2 */);
382 }
383
384 // Store!
385 BarrierSetAssembler::store_at(masm, decorators, type, Address(tmp3, 0), val, noreg, noreg, noreg);
386
387 // 3: post-barrier: card barrier needs store address
388 bool storing_non_null = (val != noreg);
389 if (ShenandoahBarrierSet::need_card_barrier(decorators, type) && storing_non_null) {
390 card_barrier(masm, tmp3);
391 }
392 }
393
394 void ShenandoahBarrierSetAssembler::try_resolve_jobject_in_native(MacroAssembler* masm, Register jni_env,
395 Register obj, Register tmp, Label& slowpath) {
396 Label done;
397 // Resolve jobject
398 BarrierSetAssembler::try_resolve_jobject_in_native(masm, jni_env, obj, tmp, slowpath);
399
400 // Check for null.
401 __ beqz(obj, done);
402
403 assert(obj != t1, "need t1");
404 Address gc_state(jni_env, ShenandoahThreadLocalData::gc_state_offset() - JavaThread::jni_environment_offset());
405 __ lbu(t1, gc_state);
406
407 // Check for heap in evacuation phase
408 __ test_bit(t0, t1, ShenandoahHeap::EVACUATION_BITPOS);
409 __ bnez(t0, slowpath);
410
411 __ bind(done);
412 }
413
414 void ShenandoahBarrierSetAssembler::try_peek_weak_handle_in_nmethod(MacroAssembler *masm, Register weak_handle,
415 Register obj, Register tmp, Label& slow_path) {
416 assert_different_registers(weak_handle, tmp, noreg);
417 assert_different_registers(obj, tmp, noreg);
418
419
420 Label done;
421
422 // Peek weak handle using the standard implementation.
423 BarrierSetAssembler::try_peek_weak_handle_in_nmethod(masm, weak_handle, obj, tmp, slow_path);
424
425 // Check if the reference is null, and if it is, take the fast path.
426 __ beqz(obj, done);
427
428 Address gc_state(xthread, ShenandoahThreadLocalData::gc_state_offset());
429 __ lbu(tmp, gc_state);
430
431 // Check if the heap is under weak-reference/roots processing, in
432 // which case we need to take the slow path.
433 __ test_bit(tmp, tmp, ShenandoahHeap::WEAK_ROOTS_BITPOS);
434 __ bnez(tmp, slow_path);
435 __ bind(done);
436 }
437
438 void ShenandoahBarrierSetAssembler::gen_write_ref_array_post_barrier(MacroAssembler* masm, DecoratorSet decorators,
439 Register start, Register count, Register tmp) {
440 assert(ShenandoahCardBarrier, "Did you mean to enable ShenandoahCardBarrier?");
441
442 Label L_loop, L_done;
443 const Register end = count;
444
445 // Zero count? Nothing to do.
446 __ beqz(count, L_done);
447
448 // end = start + count << LogBytesPerHeapOop
449 // last element address to make inclusive
450 __ shadd(end, count, start, tmp, LogBytesPerHeapOop);
451 __ subi(end, end, BytesPerHeapOop);
452 __ srli(start, start, CardTable::card_shift());
453 __ srli(end, end, CardTable::card_shift());
454
455 // number of bytes to copy
456 __ sub(count, end, start);
457
458 Address curr_ct_holder_addr(xthread, in_bytes(ShenandoahThreadLocalData::card_table_offset()));
459 __ ld(tmp, curr_ct_holder_addr);
460 __ add(start, start, tmp);
461
462 __ bind(L_loop);
463 __ add(tmp, start, count);
464 __ sb(zr, Address(tmp));
465 __ subi(count, count, 1);
466 __ bgez(count, L_loop);
467 __ bind(L_done);
468 }
469
470 #undef __
471
472 #ifdef COMPILER1
473
474 #define __ ce->masm()->
475
476 void ShenandoahBarrierSetAssembler::gen_pre_barrier_stub(LIR_Assembler* ce, ShenandoahPreBarrierStub* stub) {
477 ShenandoahBarrierSetC1* bs = (ShenandoahBarrierSetC1*)BarrierSet::barrier_set()->barrier_set_c1();
478 // At this point we know that marking is in progress.
479 // If do_load() is true then we have to emit the
480 // load of the previous value; otherwise it has already
481 // been loaded into _pre_val.
482 __ bind(*stub->entry());
483
484 assert(stub->pre_val()->is_register(), "Precondition.");
485
486 Register pre_val_reg = stub->pre_val()->as_register();
487
488 if (stub->do_load()) {
489 ce->mem2reg(stub->addr(), stub->pre_val(), T_OBJECT, stub->patch_code(), stub->info(), false /* wide */);
490 }
491 __ beqz(pre_val_reg, *stub->continuation(), /* is_far */ true);
492 ce->store_parameter(stub->pre_val()->as_register(), 0);
493 __ far_call(RuntimeAddress(bs->pre_barrier_c1_runtime_code_blob()->code_begin()));
494 __ j(*stub->continuation());
495 }
496
497 void ShenandoahBarrierSetAssembler::gen_load_reference_barrier_stub(LIR_Assembler* ce,
498 ShenandoahLoadReferenceBarrierStub* stub) {
499 ShenandoahBarrierSetC1* bs = (ShenandoahBarrierSetC1*)BarrierSet::barrier_set()->barrier_set_c1();
500 __ bind(*stub->entry());
501
502 DecoratorSet decorators = stub->decorators();
503 bool is_strong = ShenandoahBarrierSet::is_strong_access(decorators);
504 bool is_weak = ShenandoahBarrierSet::is_weak_access(decorators);
505 bool is_phantom = ShenandoahBarrierSet::is_phantom_access(decorators);
506 bool is_native = ShenandoahBarrierSet::is_native_access(decorators);
507
508 Register obj = stub->obj()->as_register();
509 Register res = stub->result()->as_register();
510 Register addr = stub->addr()->as_pointer_register();
511 Register tmp1 = stub->tmp1()->as_register();
512 Register tmp2 = stub->tmp2()->as_register();
513
514 assert(res == x10, "result must arrive in x10");
515 assert_different_registers(tmp1, tmp2, t0);
516
517 if (res != obj) {
518 __ mv(res, obj);
519 }
520
521 if (is_strong) {
522 // Check for object in cset.
523 __ mv(tmp2, ShenandoahHeap::in_cset_fast_test_addr());
524 __ srli(tmp1, res, ShenandoahHeapRegion::region_size_bytes_shift_jint());
525 __ add(tmp2, tmp2, tmp1);
526 __ lbu(tmp2, Address(tmp2));
527 __ beqz(tmp2, *stub->continuation(), true /* is_far */);
528 }
529
530 ce->store_parameter(res, 0);
531 ce->store_parameter(addr, 1);
532
533 if (is_strong) {
534 if (is_native) {
535 __ far_call(RuntimeAddress(bs->load_reference_barrier_strong_native_rt_code_blob()->code_begin()));
536 } else {
537 __ far_call(RuntimeAddress(bs->load_reference_barrier_strong_rt_code_blob()->code_begin()));
538 }
539 } else if (is_weak) {
540 __ far_call(RuntimeAddress(bs->load_reference_barrier_weak_rt_code_blob()->code_begin()));
541 } else {
542 assert(is_phantom, "only remaining strength");
543 __ far_call(RuntimeAddress(bs->load_reference_barrier_phantom_rt_code_blob()->code_begin()));
544 }
545
546 __ j(*stub->continuation());
547 }
548
549 #undef __
550
551 #define __ sasm->
552
553 void ShenandoahBarrierSetAssembler::generate_c1_pre_barrier_runtime_stub(StubAssembler* sasm) {
554 __ prologue("shenandoah_pre_barrier", false);
555
556 // arg0 : previous value of memory
557
558 BarrierSet* bs = BarrierSet::barrier_set();
559
560 const Register pre_val = x10;
561 const Register thread = xthread;
562 const Register tmp = t0;
563
564 Address queue_index(thread, in_bytes(ShenandoahThreadLocalData::satb_mark_queue_index_offset()));
565 Address buffer(thread, in_bytes(ShenandoahThreadLocalData::satb_mark_queue_buffer_offset()));
566
567 Label done;
568 Label runtime;
569
570 // Is marking still active?
571 Address gc_state(thread, in_bytes(ShenandoahThreadLocalData::gc_state_offset()));
572 __ lb(tmp, gc_state);
573 __ test_bit(tmp, tmp, ShenandoahHeap::MARKING_BITPOS);
574 __ beqz(tmp, done);
575
576 // Can we store original value in the thread's buffer?
577 __ ld(tmp, queue_index);
578 __ beqz(tmp, runtime);
579
580 __ subi(tmp, tmp, wordSize);
581 __ sd(tmp, queue_index);
582 __ ld(t1, buffer);
583 __ add(tmp, tmp, t1);
584 __ load_parameter(0, t1);
585 __ sd(t1, Address(tmp, 0));
586 __ j(done);
587
588 __ bind(runtime);
589 __ push_call_clobbered_registers();
590 __ load_parameter(0, pre_val);
591 __ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::write_barrier_pre), pre_val);
592 __ pop_call_clobbered_registers();
593 __ bind(done);
594
595 __ epilogue();
596 }
597
598 void ShenandoahBarrierSetAssembler::generate_c1_load_reference_barrier_runtime_stub(StubAssembler* sasm,
599 DecoratorSet decorators) {
600 __ prologue("shenandoah_load_reference_barrier", false);
601 // arg0 : object to be resolved
602
603 __ push_call_clobbered_registers();
604 __ load_parameter(0, x10);
605 __ load_parameter(1, x11);
606
607 bool is_strong = ShenandoahBarrierSet::is_strong_access(decorators);
608 bool is_weak = ShenandoahBarrierSet::is_weak_access(decorators);
609 bool is_phantom = ShenandoahBarrierSet::is_phantom_access(decorators);
610 bool is_native = ShenandoahBarrierSet::is_native_access(decorators);
611 address target = nullptr;
612 if (is_strong) {
613 if (is_native) {
614 target = CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_strong);
615 } else {
616 if (UseCompressedOops) {
617 target = CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_strong_narrow);
618 } else {
619 target = CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_strong);
620 }
621 }
622 } else if (is_weak) {
623 assert(!is_native, "weak must not be called off-heap");
624 if (UseCompressedOops) {
625 target = CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_weak_narrow);
626 } else {
627 target = CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_weak);
628 }
629 } else {
630 assert(is_phantom, "only remaining strength");
631 assert(is_native, "phantom must only be called off-heap");
632 target = CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_phantom);
633 }
634 __ rt_call(target);
635 __ mv(t0, x10);
636 __ pop_call_clobbered_registers();
637 __ mv(x10, t0);
638
639 __ epilogue();
640 }
641
642 #undef __
643
644 #endif // COMPILER1
645
646 #ifdef COMPILER2
647
648 #undef __
649 #define __ masm->
650
651 void ShenandoahBarrierSetAssembler::load_c2(const MachNode* node, MacroAssembler* masm, Register dst, Address src, Register tmp1, Register tmp2, bool is_narrow) {
652 // Do the actual load. This load is the candidate for implicit null check, and MUST come first.
653 if (is_narrow) {
654 __ lwu(dst, src);
655 } else {
656 __ ld(dst, src);
657 }
658
659 ShenandoahBarrierStubC2::load_post(masm, node, dst, src, tmp1, tmp2, is_narrow);
660 }
661
662 void ShenandoahBarrierSetAssembler::store_c2(const MachNode* node, MacroAssembler* masm, Address dst, bool dst_narrow,
663 Register src, bool src_narrow, Register tmp1, Register tmp2, Register tmp3) {
664
665 ShenandoahBarrierStubC2::store_pre(masm, node, dst, tmp1, tmp2, tmp3, dst_narrow);
666
667 // Do the actual store
668 if (dst_narrow) {
669 if (!src_narrow) {
670 // Need to encode into tmp, because we cannot clobber src.
671 assert(tmp1 != noreg, "need temp register");
672 if ((node->barrier_data() & ShenandoahBitNotNull) == 0) {
673 __ encode_heap_oop(tmp1, src);
674 } else {
675 __ encode_heap_oop_not_null(tmp1, src);
676 }
677 src = tmp1;
678 }
679 __ sw(src, dst);
680 } else {
681 __ sd(src, dst);
682 }
683
684 ShenandoahBarrierStubC2::store_post(masm, node, dst, tmp2, tmp3);
685 }
686
687 void ShenandoahBarrierSetAssembler::compare_and_set_c2(const MachNode* node, MacroAssembler* masm, Register res, Register addr,
688 Register oldval, Register newval, Register tmp1, Register tmp2, Register tmp3, bool exchange, bool narrow, bool is_acquire) {
689 const Assembler::Aqrl acquire = is_acquire ? Assembler::aq : Assembler::relaxed;
690 const Assembler::Aqrl release = Assembler::rl;
691 const Assembler::operand_size size = narrow ? Assembler::uint32 : Assembler::int64;
692
693 ShenandoahBarrierStubC2::load_store_pre(masm, node, Address(addr), tmp1, tmp2, tmp3, narrow);
694
695 // CAS!
696 __ cmpxchg(addr, oldval, newval, size, acquire, release, /* result */ res, !exchange /* result_as_bool */);
697
698 ShenandoahBarrierStubC2::load_store_post(masm, node, Address(addr, 0), tmp2, tmp3);
699 }
700
701 void ShenandoahBarrierSetAssembler::get_and_set_c2(const MachNode* node, MacroAssembler* masm, Register preval,
702 Register newval, Register addr, Register tmp1, Register tmp2, Register tmp3, bool is_acquire) {
703 const bool is_narrow = node->bottom_type()->isa_narrowoop();
704
705 ShenandoahBarrierStubC2::load_store_pre(masm, node, Address(addr, 0), tmp1, tmp2, tmp3, is_narrow);
706
707 if (is_narrow) {
708 if (is_acquire) {
709 __ atomic_xchgalwu(preval, newval, addr);
710 } else {
711 __ atomic_xchgwu(preval, newval, addr);
712 }
713 } else {
714 if (is_acquire) {
715 __ atomic_xchgal(preval, newval, addr);
716 } else {
717 __ atomic_xchg(preval, newval, addr);
718 }
719 }
720
721 ShenandoahBarrierStubC2::load_store_post(masm, node, Address(addr, 0), tmp2, tmp3);
722 }
723
724 #undef __
725 #define __ masm.
726
727 void ShenandoahBarrierStubC2::cardtable(MacroAssembler& masm, Address address, Register tmp1, Register tmp2) {
728 assert(CardTable::dirty_card_val() == 0, "must be");
729 Assembler::InlineSkippedInstructionsCounter skip_counter(&masm);
730
731 // tmp1 = card table base (holder)
732 Address curr_ct_holder_addr(xthread, in_bytes(ShenandoahThreadLocalData::card_table_offset()));
733 __ ld(tmp1, curr_ct_holder_addr);
734
735 // tmp1 = effective address
736 __ la(tmp2, address);
737
738 // tmp2 = &card_table[ addr >> CardTable::card_shift() ] ; card index
739 __ srli(tmp2, tmp2, CardTable::card_shift());
740 __ add(tmp2, tmp2, tmp1);
741
742 if (UseCondCardMark) {
743 Label L_already_dirty;
744 __ lbu(tmp1, Address(tmp2));
745 __ beqz(tmp1, L_already_dirty);
746 __ sb(zr, Address(tmp2));
747 __ bind(L_already_dirty);
748 } else {
749 __ sb(zr, Address(tmp2));
750 }
751 }
752
753 void ShenandoahBarrierStubC2::enter_if_gc_state(MacroAssembler& masm, const char test_state, Register tmp) {
754 Assembler::InlineSkippedInstructionsCounter skip_counter(&masm);
755
756 // Emit the unconditional branch in the first version of the method.
757 // Let the rest of runtime figure out how to manage it.
758 __ relocate(barrier_Relocation::spec(), (int)ShenandoahNMethod::gc_state_to_reloc(test_state));
759 __ j(*entry());
760
761 // This is were the slowpath stub will return to or the code above will
762 // jump to if the checks are false
763 __ bind(*continuation());
764 }
765
766 address ShenandoahBarrierSetAssembler::parse_stub_address(address pc) {
767 NativeInstruction* ni = nativeInstruction_at(pc);
768 assert(ni->is_jump(), "Initial code version: GC barrier fastpath must be a jump");
769 NativeJump* jmp = nativeJump_at(pc);
770 return jmp->jump_destination();
771 }
772
773 static bool is_nop(address pc) {
774 if (*(pc + 0) != 0x13) return false;
775 if (*(pc + 1) != 0x00) return false;
776 if (*(pc + 2) != 0x00) return false;
777 if (*(pc + 3) != 0x00) return false;
778 return true;
779 }
780
781 static void insert_nop(address pc) {
782 *reinterpret_cast<int32_t*>(pc) = 0x00000013;
783 assert(is_nop(pc), "Should be");
784 ICache::invalidate_range(pc, 4);
785 }
786
787 static void check_at(bool cond, address pc, const char* msg) {
788 assert(cond, "%s: at PC " PTR_FORMAT ": %02x%02x%02x%02x",
789 msg, p2i(pc), *(pc + 0), *(pc + 1), *(pc + 2), *(pc + 3));
790 }
791
792 bool ShenandoahBarrierSetAssembler::is_active(address pc) {
793 NativeInstruction* ni = nativeInstruction_at(pc);
794 return ni->is_jump();
795 }
796
797 void ShenandoahBarrierSetAssembler::patch_branch_to_nop(address pc) {
798 NativeInstruction* ni = nativeInstruction_at(pc);
799 if (ni->is_jump()) {
800 insert_nop(pc);
801 } else {
802 check_at(is_nop(pc), pc, "Should already be nop");
803 }
804 }
805
806 void ShenandoahBarrierSetAssembler::patch_nop_to_branch(address pc, address stub_addr) {
807 NativeInstruction* ni = nativeInstruction_at(pc);
808 if (is_nop(pc)) {
809 NativeJump::insert(pc, stub_addr);
810 } else {
811 check_at(ni->is_jump(), pc, "Should already be jump");
812 check_at(nativeJump_at(pc)->jump_destination() == stub_addr, pc, "Jump should be to the same address");
813 }
814 }
815
816 void ShenandoahBarrierStubC2::emit_code(MacroAssembler& masm) {
817 Assembler::InlineSkippedInstructionsCounter skip_counter(&masm);
818 assert(_needs_keep_alive_barrier || _needs_load_ref_barrier, "Why are you here?");
819
820 __ bind(*entry());
821
822 // If we need to load ourselves, do it here.
823 if (_do_load) {
824 if (_narrow) {
825 __ lwu(_obj, _addr);
826 } else {
827 __ ld(_obj, _addr);
828 }
829 }
830
831 // If the object is null, there is no point in applying barriers.
832 maybe_far_jump_if_zero(masm, _obj);
833
834 // We need to make sure that loads done by callers survive across slow-path calls.
835 // For self-loads, we need to care about the case when both KA and LRB are enabled (rare).
836 bool needs_both_barriers = _needs_keep_alive_barrier && _needs_load_ref_barrier;
837 if (!_do_load || needs_both_barriers) {
838 preserve(_obj);
839 }
840
841 // Go for barriers. Barriers can return straight to continuation, as long
842 // as another barrier is not needed and we can reach the fastpath.
843 if (needs_both_barriers) {
844 keepalive(masm, nullptr);
845 lrb(masm);
846 } else if (_needs_keep_alive_barrier) {
847 keepalive(masm, continuation());
848 } else if (_needs_load_ref_barrier) {
849 lrb(masm);
850 } else {
851 ShouldNotReachHere();
852 }
853 }
854
855 void ShenandoahBarrierStubC2::maybe_far_jump_if_zero(MacroAssembler& masm, Register reg) {
856 Label L_short_jump;
857 __ bnez(reg, L_short_jump);
858 __ j(*continuation());
859 __ bind(L_short_jump);
860 }
861
862 void ShenandoahBarrierStubC2::keepalive(MacroAssembler& masm, Label* L_done) {
863 Address index(xthread, in_bytes(ShenandoahThreadLocalData::satb_mark_queue_index_offset()));
864 Address buffer(xthread, in_bytes(ShenandoahThreadLocalData::satb_mark_queue_buffer_offset()));
865 Label L_through, L_slowpath;
866
867 // If another barrier is enabled as well, do a check for a specific barrier.
868 if (_needs_load_ref_barrier) {
869 assert(L_done == nullptr, "Should be");
870 // Emit the unconditional branch in the first version of the method.
871 // Let the rest of runtime figure out how to manage it.
872 // TODO: We could have spared the over-jump if patching knew we need the inverse branch.
873 char state_to_check = ShenandoahHeap::MARKING;
874 Label L_over;
875 __ relocate(barrier_Relocation::spec(), ShenandoahNMethod::gc_state_to_reloc(state_to_check));
876 __ j(L_over);
877 __ j(L_through);
878 __ bind(L_over);
879 }
880
881 // Fast-path: put object into buffer.
882 // If buffer is already full, go slow.
883 __ ld(_tmp1, index);
884 __ beqz(_tmp1, L_slowpath);
885 __ subi(_tmp1, _tmp1, wordSize);
886 __ sd(_tmp1, index);
887 __ ld(_tmp2, buffer);
888
889 // Store the object in queue.
890 // If object is narrow, we need to decode it before inserting.
891 __ add(_tmp1, _tmp1, _tmp2);
892 if (_narrow) {
893 __ decode_heap_oop_not_null(_tmp2, _obj);
894 __ sd(_tmp2, Address(_tmp1));
895 } else {
896 __ sd(_obj, Address(_tmp1));
897 }
898
899 // Fast-path exits here.
900 if (L_done != nullptr) {
901 __ j(*L_done);
902 } else {
903 __ j(L_through);
904 }
905
906 // Slow-path: call runtime to handle.
907 __ bind(L_slowpath);
908
909 {
910 SaveLiveRegisters slr(&masm, this);
911
912 // Go to runtime and handle the rest there.
913 __ mv(c_rarg0, _obj);
914 __ la(ra, RuntimeAddress(keepalive_runtime_entry_addr()));
915 __ jalr(ra);
916 }
917 if (L_done != nullptr) {
918 __ j(*L_done);
919 } else {
920 __ bind(L_through);
921 }
922 }
923
924 void ShenandoahBarrierStubC2::lrb(MacroAssembler& masm) {
925 Label L_slow;
926
927 // If weak references are being processed, weak/phantom loads need to go slow,
928 // regardless of their cset status.
929 if (_needs_load_ref_weak_barrier) {
930 char state_to_check = ShenandoahHeap::WEAK_ROOTS;
931 __ relocate(barrier_Relocation::spec(), ShenandoahNMethod::gc_state_to_reloc(state_to_check));
932 __ j(L_slow);
933 }
934
935 if (_needs_keep_alive_barrier) {
936 // Emit the unconditional branch in the first version of the method.
937 // Let the rest of runtime figure out how to manage it.
938 // TODO: We could have spared the over-jump if patching knew we need the inverse branch.
939 char state_to_check = ShenandoahHeap::HAS_FORWARDED | (_needs_load_ref_weak_barrier ? ShenandoahHeap::WEAK_ROOTS : 0);
940 Label L_over;
941 __ relocate(barrier_Relocation::spec(), ShenandoahNMethod::gc_state_to_reloc(state_to_check));
942 __ j(L_over);
943 __ j(*continuation());
944 __ bind(L_over);
945 }
946
947 // Cset-check. Fall-through to slow if in collection set.
948 if (_narrow) {
949 __ decode_heap_oop_not_null(_tmp2, _obj);
950 } else {
951 __ mv(_tmp2, _obj);
952 }
953
954 __ mv(_tmp1, ShenandoahHeap::in_cset_fast_test_addr());
955 __ srli(_tmp2, _tmp2, ShenandoahHeapRegion::region_size_bytes_shift_jint());
956 __ add(_tmp1, _tmp1, _tmp2);
957 __ lbu(_tmp1, Address(_tmp1, 0));
958 maybe_far_jump_if_zero(masm, _tmp1);
959
960 // Slow path
961 __ bind(L_slow);
962
963 // Obj is the result, need to temporarily stop preserving it.
964 bool is_obj_preserved = is_preserved(_obj);
965 if (is_obj_preserved) {
966 dont_preserve(_obj);
967 }
968 {
969 SaveLiveRegisters slr(&masm, this);
970
971 // Shuffle in the arguments. The end result should be:
972 // c_rarg0 <- obj
973 // c_rarg1 <- lea(addr)
974 if (c_rarg0 == _obj) {
975 __ la(c_rarg1, _addr);
976 } else if (c_rarg1 == _obj) {
977 __ mv(_tmp1, c_rarg1);
978 __ la(c_rarg1, _addr);
979 __ mv(c_rarg0, _tmp1);
980 } else {
981 assert_different_registers(c_rarg1, _obj);
982 __ la(c_rarg1, _addr);
983 __ mv(c_rarg0, _obj);
984 }
985
986 // Go to runtime and handle the rest there.
987 __ la(ra, RuntimeAddress(lrb_runtime_entry_addr()));
988 __ jalr(ra);
989
990 // Save the result where needed. Narrow entries return narrowOop (32 bits)
991 // we need to zero the upper 32 bits of x10.
992 if (_narrow) {
993 __ zext_w(_obj, x10);
994 } else {
995 __ mv(_obj, x10);
996 }
997 }
998 if (is_obj_preserved) {
999 preserve(_obj);
1000 }
1001
1002 __ j(*continuation());
1003 }
1004
1005 int ShenandoahBarrierStubC2::available_gp_registers() {
1006 Unimplemented(); // Not used
1007 return 0;
1008 }
1009
1010 bool ShenandoahBarrierStubC2::is_special_register(Register r) {
1011 Unimplemented(); // Not used
1012 return true;
1013 }
1014
1015 void ShenandoahBarrierStubC2::post_init() {
1016 // Do nothing.
1017 }
1018
1019 #endif // COMPILER2