1 /*
2 * Copyright (c) 2026, Oracle and/or its affiliates. All rights reserved.
3 * Copyright (c) 2018, 2020, Red Hat, Inc. All rights reserved.
4 * Copyright (c) 2020, 2021, Huawei Technologies Co., Ltd. All rights reserved.
5 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
6 *
7 * This code is free software; you can redistribute it and/or modify it
8 * under the terms of the GNU General Public License version 2 only, as
9 * published by the Free Software Foundation.
10 *
11 * This code is distributed in the hope that it will be useful, but WITHOUT
12 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
13 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
14 * version 2 for more details (a copy is included in the LICENSE file that
15 * accompanied this code).
16 *
17 * You should have received a copy of the GNU General Public License version
18 * 2 along with this work; if not, write to the Free Software Foundation,
19 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
20 *
21 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
22 * or visit www.oracle.com if you need additional information or have any
23 * questions.
24 *
25 */
26
27 #include "code/aotCodeCache.hpp"
28 #include "gc/shenandoah/heuristics/shenandoahHeuristics.hpp"
29 #include "gc/shenandoah/mode/shenandoahMode.hpp"
30 #include "gc/shenandoah/shenandoahBarrierSet.hpp"
31 #include "gc/shenandoah/shenandoahBarrierSetAssembler.hpp"
32 #include "gc/shenandoah/shenandoahHeap.inline.hpp"
33 #include "gc/shenandoah/shenandoahHeapRegion.hpp"
34 #include "gc/shenandoah/shenandoahNMethod.inline.hpp"
35 #include "gc/shenandoah/shenandoahRuntime.hpp"
36 #include "gc/shenandoah/shenandoahThreadLocalData.hpp"
37 #include "interpreter/interp_masm.hpp"
38 #include "interpreter/interpreter.hpp"
39 #include "nativeInst_riscv.hpp"
40 #include "runtime/icache.hpp"
41 #include "runtime/javaThread.hpp"
42 #include "runtime/sharedRuntime.hpp"
43 #ifdef COMPILER1
44 #include "c1/c1_LIRAssembler.hpp"
45 #include "c1/c1_MacroAssembler.hpp"
46 #include "gc/shenandoah/c1/shenandoahBarrierSetC1.hpp"
47 #endif
48 #ifdef COMPILER2
49 #include "gc/shenandoah/c2/shenandoahBarrierSetC2.hpp"
50 #include "opto/output.hpp"
51 #endif
52
53 #define __ masm->
54
55 void ShenandoahBarrierSetAssembler::arraycopy_prologue(MacroAssembler* masm, DecoratorSet decorators, bool is_oop,
56 Register src, Register dst, Register count, RegSet saved_regs) {
57 if (is_oop) {
58 bool dest_uninitialized = (decorators & IS_DEST_UNINITIALIZED) != 0;
59 if ((ShenandoahSATBBarrier && !dest_uninitialized) || ShenandoahLoadRefBarrier) {
60
61 Label done;
62
63 // Avoid calling runtime if count == 0
64 __ beqz(count, done);
65
66 // Is GC active?
67 Address gc_state(xthread, in_bytes(ShenandoahThreadLocalData::gc_state_offset()));
68 assert_different_registers(src, dst, count, t0);
69
70 assert(!saved_regs.contains(t0), "Sanity: about to clobber t0");
71
72 __ lbu(t0, gc_state);
73 if (ShenandoahSATBBarrier && dest_uninitialized) {
74 __ test_bit(t0, t0, ShenandoahHeap::HAS_FORWARDED_BITPOS);
75 __ beqz(t0, done);
76 } else {
77 __ andi(t0, t0, ShenandoahHeap::HAS_FORWARDED | ShenandoahHeap::MARKING);
78 __ beqz(t0, done);
79 }
80
81 __ push_call_clobbered_registers();
82 // If arguments are not in proper places, shuffle them.
83 // Doing this via the stack is the most straight-forward way to avoid
84 // accidentally smashing any register.
85 if (c_rarg0 != src || c_rarg1 != dst || c_rarg2 != count) {
86 __ push_reg(RegSet::of(src), sp);
87 __ push_reg(RegSet::of(dst), sp);
88 __ push_reg(RegSet::of(count), sp);
89 __ pop_reg(RegSet::of(c_rarg2), sp);
90 __ pop_reg(RegSet::of(c_rarg1), sp);
91 __ pop_reg(RegSet::of(c_rarg0), sp);
92 }
93 address target = nullptr;
94 if (UseCompressedOops) {
95 target = CAST_FROM_FN_PTR(address, ShenandoahRuntime::arraycopy_barrier_narrow_oop);
96 } else {
97 target = CAST_FROM_FN_PTR(address, ShenandoahRuntime::arraycopy_barrier_oop);
98 }
99 __ call_VM_leaf(target, 3);
100 __ pop_call_clobbered_registers();
101 __ bind(done);
102 }
103 }
104 }
105
106 void ShenandoahBarrierSetAssembler::arraycopy_epilogue(MacroAssembler* masm, DecoratorSet decorators, bool is_oop,
107 Register start, Register count, Register tmp) {
108 if (ShenandoahCardBarrier && is_oop) {
109 gen_write_ref_array_post_barrier(masm, decorators, start, count, tmp);
110 }
111 }
112
113 void ShenandoahBarrierSetAssembler::satb_barrier(MacroAssembler* masm,
114 Register obj,
115 Register pre_val,
116 Register thread,
117 Register tmp1,
118 Register tmp2) {
119 assert(ShenandoahSATBBarrier, "Should be checked by caller");
120 assert(thread == xthread, "must be");
121
122 Label done;
123 Label runtime;
124
125 assert_different_registers(obj, pre_val, tmp1, tmp2);
126 assert(pre_val != noreg && tmp1 != noreg && tmp2 != noreg, "expecting a register");
127
128 Address index(thread, in_bytes(ShenandoahThreadLocalData::satb_mark_queue_index_offset()));
129 Address buffer(thread, in_bytes(ShenandoahThreadLocalData::satb_mark_queue_buffer_offset()));
130
131 // Is marking active?
132 Address gc_state(xthread, in_bytes(ShenandoahThreadLocalData::gc_state_offset()));
133 __ lbu(t1, gc_state);
134 __ test_bit(t1, t1, ShenandoahHeap::MARKING_BITPOS);
135 __ beqz(t1, done);
136
137 // Do we need to load the previous value?
138 if (obj != noreg) {
139 if (UseCompressedOops) {
140 __ lwu(pre_val, Address(obj, 0));
141 __ decode_heap_oop(pre_val);
142 } else {
143 __ ld(pre_val, Address(obj, 0));
144 }
145 }
146
147 // Is the previous value null?
148 __ beqz(pre_val, done);
149
150 // Can we store original value in the thread's buffer?
151 // Is index == 0?
152 // (The index field is typed as size_t.)
153 __ ld(tmp1, index); // tmp := *index_adr
154 __ beqz(tmp1, runtime); // tmp == 0? If yes, goto runtime
155
156 __ subi(tmp1, tmp1, wordSize); // tmp := tmp - wordSize
157 __ sd(tmp1, index); // *index_adr := tmp
158 __ ld(tmp2, buffer);
159 __ add(tmp1, tmp1, tmp2); // tmp := tmp + *buffer_adr
160
161 // Record the previous value
162 __ sd(pre_val, Address(tmp1, 0));
163 __ j(done);
164
165 // Slow-path call.
166 __ bind(runtime);
167 __ enter();
168 __ push_call_clobbered_registers();
169 if (c_rarg0 != pre_val) {
170 __ mv(c_rarg0, pre_val);
171 }
172 // Calling with super_call_VM_leaf with c_rarg0 bypasses interpreter checks and avoids any moves.
173 __ super_call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::write_barrier_pre), c_rarg0);
174 __ pop_call_clobbered_registers();
175 __ leave();
176
177 __ bind(done);
178 }
179
180 void ShenandoahBarrierSetAssembler::load_reference_barrier(MacroAssembler* masm,
181 Register dst,
182 Address load_addr,
183 DecoratorSet decorators) {
184 assert(ShenandoahLoadRefBarrier, "Should be enabled");
185 assert(dst != t1 && load_addr.base() != t1, "need t1");
186 assert_different_registers(load_addr.base(), t0, t1);
187
188 bool is_strong = ShenandoahBarrierSet::is_strong_access(decorators);
189 bool is_weak = ShenandoahBarrierSet::is_weak_access(decorators);
190 bool is_phantom = ShenandoahBarrierSet::is_phantom_access(decorators);
191 bool is_native = ShenandoahBarrierSet::is_native_access(decorators);
192 bool is_narrow = UseCompressedOops && !is_native;
193
194 Label heap_stable, not_cset;
195 Address gc_state(xthread, in_bytes(ShenandoahThreadLocalData::gc_state_offset()));
196 __ lbu(t1, gc_state);
197
198 // Check for heap stability
199 if (is_strong) {
200 __ test_bit(t1, t1, ShenandoahHeap::HAS_FORWARDED_BITPOS);
201 __ beqz(t1, heap_stable);
202 } else {
203 Label lrb;
204 __ test_bit(t0, t1, ShenandoahHeap::WEAK_ROOTS_BITPOS);
205 __ bnez(t0, lrb);
206 __ test_bit(t0, t1, ShenandoahHeap::HAS_FORWARDED_BITPOS);
207 __ beqz(t0, heap_stable);
208 __ bind(lrb);
209 }
210
211 // use x11 for load address
212 Register result_dst = dst;
213 if (dst == x11) {
214 __ mv(t1, dst);
215 dst = t1;
216 }
217
218 // Save x10 and x11, unless it is an output register
219 RegSet saved_regs = RegSet::of(x10, x11) - result_dst;
220 __ push_reg(saved_regs, sp);
221 __ la(x11, load_addr);
222 __ mv(x10, dst);
223
224 // Test for in-cset
225 if (is_strong) {
226 #if INCLUDE_CDS
227 if (AOTCodeCache::is_on_for_dump()) {
228 __ ld(t1, ExternalAddress(AOTRuntimeConstants::cset_base_address()));
229 __ lwu(t0, ExternalAddress(AOTRuntimeConstants::grain_shift_address()));
230 __ srl(t0, x10, t0);
231 } else
232 #endif
233 {
234 __ mv(t1, ShenandoahHeap::in_cset_fast_test_addr());
235 __ srli(t0, x10, ShenandoahHeapRegion::region_size_bytes_shift_jint());
236 }
237 __ add(t1, t1, t0);
238 __ lbu(t1, Address(t1));
239 __ test_bit(t0, t1, 0);
240 __ beqz(t0, not_cset);
241 }
242
243 // Slow-path call
244 __ enter();
245 __ push_call_clobbered_registers();
246 address target = nullptr;
247 if (is_strong) {
248 if (is_narrow) {
249 target = CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_strong_narrow);
250 } else {
251 target = CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_strong);
252 }
253 } else if (is_weak) {
254 if (is_narrow) {
255 target = CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_weak_narrow);
256 } else {
257 target = CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_weak);
258 }
259 } else {
260 assert(is_phantom, "only remaining strength");
261 assert(!is_narrow, "phantom access cannot be narrow");
262 target = CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_phantom);
263 }
264 // Calling with super_call_VM_leaf with c_rarg0/1 bypasses interpreter checks and avoids any moves.
265 __ super_call_VM_leaf(target, c_rarg0, c_rarg1);
266 __ mv(t0, x10);
267 __ pop_call_clobbered_registers();
268 __ mv(x10, t0);
269 __ leave();
270
271 __ bind(not_cset);
272 __ mv(result_dst, x10);
273 __ pop_reg(saved_regs, sp);
274
275 __ bind(heap_stable);
276 }
277
278 //
279 // Arguments:
280 //
281 // Inputs:
282 // src: oop location to load from, might be clobbered
283 //
284 // Output:
285 // dst: oop loaded from src location
286 //
287 // Kill:
288 // x30 (tmp reg)
289 //
290 // Alias:
291 // dst: x30 (might use x30 as temporary output register to avoid clobbering src)
292 //
293 void ShenandoahBarrierSetAssembler::load_at(MacroAssembler* masm,
294 DecoratorSet decorators,
295 BasicType type,
296 Register dst,
297 Address src,
298 Register tmp1,
299 Register tmp2) {
300 // 1: non-reference load, no additional barrier is needed
301 if (!is_reference_type(type)) {
302 BarrierSetAssembler::load_at(masm, decorators, type, dst, src, tmp1, tmp2);
303 return;
304 }
305
306 // 2: load a reference from src location and apply LRB if needed
307 if (ShenandoahBarrierSet::need_load_reference_barrier(decorators, type)) {
308 Register result_dst = dst;
309
310 // Preserve src location for LRB
311 RegSet saved_regs;
312 if (dst == src.base()) {
313 dst = (src.base() == x28) ? x29 : x28;
314 saved_regs = RegSet::of(dst);
315 __ push_reg(saved_regs, sp);
316 }
317 assert_different_registers(dst, src.base());
318
319 BarrierSetAssembler::load_at(masm, decorators, type, dst, src, tmp1, tmp2);
320
321 load_reference_barrier(masm, dst, src, decorators);
322
323 if (dst != result_dst) {
324 __ mv(result_dst, dst);
325 dst = result_dst;
326 }
327
328 if (saved_regs.bits() != 0) {
329 __ pop_reg(saved_regs, sp);
330 }
331 } else {
332 BarrierSetAssembler::load_at(masm, decorators, type, dst, src, tmp1, tmp2);
333 }
334
335 // 3: apply keep-alive barrier if needed
336 if (ShenandoahBarrierSet::need_keep_alive_barrier(decorators, type)) {
337 satb_barrier(masm /* masm */,
338 noreg /* obj */,
339 dst /* pre_val */,
340 xthread /* thread */,
341 tmp1 /* tmp1 */,
342 tmp2 /* tmp2 */);
343 }
344 }
345
346 void ShenandoahBarrierSetAssembler::card_barrier(MacroAssembler* masm, Register obj, Register tmp1, Register tmp2) {
347 assert(ShenandoahCardBarrier, "Should have been checked by caller");
348 assert(CardTable::dirty_card_val() == 0, "must be");
349 assert_different_registers(obj, tmp1, tmp2);
350
351 __ srli(obj, obj, CardTable::card_shift());
352
353 Address curr_ct_holder_addr(xthread, in_bytes(ShenandoahThreadLocalData::card_table_offset()));
354 __ ld(tmp1, curr_ct_holder_addr);
355 __ add(tmp1, obj, tmp1);
356
357 if (UseCondCardMark) {
358 Label L_already_dirty;
359 __ lbu(tmp2, Address(tmp1));
360 __ beqz(tmp2, L_already_dirty);
361 __ sb(zr, Address(tmp1));
362 __ bind(L_already_dirty);
363 } else {
364 __ sb(zr, Address(tmp1));
365 }
366 }
367
368 void ShenandoahBarrierSetAssembler::store_at(MacroAssembler* masm, DecoratorSet decorators, BasicType type,
369 Address dst, Register val, Register tmp1, Register tmp2, Register tmp3) {
370 // 1: non-reference types require no barriers
371 if (!is_reference_type(type)) {
372 BarrierSetAssembler::store_at(masm, decorators, type, dst, val, tmp1, tmp2, tmp3);
373 return;
374 }
375
376 // Flatten object address right away for simplicity: likely needed by barriers
377 if (dst.offset() == 0) {
378 if (dst.base() != tmp3) {
379 __ mv(tmp3, dst.base());
380 }
381 } else {
382 __ la(tmp3, dst);
383 }
384
385 // 2: pre-barrier: SATB needs the previous value
386 if (ShenandoahBarrierSet::need_satb_barrier(decorators, type)) {
387 satb_barrier(masm,
388 tmp3 /* obj */,
389 tmp2 /* pre_val */,
390 xthread /* thread */,
391 tmp1 /* tmp */,
392 t0 /* tmp2 */);
393 }
394
395 // Store!
396 BarrierSetAssembler::store_at(masm, decorators, type, Address(tmp3, 0), val, noreg, noreg, noreg);
397
398 // 3: post-barrier: card barrier needs store address
399 bool storing_non_null = (val != noreg);
400 if (ShenandoahBarrierSet::need_card_barrier(decorators, type) && storing_non_null) {
401 card_barrier(masm, tmp3, tmp1, tmp2);
402 }
403 }
404
405 void ShenandoahBarrierSetAssembler::try_resolve_jobject_in_native(MacroAssembler* masm, Register jni_env,
406 Register obj, Register tmp, Label& slowpath) {
407 Label done;
408 // Resolve jobject
409 BarrierSetAssembler::try_resolve_jobject_in_native(masm, jni_env, obj, tmp, slowpath);
410
411 // Check for null.
412 __ beqz(obj, done);
413
414 assert(obj != t1, "need t1");
415 Address gc_state(jni_env, ShenandoahThreadLocalData::gc_state_offset() - JavaThread::jni_environment_offset());
416 __ lbu(t1, gc_state);
417
418 // Check for heap in evacuation phase
419 __ test_bit(t0, t1, ShenandoahHeap::EVACUATION_BITPOS);
420 __ bnez(t0, slowpath);
421
422 __ bind(done);
423 }
424
425 void ShenandoahBarrierSetAssembler::try_peek_weak_handle_in_nmethod(MacroAssembler *masm, Register weak_handle,
426 Register obj, Register tmp, Label& slow_path) {
427 assert_different_registers(weak_handle, tmp, noreg);
428 assert_different_registers(obj, tmp, noreg);
429
430
431 Label done;
432
433 // Peek weak handle using the standard implementation.
434 BarrierSetAssembler::try_peek_weak_handle_in_nmethod(masm, weak_handle, obj, tmp, slow_path);
435
436 // Check if the reference is null, and if it is, take the fast path.
437 __ beqz(obj, done);
438
439 Address gc_state(xthread, ShenandoahThreadLocalData::gc_state_offset());
440 __ lbu(tmp, gc_state);
441
442 // Check if the heap is under weak-reference/roots processing, in
443 // which case we need to take the slow path.
444 __ test_bit(tmp, tmp, ShenandoahHeap::WEAK_ROOTS_BITPOS);
445 __ bnez(tmp, slow_path);
446 __ bind(done);
447 }
448
449 void ShenandoahBarrierSetAssembler::check_oop(MacroAssembler* masm, Register obj, Register tmp1, Register tmp2, Label& L_error) {
450 assert_different_registers(obj, tmp1, tmp2);
451 // Check if the oop is in the right area of memory
452 #if INCLUDE_CDS
453 if (AOTCodeCache::is_on_for_dump()) {
454 __ ld(tmp2, ExternalAddress(AOTRuntimeConstants::verify_oop_mask_address()));
455 __ andr(tmp1, obj, tmp2);
456 __ ld(tmp2, ExternalAddress(AOTRuntimeConstants::verify_oop_bits_address()));
457 } else
458 #endif
459 {
460 __ mv(tmp2, (intptr_t) Universe::verify_oop_mask());
461 __ andr(tmp1, obj, tmp2);
462 __ mv(tmp2, (intptr_t) Universe::verify_oop_bits());
463 }
464
465 // Compare tmp1 and tmp2.
466 __ bne(tmp1, tmp2, L_error);
467
468 // This routine is sometimes called before applying GC barriers.
469 // With +COH, loading the klass may end up loading forwarding pointer instead.
470 Label L_skip;
471 if (UseCompactObjectHeaders) {
472 Address gc_state(xthread, ShenandoahThreadLocalData::gc_state_offset());
473 __ lbu(tmp1, gc_state);
474 __ test_bit(tmp1, tmp1, ShenandoahHeap::HAS_FORWARDED_BITPOS);
475 __ bnez(tmp1, L_skip);
476 }
477
478 // Make sure klass is 'reasonable', which is not zero.
479 __ load_narrow_klass(tmp1, obj);
480 __ beqz(tmp1, L_error);
481
482 __ bind(L_skip);
483 }
484
485 void ShenandoahBarrierSetAssembler::gen_write_ref_array_post_barrier(MacroAssembler* masm, DecoratorSet decorators,
486 Register start, Register count, Register tmp) {
487 assert(ShenandoahCardBarrier, "Did you mean to enable ShenandoahCardBarrier?");
488
489 Label L_loop, L_done;
490 const Register end = count;
491
492 // Zero count? Nothing to do.
493 __ beqz(count, L_done);
494
495 // end = start + count << LogBytesPerHeapOop
496 // last element address to make inclusive
497 __ shadd(end, count, start, tmp, LogBytesPerHeapOop);
498 __ subi(end, end, BytesPerHeapOop);
499 __ srli(start, start, CardTable::card_shift());
500 __ srli(end, end, CardTable::card_shift());
501
502 // number of bytes to copy
503 __ sub(count, end, start);
504
505 Address curr_ct_holder_addr(xthread, in_bytes(ShenandoahThreadLocalData::card_table_offset()));
506 __ ld(tmp, curr_ct_holder_addr);
507 __ add(start, start, tmp);
508
509 __ bind(L_loop);
510 __ add(tmp, start, count);
511 __ sb(zr, Address(tmp));
512 __ subi(count, count, 1);
513 __ bgez(count, L_loop);
514 __ bind(L_done);
515 }
516
517 #undef __
518
519 address ShenandoahBarrierSetAssembler::parse_jump_address(address pc) {
520 NativeInstruction* ni = nativeInstruction_at(pc);
521 assert(ni->is_jump(), "Must be a jump");
522 NativeJump* jmp = nativeJump_at(pc);
523 return jmp->jump_destination();
524 }
525
526 static uint32_t encode_patchable_nop() {
527 return 0x00000013;
528 }
529
530 static uint32_t encode_patchable_jump(address pc, address target_pc) {
531 int32_t disp = checked_cast<int32_t>((intptr_t)target_pc - (intptr_t)pc);
532 return Assembler::encode_jal(x0, disp);
533 }
534
535 void ShenandoahBarrierSetAssembler::insert_patchable_nop(address pc) {
536 *((uint32_t*)pc) = encode_patchable_nop();
537 assert(nativeInstruction_at(pc)->is_nop(), "Sanity");
538 if (!UseCtxFencei) {
539 ICache::invalidate_word(pc);
540 }
541 }
542
543 void ShenandoahBarrierSetAssembler::insert_patchable_jump(address pc, address target_pc) {
544 *((uint32_t*)pc) = encode_patchable_jump(pc, target_pc);
545 if (!UseCtxFencei) {
546 ICache::invalidate_word(pc);
547 }
548 }
549
550 bool ShenandoahBarrierSetAssembler::is_patchable_nop(address pc) {
551 return *((uint32_t*)pc) == encode_patchable_nop();
552 }
553
554 bool ShenandoahBarrierSetAssembler::is_patchable_jump(address pc, address target_pc) {
555 return *((uint32_t*)pc) == encode_patchable_jump(pc, target_pc);
556 }
557
558 #ifdef COMPILER1
559
560 #define __ ce->masm()->
561
562 void ShenandoahBarrierSetAssembler::keepalive_barrier_c1_stub(LIR_Assembler* ce, ShenandoahKeepaliveBarrierStub* stub) {
563 __ bind(*stub->entry());
564
565 ShenandoahBarrierSetC1* bs = (ShenandoahBarrierSetC1*)BarrierSet::barrier_set()->barrier_set_c1();
566
567 Register obj = stub->obj()->as_register();
568
569 if (stub->do_load()) {
570 ce->mem2reg(stub->addr(), stub->obj(), T_OBJECT, lir_patch_none, nullptr, false /* wide */);
571 }
572 __ beqz(obj, *stub->continuation(), /* is_far */ true);
573
574 ce->store_parameter(obj, 0);
575 __ far_call(RuntimeAddress(bs->keepalive_barrier_stub()));
576 __ j(*stub->continuation());
577 }
578
579 void ShenandoahBarrierSetAssembler::load_reference_barrier_c1_stub(LIR_Assembler* ce, ShenandoahLoadReferenceBarrierStub* stub) {
580 __ bind(*stub->entry());
581
582 ShenandoahBarrierSetC1* bs = (ShenandoahBarrierSetC1*) BarrierSet::barrier_set()->barrier_set_c1();
583
584 Register obj = stub->obj()->as_register();
585 Register addr = stub->addr()->as_pointer_register();
586 Register slow_result = stub->slow_result()->as_register();
587 assert_different_registers(obj, addr, slow_result);
588 assert(slow_result == x10, "C1 must know about our slow call result register");
589
590 ce->store_parameter(obj, 0);
591 ce->store_parameter(addr, 1);
592 __ far_call(RuntimeAddress(bs->load_reference_barrier_stub(stub->decorators())));
593 if (obj != slow_result) {
594 __ mv(obj, slow_result);
595 }
596
597 __ j(*stub->continuation());
598 }
599
600 #undef __
601
602 #define __ sasm->
603
604 void ShenandoahBarrierSetAssembler::keepalive_barrier_c1_runtime_stub(StubAssembler* sasm) {
605 __ prologue("shenandoah_keepalive_barrier", false);
606 const Register tmp_obj = x10;
607 const Register tmp1 = x11;
608 const Register tmp2 = x12;
609 __ push_reg(RegSet::of(tmp1, tmp2, tmp_obj), sp);
610 __ load_parameter(0, tmp_obj);
611 satb_barrier(sasm, noreg, tmp_obj, xthread, tmp1, tmp2);
612 __ pop_reg(RegSet::of(tmp1, tmp2, tmp_obj), sp);
613 __ epilogue();
614 }
615
616 void ShenandoahBarrierSetAssembler::load_reference_barrier_c1_runtime_stub(StubAssembler* sasm, DecoratorSet decorators) {
617 __ prologue("shenandoah_load_reference_barrier", false);
618 const Register tmp_obj = x10;
619 const Register tmp_addr = x11;
620 __ push_reg(RegSet::of(tmp_addr), sp);
621 __ load_parameter(0, tmp_obj);
622 __ load_parameter(1, tmp_addr);
623 load_reference_barrier(sasm, tmp_obj, Address(tmp_addr, 0), decorators);
624 __ pop_reg(RegSet::of(tmp_addr), sp);
625 __ epilogue();
626 }
627
628 #undef __
629
630 #endif // COMPILER1
631
632 #ifdef COMPILER2
633
634 #undef __
635 #define __ masm->
636
637 void ShenandoahBarrierSetAssembler::load_c2(const MachNode* node, MacroAssembler* masm, Register dst, Address src, Register tmp1, Register tmp2, bool is_narrow) {
638 // Do the actual load. This load is the candidate for implicit null check, and MUST come first.
639 if (is_narrow) {
640 __ lwu(dst, src);
641 } else {
642 __ ld(dst, src);
643 }
644
645 ShenandoahBarrierStubC2::load_post(masm, node, dst, src, tmp1, tmp2, is_narrow);
646 }
647
648 void ShenandoahBarrierSetAssembler::store_c2(const MachNode* node, MacroAssembler* masm, Address dst, bool dst_narrow,
649 Register src, bool src_narrow, Register tmp1, Register tmp2, Register tmp3) {
650
651 ShenandoahBarrierStubC2::store_pre(masm, node, dst, tmp1, tmp2, tmp3, dst_narrow);
652
653 // Do the actual store
654 if (dst_narrow) {
655 if (!src_narrow) {
656 // Need to encode into tmp, because we cannot clobber src.
657 assert(tmp1 != noreg, "need temp register");
658 if ((node->barrier_data() & ShenandoahBitNotNull) == 0) {
659 __ encode_heap_oop(tmp1, src);
660 } else {
661 __ encode_heap_oop_not_null(tmp1, src);
662 }
663 src = tmp1;
664 }
665 __ sw(src, dst);
666 } else {
667 __ sd(src, dst);
668 }
669
670 ShenandoahBarrierStubC2::store_post(masm, node, dst, tmp2, tmp3);
671 }
672
673 void ShenandoahBarrierSetAssembler::compare_and_set_c2(const MachNode* node, MacroAssembler* masm, Register res, Register addr,
674 Register oldval, Register newval, Register tmp1, Register tmp2, Register tmp3, bool exchange, bool narrow, bool is_acquire) {
675 const Assembler::Aqrl acquire = is_acquire ? Assembler::aq : Assembler::relaxed;
676 const Assembler::Aqrl release = Assembler::rl;
677 const Assembler::operand_size size = narrow ? Assembler::uint32 : Assembler::int64;
678
679 ShenandoahBarrierStubC2::load_store_pre(masm, node, Address(addr), tmp1, tmp2, tmp3, narrow);
680
681 // CAS!
682 __ cmpxchg(addr, oldval, newval, size, acquire, release, /* result */ res, !exchange /* result_as_bool */);
683
684 ShenandoahBarrierStubC2::load_store_post(masm, node, Address(addr, 0), tmp2, tmp3);
685 }
686
687 void ShenandoahBarrierSetAssembler::get_and_set_c2(const MachNode* node, MacroAssembler* masm, Register preval,
688 Register newval, Register addr, Register tmp1, Register tmp2, Register tmp3, bool is_acquire) {
689 const bool is_narrow = node->bottom_type()->isa_narrowoop();
690
691 ShenandoahBarrierStubC2::load_store_pre(masm, node, Address(addr, 0), tmp1, tmp2, tmp3, is_narrow);
692
693 if (is_narrow) {
694 if (is_acquire) {
695 __ atomic_xchgalwu(preval, newval, addr);
696 } else {
697 __ atomic_xchgwu(preval, newval, addr);
698 }
699 } else {
700 if (is_acquire) {
701 __ atomic_xchgal(preval, newval, addr);
702 } else {
703 __ atomic_xchg(preval, newval, addr);
704 }
705 }
706
707 ShenandoahBarrierStubC2::load_store_post(masm, node, Address(addr, 0), tmp2, tmp3);
708 }
709
710 #undef __
711 #define __ masm.
712
713 void ShenandoahBarrierStubC2::cardtable(MacroAssembler& masm, Address address, Register tmp1, Register tmp2) {
714 assert(CardTable::dirty_card_val() == 0, "must be");
715 Assembler::InlineSkippedInstructionsCounter skip_counter(&masm);
716
717 // tmp1 = card table base (holder)
718 Address curr_ct_holder_addr(xthread, in_bytes(ShenandoahThreadLocalData::card_table_offset()));
719 __ ld(tmp1, curr_ct_holder_addr);
720
721 // tmp1 = effective address
722 __ la(tmp2, address);
723
724 // tmp2 = &card_table[ addr >> CardTable::card_shift() ] ; card index
725 __ srli(tmp2, tmp2, CardTable::card_shift());
726 __ add(tmp2, tmp2, tmp1);
727
728 if (UseCondCardMark) {
729 Label L_already_dirty;
730 __ lbu(tmp1, Address(tmp2));
731 __ beqz(tmp1, L_already_dirty);
732 __ sb(zr, Address(tmp2));
733 __ bind(L_already_dirty);
734 } else {
735 __ sb(zr, Address(tmp2));
736 }
737 }
738
739 void ShenandoahBarrierStubC2::patchable_jump(MacroAssembler& masm, const char gc_state, bool jump_when_state, Register tmp1, Register tmp2, Label* L_target) {
740 PhaseOutput* const output = Compile::current()->output();
741 if (output->in_scratch_emit_size()) {
742 // Avoid binding L_target in scratch emits.
743 // We know the patched check is exactly one incompressible instruction long.
744 Assembler::IncompressibleScope scope(&masm);
745 __ nop();
746 return;
747 }
748
749 // Emit the unconditional branch in the first version of the method.
750 // Let the rest of runtime figure out how to manage it.
751 __ relocate(patchable_barrier_Relocation::spec(ShenandoahNMethod::encode_to_reloc(gc_state, jump_when_state)));
752 __ j(*L_target);
753 }
754
755 void ShenandoahBarrierStubC2::enter_if_gc_state(MacroAssembler& masm, const char test_state, Register tmp1, Register tmp2) {
756 Assembler::InlineSkippedInstructionsCounter skip_counter(&masm);
757 patchable_jump_if_gc_state(masm, test_state, tmp1, tmp2, entry());
758 __ bind(*continuation());
759 }
760
761 void ShenandoahBarrierStubC2::emit_code(MacroAssembler& masm) {
762 Assembler::InlineSkippedInstructionsCounter skip_counter(&masm);
763 assert(_needs_keep_alive_barrier || _needs_load_ref_barrier, "Why are you here?");
764
765 __ bind(*entry());
766
767 // If we need to load ourselves, do it here.
768 if (_do_load) {
769 if (_narrow) {
770 __ lwu(_obj, _addr);
771 } else {
772 __ ld(_obj, _addr);
773 }
774 }
775
776 // If the object is null, there is no point in applying barriers.
777 maybe_far_jump_if_zero(masm, _obj);
778
779 // We need to make sure that loads done by callers survive across slow-path calls.
780 // For self-loads, we need to care about the case when both KA and LRB are enabled (rare).
781 bool needs_both_barriers = _needs_keep_alive_barrier && _needs_load_ref_barrier;
782 if (!_do_load || needs_both_barriers) {
783 preserve(_obj);
784 }
785
786 // Go for barriers. Barriers can return straight to continuation, as long
787 // as another barrier is not needed and we can reach the fastpath.
788 if (needs_both_barriers) {
789 keepalive(masm, nullptr);
790 lrb(masm);
791 } else if (_needs_keep_alive_barrier) {
792 keepalive(masm, continuation());
793 } else if (_needs_load_ref_barrier) {
794 lrb(masm);
795 } else {
796 ShouldNotReachHere();
797 }
798 }
799
800 void ShenandoahBarrierStubC2::maybe_far_jump_if_zero(MacroAssembler& masm, Register reg) {
801 Label L_short_jump;
802 __ bnez(reg, L_short_jump);
803 __ j(*continuation());
804 __ bind(L_short_jump);
805 }
806
807 void ShenandoahBarrierStubC2::keepalive(MacroAssembler& masm, Label* L_done) {
808 Address index(xthread, in_bytes(ShenandoahThreadLocalData::satb_mark_queue_index_offset()));
809 Address buffer(xthread, in_bytes(ShenandoahThreadLocalData::satb_mark_queue_buffer_offset()));
810 Label L_through, L_slowpath;
811
812 // If another barrier is enabled as well, do a check for a specific barrier.
813 if (_needs_load_ref_barrier) {
814 assert(L_done == nullptr, "Should be");
815 char state_to_check = ShenandoahHeap::MARKING;
816 patchable_jump_if_not_gc_state(masm, state_to_check, _tmp1, _tmp2, &L_through);
817 }
818
819 // Fast-path: put object into buffer.
820 // If buffer is already full, go slow.
821 __ ld(_tmp1, index);
822 __ beqz(_tmp1, L_slowpath);
823 __ subi(_tmp1, _tmp1, wordSize);
824 __ sd(_tmp1, index);
825 __ ld(_tmp2, buffer);
826
827 // Store the object in queue.
828 // If object is narrow, we need to decode it before inserting.
829 __ add(_tmp1, _tmp1, _tmp2);
830 if (_narrow) {
831 __ decode_heap_oop_not_null(_tmp2, _obj);
832 __ sd(_tmp2, Address(_tmp1));
833 } else {
834 __ sd(_obj, Address(_tmp1));
835 }
836
837 // Fast-path exits here.
838 if (L_done != nullptr) {
839 __ j(*L_done);
840 } else {
841 __ j(L_through);
842 }
843
844 // Slow-path: call runtime to handle.
845 __ bind(L_slowpath);
846
847 {
848 SaveLiveRegisters slr(&masm, this);
849
850 // Go to runtime and handle the rest there.
851 __ mv(c_rarg0, _obj);
852 __ la(ra, RuntimeAddress(keepalive_runtime_entry_addr()));
853 __ jalr(ra);
854 }
855 if (L_done != nullptr) {
856 __ j(*L_done);
857 } else {
858 __ bind(L_through);
859 }
860 }
861
862 void ShenandoahBarrierStubC2::lrb(MacroAssembler& masm) {
863 Label L_slow;
864
865 // If another barrier is enabled as well, do a check for a specific barrier.
866 if (_needs_keep_alive_barrier) {
867 char state_to_check = ShenandoahHeap::HAS_FORWARDED | (_needs_load_ref_weak_barrier ? ShenandoahHeap::WEAK_ROOTS : 0);
868 patchable_jump_if_not_gc_state(masm, state_to_check, _tmp1, _tmp2, continuation());
869 }
870
871 // If weak references are being processed, weak/phantom loads need to go slow,
872 // regardless of their cset status.
873 if (_needs_load_ref_weak_barrier) {
874 char state_to_check = ShenandoahHeap::WEAK_ROOTS;
875 patchable_jump_if_gc_state(masm, state_to_check, _tmp1, _tmp2, &L_slow);
876 }
877
878 // Cset-check. Fall-through to slow if in collection set.
879 if (_narrow) {
880 __ decode_heap_oop_not_null(_tmp2, _obj);
881 } else {
882 __ mv(_tmp2, _obj);
883 }
884
885 if (AOTCodeCache::is_on_for_dump()) {
886 __ lwu(_tmp1, ExternalAddress(AOTRuntimeConstants::grain_shift_address()));
887 __ srl(_tmp2, _tmp2, _tmp1);
888 __ ld(_tmp1, ExternalAddress(AOTRuntimeConstants::cset_base_address()));
889 } else {
890 __ mv(_tmp1, ShenandoahHeap::in_cset_fast_test_addr());
891 __ srli(_tmp2, _tmp2, ShenandoahHeapRegion::region_size_bytes_shift_jint());
892 }
893 __ add(_tmp1, _tmp1, _tmp2);
894 __ lbu(_tmp1, Address(_tmp1, 0));
895 maybe_far_jump_if_zero(masm, _tmp1);
896
897 // Slow path
898 __ bind(L_slow);
899
900 // Obj is the result, need to temporarily stop preserving it.
901 bool is_obj_preserved = is_preserved(_obj);
902 if (is_obj_preserved) {
903 dont_preserve(_obj);
904 }
905 {
906 SaveLiveRegisters slr(&masm, this);
907
908 // Shuffle in the arguments. The end result should be:
909 // c_rarg0 <- obj
910 // c_rarg1 <- lea(addr)
911 if (c_rarg0 == _obj) {
912 __ la(c_rarg1, _addr);
913 } else if (c_rarg1 == _obj) {
914 __ mv(_tmp1, c_rarg1);
915 __ la(c_rarg1, _addr);
916 __ mv(c_rarg0, _tmp1);
917 } else {
918 assert_different_registers(c_rarg1, _obj);
919 __ la(c_rarg1, _addr);
920 __ mv(c_rarg0, _obj);
921 }
922
923 // Go to runtime and handle the rest there.
924 __ la(ra, RuntimeAddress(lrb_runtime_entry_addr()));
925 __ jalr(ra);
926
927 // Save the result where needed. Narrow entries return narrowOop (32 bits)
928 // we need to zero the upper 32 bits of x10.
929 if (_narrow) {
930 __ zext(_obj, x10, 32);
931 } else {
932 __ mv(_obj, x10);
933 }
934 }
935 if (is_obj_preserved) {
936 preserve(_obj);
937 }
938
939 __ j(*continuation());
940 }
941
942 int ShenandoahBarrierStubC2::available_gp_registers() {
943 Unimplemented(); // Not used
944 return 0;
945 }
946
947 bool ShenandoahBarrierStubC2::is_special_register(Register r) {
948 Unimplemented(); // Not used
949 return true;
950 }
951
952 void ShenandoahBarrierStubC2::post_init() {
953 // Do nothing.
954 }
955
956 #endif // COMPILER2