1 /*
2 * Copyright (c) 2026, Oracle and/or its affiliates. All rights reserved.
3 * Copyright (c) 2018, 2020, Red Hat, Inc. All rights reserved.
4 * Copyright (c) 2020, 2021, Huawei Technologies Co., Ltd. All rights reserved.
5 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
6 *
7 * This code is free software; you can redistribute it and/or modify it
8 * under the terms of the GNU General Public License version 2 only, as
9 * published by the Free Software Foundation.
10 *
11 * This code is distributed in the hope that it will be useful, but WITHOUT
12 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
13 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
14 * version 2 for more details (a copy is included in the LICENSE file that
15 * accompanied this code).
16 *
17 * You should have received a copy of the GNU General Public License version
18 * 2 along with this work; if not, write to the Free Software Foundation,
19 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
20 *
21 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
22 * or visit www.oracle.com if you need additional information or have any
23 * questions.
24 *
25 */
26
27 #include "gc/shenandoah/heuristics/shenandoahHeuristics.hpp"
28 #include "gc/shenandoah/mode/shenandoahMode.hpp"
29 #include "gc/shenandoah/shenandoahBarrierSet.hpp"
30 #include "gc/shenandoah/shenandoahBarrierSetAssembler.hpp"
31 #include "gc/shenandoah/shenandoahHeap.inline.hpp"
32 #include "gc/shenandoah/shenandoahHeapRegion.hpp"
33 #include "gc/shenandoah/shenandoahNMethod.inline.hpp"
34 #include "gc/shenandoah/shenandoahRuntime.hpp"
35 #include "gc/shenandoah/shenandoahThreadLocalData.hpp"
36 #include "interpreter/interp_masm.hpp"
37 #include "interpreter/interpreter.hpp"
38 #include "nativeInst_riscv.hpp"
39 #include "runtime/javaThread.hpp"
40 #include "runtime/sharedRuntime.hpp"
41 #ifdef COMPILER1
42 #include "c1/c1_LIRAssembler.hpp"
43 #include "c1/c1_MacroAssembler.hpp"
44 #include "gc/shenandoah/c1/shenandoahBarrierSetC1.hpp"
45 #endif
46 #ifdef COMPILER2
47 #include "gc/shenandoah/c2/shenandoahBarrierSetC2.hpp"
48 #include "opto/output.hpp"
49 #endif
50
51 #define __ masm->
52
53 void ShenandoahBarrierSetAssembler::arraycopy_prologue(MacroAssembler* masm, DecoratorSet decorators, bool is_oop,
54 Register src, Register dst, Register count, RegSet saved_regs) {
55 if (is_oop) {
56 bool dest_uninitialized = (decorators & IS_DEST_UNINITIALIZED) != 0;
57 if ((ShenandoahSATBBarrier && !dest_uninitialized) || ShenandoahLoadRefBarrier) {
58
59 Label done;
60
61 // Avoid calling runtime if count == 0
62 __ beqz(count, done);
63
64 // Is GC active?
65 Address gc_state(xthread, in_bytes(ShenandoahThreadLocalData::gc_state_offset()));
66 assert_different_registers(src, dst, count, t0);
67
68 assert(!saved_regs.contains(t0), "Sanity: about to clobber t0");
69
70 __ lbu(t0, gc_state);
71 if (ShenandoahSATBBarrier && dest_uninitialized) {
72 __ test_bit(t0, t0, ShenandoahHeap::HAS_FORWARDED_BITPOS);
73 __ beqz(t0, done);
74 } else {
75 __ andi(t0, t0, ShenandoahHeap::HAS_FORWARDED | ShenandoahHeap::MARKING);
76 __ beqz(t0, done);
77 }
78
79 __ push_call_clobbered_registers();
80 // If arguments are not in proper places, shuffle them.
81 // Doing this via the stack is the most straight-forward way to avoid
82 // accidentally smashing any register.
83 if (c_rarg0 != src || c_rarg1 != dst || c_rarg2 != count) {
84 __ push_reg(RegSet::of(src), sp);
85 __ push_reg(RegSet::of(dst), sp);
86 __ push_reg(RegSet::of(count), sp);
87 __ pop_reg(RegSet::of(c_rarg2), sp);
88 __ pop_reg(RegSet::of(c_rarg1), sp);
89 __ pop_reg(RegSet::of(c_rarg0), sp);
90 }
91 address target = nullptr;
92 if (UseCompressedOops) {
93 target = CAST_FROM_FN_PTR(address, ShenandoahRuntime::arraycopy_barrier_narrow_oop);
94 } else {
95 target = CAST_FROM_FN_PTR(address, ShenandoahRuntime::arraycopy_barrier_oop);
96 }
97 __ call_VM_leaf(target, 3);
98 __ pop_call_clobbered_registers();
99 __ bind(done);
100 }
101 }
102 }
103
104 void ShenandoahBarrierSetAssembler::arraycopy_epilogue(MacroAssembler* masm, DecoratorSet decorators, bool is_oop,
105 Register start, Register count, Register tmp) {
106 if (ShenandoahCardBarrier && is_oop) {
107 gen_write_ref_array_post_barrier(masm, decorators, start, count, tmp);
108 }
109 }
110
111 void ShenandoahBarrierSetAssembler::satb_barrier(MacroAssembler* masm,
112 Register obj,
113 Register pre_val,
114 Register thread,
115 Register tmp1,
116 Register tmp2) {
117 assert(ShenandoahSATBBarrier, "Should be checked by caller");
118 assert(thread == xthread, "must be");
119
120 Label done;
121 Label runtime;
122
123 assert_different_registers(obj, pre_val, tmp1, tmp2);
124 assert(pre_val != noreg && tmp1 != noreg && tmp2 != noreg, "expecting a register");
125
126 Address index(thread, in_bytes(ShenandoahThreadLocalData::satb_mark_queue_index_offset()));
127 Address buffer(thread, in_bytes(ShenandoahThreadLocalData::satb_mark_queue_buffer_offset()));
128
129 // Is marking active?
130 Address gc_state(xthread, in_bytes(ShenandoahThreadLocalData::gc_state_offset()));
131 __ lbu(t1, gc_state);
132 __ test_bit(t1, t1, ShenandoahHeap::MARKING_BITPOS);
133 __ beqz(t1, done);
134
135 // Do we need to load the previous value?
136 if (obj != noreg) {
137 if (UseCompressedOops) {
138 __ lwu(pre_val, Address(obj, 0));
139 __ decode_heap_oop(pre_val);
140 } else {
141 __ ld(pre_val, Address(obj, 0));
142 }
143 }
144
145 // Is the previous value null?
146 __ beqz(pre_val, done);
147
148 // Can we store original value in the thread's buffer?
149 // Is index == 0?
150 // (The index field is typed as size_t.)
151 __ ld(tmp1, index); // tmp := *index_adr
152 __ beqz(tmp1, runtime); // tmp == 0? If yes, goto runtime
153
154 __ subi(tmp1, tmp1, wordSize); // tmp := tmp - wordSize
155 __ sd(tmp1, index); // *index_adr := tmp
156 __ ld(tmp2, buffer);
157 __ add(tmp1, tmp1, tmp2); // tmp := tmp + *buffer_adr
158
159 // Record the previous value
160 __ sd(pre_val, Address(tmp1, 0));
161 __ j(done);
162
163 // Slow-path call.
164 __ bind(runtime);
165 __ enter();
166 __ push_call_clobbered_registers();
167 if (c_rarg0 != pre_val) {
168 __ mv(c_rarg0, pre_val);
169 }
170 // Calling with super_call_VM_leaf with c_rarg0 bypasses interpreter checks and avoids any moves.
171 __ super_call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::write_barrier_pre), c_rarg0);
172 __ pop_call_clobbered_registers();
173 __ leave();
174
175 __ bind(done);
176 }
177
178 void ShenandoahBarrierSetAssembler::load_reference_barrier(MacroAssembler* masm,
179 Register dst,
180 Address load_addr,
181 DecoratorSet decorators) {
182 assert(ShenandoahLoadRefBarrier, "Should be enabled");
183 assert(dst != t1 && load_addr.base() != t1, "need t1");
184 assert_different_registers(load_addr.base(), t0, t1);
185
186 bool is_strong = ShenandoahBarrierSet::is_strong_access(decorators);
187 bool is_weak = ShenandoahBarrierSet::is_weak_access(decorators);
188 bool is_phantom = ShenandoahBarrierSet::is_phantom_access(decorators);
189 bool is_native = ShenandoahBarrierSet::is_native_access(decorators);
190 bool is_narrow = UseCompressedOops && !is_native;
191
192 Label heap_stable, not_cset;
193 Address gc_state(xthread, in_bytes(ShenandoahThreadLocalData::gc_state_offset()));
194 __ lbu(t1, gc_state);
195
196 // Check for heap stability
197 if (is_strong) {
198 __ test_bit(t1, t1, ShenandoahHeap::HAS_FORWARDED_BITPOS);
199 __ beqz(t1, heap_stable);
200 } else {
201 Label lrb;
202 __ test_bit(t0, t1, ShenandoahHeap::WEAK_ROOTS_BITPOS);
203 __ bnez(t0, lrb);
204 __ test_bit(t0, t1, ShenandoahHeap::HAS_FORWARDED_BITPOS);
205 __ beqz(t0, heap_stable);
206 __ bind(lrb);
207 }
208
209 // use x11 for load address
210 Register result_dst = dst;
211 if (dst == x11) {
212 __ mv(t1, dst);
213 dst = t1;
214 }
215
216 // Save x10 and x11, unless it is an output register
217 RegSet saved_regs = RegSet::of(x10, x11) - result_dst;
218 __ push_reg(saved_regs, sp);
219 __ la(x11, load_addr);
220 __ mv(x10, dst);
221
222 // Test for in-cset
223 if (is_strong) {
224 if (AOTCodeCache::is_on_for_dump()) {
225 __ ld(t1, ExternalAddress(AOTRuntimeConstants::cset_base_address()));
226 __ lwu(t0, ExternalAddress(AOTRuntimeConstants::grain_shift_address()));
227 __ srl(t0, x10, t0);
228 } else {
229 __ mv(t1, ShenandoahHeap::in_cset_fast_test_addr());
230 __ srli(t0, x10, ShenandoahHeapRegion::region_size_bytes_shift_jint());
231 }
232 __ add(t1, t1, t0);
233 __ lbu(t1, Address(t1));
234 __ test_bit(t0, t1, 0);
235 __ beqz(t0, not_cset);
236 }
237
238 // Slow-path call
239 __ enter();
240 __ push_call_clobbered_registers();
241 address target = nullptr;
242 if (is_strong) {
243 if (is_narrow) {
244 target = CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_strong_narrow);
245 } else {
246 target = CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_strong);
247 }
248 } else if (is_weak) {
249 if (is_narrow) {
250 target = CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_weak_narrow);
251 } else {
252 target = CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_weak);
253 }
254 } else {
255 assert(is_phantom, "only remaining strength");
256 assert(!is_narrow, "phantom access cannot be narrow");
257 target = CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_phantom);
258 }
259 // Calling with super_call_VM_leaf with c_rarg0/1 bypasses interpreter checks and avoids any moves.
260 __ super_call_VM_leaf(target, c_rarg0, c_rarg1);
261 __ mv(t0, x10);
262 __ pop_call_clobbered_registers();
263 __ mv(x10, t0);
264 __ leave();
265
266 __ bind(not_cset);
267 __ mv(result_dst, x10);
268 __ pop_reg(saved_regs, sp);
269
270 __ bind(heap_stable);
271 }
272
273 //
274 // Arguments:
275 //
276 // Inputs:
277 // src: oop location to load from, might be clobbered
278 //
279 // Output:
280 // dst: oop loaded from src location
281 //
282 // Kill:
283 // x30 (tmp reg)
284 //
285 // Alias:
286 // dst: x30 (might use x30 as temporary output register to avoid clobbering src)
287 //
288 void ShenandoahBarrierSetAssembler::load_at(MacroAssembler* masm,
289 DecoratorSet decorators,
290 BasicType type,
291 Register dst,
292 Address src,
293 Register tmp1,
294 Register tmp2) {
295 // 1: non-reference load, no additional barrier is needed
296 if (!is_reference_type(type)) {
297 BarrierSetAssembler::load_at(masm, decorators, type, dst, src, tmp1, tmp2);
298 return;
299 }
300
301 // 2: load a reference from src location and apply LRB if needed
302 if (ShenandoahBarrierSet::need_load_reference_barrier(decorators, type)) {
303 Register result_dst = dst;
304
305 // Preserve src location for LRB
306 RegSet saved_regs;
307 if (dst == src.base()) {
308 dst = (src.base() == x28) ? x29 : x28;
309 saved_regs = RegSet::of(dst);
310 __ push_reg(saved_regs, sp);
311 }
312 assert_different_registers(dst, src.base());
313
314 BarrierSetAssembler::load_at(masm, decorators, type, dst, src, tmp1, tmp2);
315
316 load_reference_barrier(masm, dst, src, decorators);
317
318 if (dst != result_dst) {
319 __ mv(result_dst, dst);
320 dst = result_dst;
321 }
322
323 if (saved_regs.bits() != 0) {
324 __ pop_reg(saved_regs, sp);
325 }
326 } else {
327 BarrierSetAssembler::load_at(masm, decorators, type, dst, src, tmp1, tmp2);
328 }
329
330 // 3: apply keep-alive barrier if needed
331 if (ShenandoahBarrierSet::need_keep_alive_barrier(decorators, type)) {
332 satb_barrier(masm /* masm */,
333 noreg /* obj */,
334 dst /* pre_val */,
335 xthread /* thread */,
336 tmp1 /* tmp1 */,
337 tmp2 /* tmp2 */);
338 }
339 }
340
341 void ShenandoahBarrierSetAssembler::card_barrier(MacroAssembler* masm, Register obj, Register tmp1, Register tmp2) {
342 assert(ShenandoahCardBarrier, "Should have been checked by caller");
343 assert(CardTable::dirty_card_val() == 0, "must be");
344 assert_different_registers(obj, tmp1, tmp2);
345
346 __ srli(obj, obj, CardTable::card_shift());
347
348 Address curr_ct_holder_addr(xthread, in_bytes(ShenandoahThreadLocalData::card_table_offset()));
349 __ ld(tmp1, curr_ct_holder_addr);
350 __ add(tmp1, obj, tmp1);
351
352 if (UseCondCardMark) {
353 Label L_already_dirty;
354 __ lbu(tmp2, Address(tmp1));
355 __ beqz(tmp2, L_already_dirty);
356 __ sb(zr, Address(tmp1));
357 __ bind(L_already_dirty);
358 } else {
359 __ sb(zr, Address(tmp1));
360 }
361 }
362
363 void ShenandoahBarrierSetAssembler::store_at(MacroAssembler* masm, DecoratorSet decorators, BasicType type,
364 Address dst, Register val, Register tmp1, Register tmp2, Register tmp3) {
365 // 1: non-reference types require no barriers
366 if (!is_reference_type(type)) {
367 BarrierSetAssembler::store_at(masm, decorators, type, dst, val, tmp1, tmp2, tmp3);
368 return;
369 }
370
371 // Flatten object address right away for simplicity: likely needed by barriers
372 if (dst.offset() == 0) {
373 if (dst.base() != tmp3) {
374 __ mv(tmp3, dst.base());
375 }
376 } else {
377 __ la(tmp3, dst);
378 }
379
380 // 2: pre-barrier: SATB needs the previous value
381 if (ShenandoahBarrierSet::need_satb_barrier(decorators, type)) {
382 satb_barrier(masm,
383 tmp3 /* obj */,
384 tmp2 /* pre_val */,
385 xthread /* thread */,
386 tmp1 /* tmp */,
387 t0 /* tmp2 */);
388 }
389
390 // Store!
391 BarrierSetAssembler::store_at(masm, decorators, type, Address(tmp3, 0), val, noreg, noreg, noreg);
392
393 // 3: post-barrier: card barrier needs store address
394 bool storing_non_null = (val != noreg);
395 if (ShenandoahBarrierSet::need_card_barrier(decorators, type) && storing_non_null) {
396 card_barrier(masm, tmp3, tmp1, tmp2);
397 }
398 }
399
400 void ShenandoahBarrierSetAssembler::try_resolve_jobject_in_native(MacroAssembler* masm, Register jni_env,
401 Register obj, Register tmp, Label& slowpath) {
402 Label done;
403 // Resolve jobject
404 BarrierSetAssembler::try_resolve_jobject_in_native(masm, jni_env, obj, tmp, slowpath);
405
406 // Check for null.
407 __ beqz(obj, done);
408
409 assert(obj != t1, "need t1");
410 Address gc_state(jni_env, ShenandoahThreadLocalData::gc_state_offset() - JavaThread::jni_environment_offset());
411 __ lbu(t1, gc_state);
412
413 // Check for heap in evacuation phase
414 __ test_bit(t0, t1, ShenandoahHeap::EVACUATION_BITPOS);
415 __ bnez(t0, slowpath);
416
417 __ bind(done);
418 }
419
420 void ShenandoahBarrierSetAssembler::try_peek_weak_handle_in_nmethod(MacroAssembler *masm, Register weak_handle,
421 Register obj, Register tmp, Label& slow_path) {
422 assert_different_registers(weak_handle, tmp, noreg);
423 assert_different_registers(obj, tmp, noreg);
424
425
426 Label done;
427
428 // Peek weak handle using the standard implementation.
429 BarrierSetAssembler::try_peek_weak_handle_in_nmethod(masm, weak_handle, obj, tmp, slow_path);
430
431 // Check if the reference is null, and if it is, take the fast path.
432 __ beqz(obj, done);
433
434 Address gc_state(xthread, ShenandoahThreadLocalData::gc_state_offset());
435 __ lbu(tmp, gc_state);
436
437 // Check if the heap is under weak-reference/roots processing, in
438 // which case we need to take the slow path.
439 __ test_bit(tmp, tmp, ShenandoahHeap::WEAK_ROOTS_BITPOS);
440 __ bnez(tmp, slow_path);
441 __ bind(done);
442 }
443
444 void ShenandoahBarrierSetAssembler::check_oop(MacroAssembler* masm, Register obj, Register tmp1, Register tmp2, Label& L_error) {
445 // Check if the oop is in the right area of memory
446 __ mv(tmp2, (intptr_t) Universe::verify_oop_mask());
447 __ andr(tmp1, obj, tmp2);
448 __ mv(tmp2, (intptr_t) Universe::verify_oop_bits());
449
450 // Compare tmp1 and tmp2.
451 __ bne(tmp1, tmp2, L_error);
452
453 // This routine is sometimes called before applying GC barriers.
454 // With +COH, loading the klass may end up loading forwarding pointer instead.
455 Label L_skip;
456 if (UseCompactObjectHeaders) {
457 Address gc_state(xthread, ShenandoahThreadLocalData::gc_state_offset());
458 __ lbu(tmp1, gc_state);
459 __ test_bit(tmp1, tmp1, ShenandoahHeap::HAS_FORWARDED_BITPOS);
460 __ bnez(tmp1, L_skip);
461 }
462
463 // Make sure klass is 'reasonable', which is not zero.
464 __ load_narrow_klass(tmp1, obj);
465 __ beqz(tmp1, L_error);
466
467 __ bind(L_skip);
468 }
469
470 void ShenandoahBarrierSetAssembler::gen_write_ref_array_post_barrier(MacroAssembler* masm, DecoratorSet decorators,
471 Register start, Register count, Register tmp) {
472 assert(ShenandoahCardBarrier, "Did you mean to enable ShenandoahCardBarrier?");
473
474 Label L_loop, L_done;
475 const Register end = count;
476
477 // Zero count? Nothing to do.
478 __ beqz(count, L_done);
479
480 // end = start + count << LogBytesPerHeapOop
481 // last element address to make inclusive
482 __ shadd(end, count, start, tmp, LogBytesPerHeapOop);
483 __ subi(end, end, BytesPerHeapOop);
484 __ srli(start, start, CardTable::card_shift());
485 __ srli(end, end, CardTable::card_shift());
486
487 // number of bytes to copy
488 __ sub(count, end, start);
489
490 Address curr_ct_holder_addr(xthread, in_bytes(ShenandoahThreadLocalData::card_table_offset()));
491 __ ld(tmp, curr_ct_holder_addr);
492 __ add(start, start, tmp);
493
494 __ bind(L_loop);
495 __ add(tmp, start, count);
496 __ sb(zr, Address(tmp));
497 __ subi(count, count, 1);
498 __ bgez(count, L_loop);
499 __ bind(L_done);
500 }
501
502 #undef __
503
504 address ShenandoahBarrierSetAssembler::parse_jump_address(address pc) {
505 NativeInstruction* ni = nativeInstruction_at(pc);
506 assert(ni->is_jump(), "Must be a jump");
507 NativeJump* jmp = nativeJump_at(pc);
508 return jmp->jump_destination();
509 }
510
511 static uint32_t encode_patchable_nop() {
512 return 0x00000013;
513 }
514
515 static uint32_t encode_patchable_jump(address pc, address target_pc) {
516 int32_t disp = checked_cast<int32_t>((intptr_t)target_pc - (intptr_t)pc);
517 return Assembler::encode_jal(x0, disp);
518 }
519
520 void ShenandoahBarrierSetAssembler::insert_patchable_nop(address pc) {
521 *((uint32_t*)pc) = encode_patchable_nop();
522 assert(nativeInstruction_at(pc)->is_nop(), "Sanity");
523 }
524
525 void ShenandoahBarrierSetAssembler::insert_patchable_jump(address pc, address target_pc) {
526 *((uint32_t*)pc) = encode_patchable_jump(pc, target_pc);
527 }
528
529 bool ShenandoahBarrierSetAssembler::is_patchable_nop(address pc) {
530 return *((uint32_t*)pc) == encode_patchable_nop();
531 }
532
533 bool ShenandoahBarrierSetAssembler::is_patchable_jump(address pc, address target_pc) {
534 return *((uint32_t*)pc) == encode_patchable_jump(pc, target_pc);
535 }
536
537 #ifdef COMPILER1
538
539 #define __ ce->masm()->
540
541 void ShenandoahBarrierSetAssembler::keepalive_barrier_c1_stub(LIR_Assembler* ce, ShenandoahKeepaliveBarrierStub* stub) {
542 __ bind(*stub->entry());
543
544 ShenandoahBarrierSetC1* bs = (ShenandoahBarrierSetC1*)BarrierSet::barrier_set()->barrier_set_c1();
545
546 Register obj = stub->obj()->as_register();
547
548 if (stub->do_load()) {
549 ce->mem2reg(stub->addr(), stub->obj(), T_OBJECT, lir_patch_none, nullptr, false /* wide */);
550 }
551 __ beqz(obj, *stub->continuation(), /* is_far */ true);
552
553 ce->store_parameter(obj, 0);
554 __ far_call(RuntimeAddress(bs->keepalive_barrier_stub()));
555 __ j(*stub->continuation());
556 }
557
558 void ShenandoahBarrierSetAssembler::load_reference_barrier_c1_stub(LIR_Assembler* ce, ShenandoahLoadReferenceBarrierStub* stub) {
559 __ bind(*stub->entry());
560
561 ShenandoahBarrierSetC1* bs = (ShenandoahBarrierSetC1*) BarrierSet::barrier_set()->barrier_set_c1();
562
563 Register obj = stub->obj()->as_register();
564 Register addr = stub->addr()->as_pointer_register();
565 Register slow_result = stub->slow_result()->as_register();
566 assert_different_registers(obj, addr, slow_result);
567 assert(slow_result == x10, "C1 must know about our slow call result register");
568
569 ce->store_parameter(obj, 0);
570 ce->store_parameter(addr, 1);
571 __ far_call(RuntimeAddress(bs->load_reference_barrier_stub(stub->decorators())));
572 if (obj != slow_result) {
573 __ mv(obj, slow_result);
574 }
575
576 __ j(*stub->continuation());
577 }
578
579 #undef __
580
581 #define __ sasm->
582
583 void ShenandoahBarrierSetAssembler::keepalive_barrier_c1_runtime_stub(StubAssembler* sasm) {
584 __ prologue("shenandoah_keepalive_barrier", false);
585 const Register tmp_obj = x10;
586 const Register tmp1 = x11;
587 const Register tmp2 = x12;
588 __ push_reg(RegSet::of(tmp1, tmp2, tmp_obj), sp);
589 __ load_parameter(0, tmp_obj);
590 satb_barrier(sasm, noreg, tmp_obj, xthread, tmp1, tmp2);
591 __ pop_reg(RegSet::of(tmp1, tmp2, tmp_obj), sp);
592 __ epilogue();
593 }
594
595 void ShenandoahBarrierSetAssembler::load_reference_barrier_c1_runtime_stub(StubAssembler* sasm, DecoratorSet decorators) {
596 __ prologue("shenandoah_load_reference_barrier", false);
597 const Register tmp_obj = x10;
598 const Register tmp_addr = x11;
599 __ push_reg(RegSet::of(tmp_addr), sp);
600 __ load_parameter(0, tmp_obj);
601 __ load_parameter(1, tmp_addr);
602 load_reference_barrier(sasm, tmp_obj, Address(tmp_addr, 0), decorators);
603 __ pop_reg(RegSet::of(tmp_addr), sp);
604 __ epilogue();
605 }
606
607 #undef __
608
609 #endif // COMPILER1
610
611 #ifdef COMPILER2
612
613 #undef __
614 #define __ masm->
615
616 void ShenandoahBarrierSetAssembler::load_c2(const MachNode* node, MacroAssembler* masm, Register dst, Address src, Register tmp1, Register tmp2, bool is_narrow) {
617 // Do the actual load. This load is the candidate for implicit null check, and MUST come first.
618 if (is_narrow) {
619 __ lwu(dst, src);
620 } else {
621 __ ld(dst, src);
622 }
623
624 ShenandoahBarrierStubC2::load_post(masm, node, dst, src, tmp1, tmp2, is_narrow);
625 }
626
627 void ShenandoahBarrierSetAssembler::store_c2(const MachNode* node, MacroAssembler* masm, Address dst, bool dst_narrow,
628 Register src, bool src_narrow, Register tmp1, Register tmp2, Register tmp3) {
629
630 ShenandoahBarrierStubC2::store_pre(masm, node, dst, tmp1, tmp2, tmp3, dst_narrow);
631
632 // Do the actual store
633 if (dst_narrow) {
634 if (!src_narrow) {
635 // Need to encode into tmp, because we cannot clobber src.
636 assert(tmp1 != noreg, "need temp register");
637 if ((node->barrier_data() & ShenandoahBitNotNull) == 0) {
638 __ encode_heap_oop(tmp1, src);
639 } else {
640 __ encode_heap_oop_not_null(tmp1, src);
641 }
642 src = tmp1;
643 }
644 __ sw(src, dst);
645 } else {
646 __ sd(src, dst);
647 }
648
649 ShenandoahBarrierStubC2::store_post(masm, node, dst, tmp2, tmp3);
650 }
651
652 void ShenandoahBarrierSetAssembler::compare_and_set_c2(const MachNode* node, MacroAssembler* masm, Register res, Register addr,
653 Register oldval, Register newval, Register tmp1, Register tmp2, Register tmp3, bool exchange, bool narrow, bool is_acquire) {
654 const Assembler::Aqrl acquire = is_acquire ? Assembler::aq : Assembler::relaxed;
655 const Assembler::Aqrl release = Assembler::rl;
656 const Assembler::operand_size size = narrow ? Assembler::uint32 : Assembler::int64;
657
658 ShenandoahBarrierStubC2::load_store_pre(masm, node, Address(addr), tmp1, tmp2, tmp3, narrow);
659
660 // CAS!
661 __ cmpxchg(addr, oldval, newval, size, acquire, release, /* result */ res, !exchange /* result_as_bool */);
662
663 ShenandoahBarrierStubC2::load_store_post(masm, node, Address(addr, 0), tmp2, tmp3);
664 }
665
666 void ShenandoahBarrierSetAssembler::get_and_set_c2(const MachNode* node, MacroAssembler* masm, Register preval,
667 Register newval, Register addr, Register tmp1, Register tmp2, Register tmp3, bool is_acquire) {
668 const bool is_narrow = node->bottom_type()->isa_narrowoop();
669
670 ShenandoahBarrierStubC2::load_store_pre(masm, node, Address(addr, 0), tmp1, tmp2, tmp3, is_narrow);
671
672 if (is_narrow) {
673 if (is_acquire) {
674 __ atomic_xchgalwu(preval, newval, addr);
675 } else {
676 __ atomic_xchgwu(preval, newval, addr);
677 }
678 } else {
679 if (is_acquire) {
680 __ atomic_xchgal(preval, newval, addr);
681 } else {
682 __ atomic_xchg(preval, newval, addr);
683 }
684 }
685
686 ShenandoahBarrierStubC2::load_store_post(masm, node, Address(addr, 0), tmp2, tmp3);
687 }
688
689 #undef __
690 #define __ masm.
691
692 void ShenandoahBarrierStubC2::cardtable(MacroAssembler& masm, Address address, Register tmp1, Register tmp2) {
693 assert(CardTable::dirty_card_val() == 0, "must be");
694 Assembler::InlineSkippedInstructionsCounter skip_counter(&masm);
695
696 // tmp1 = card table base (holder)
697 Address curr_ct_holder_addr(xthread, in_bytes(ShenandoahThreadLocalData::card_table_offset()));
698 __ ld(tmp1, curr_ct_holder_addr);
699
700 // tmp1 = effective address
701 __ la(tmp2, address);
702
703 // tmp2 = &card_table[ addr >> CardTable::card_shift() ] ; card index
704 __ srli(tmp2, tmp2, CardTable::card_shift());
705 __ add(tmp2, tmp2, tmp1);
706
707 if (UseCondCardMark) {
708 Label L_already_dirty;
709 __ lbu(tmp1, Address(tmp2));
710 __ beqz(tmp1, L_already_dirty);
711 __ sb(zr, Address(tmp2));
712 __ bind(L_already_dirty);
713 } else {
714 __ sb(zr, Address(tmp2));
715 }
716 }
717
718 void ShenandoahBarrierStubC2::patchable_jump(MacroAssembler& masm, const char gc_state, bool jump_when_state, Register tmp1, Register tmp2, Label* L_target) {
719 PhaseOutput* const output = Compile::current()->output();
720 if (output->in_scratch_emit_size()) {
721 // Avoid binding L_target in scratch emits.
722 // We know the patched check is exactly one incompressible instruction long.
723 Assembler::IncompressibleScope scope(&masm);
724 __ nop();
725 return;
726 }
727
728 // Emit the unconditional branch in the first version of the method.
729 // Let the rest of runtime figure out how to manage it.
730 __ relocate(patchable_barrier_Relocation::spec(ShenandoahNMethod::encode_to_reloc(gc_state, jump_when_state)));
731 __ j(*L_target);
732 }
733
734 void ShenandoahBarrierStubC2::enter_if_gc_state(MacroAssembler& masm, const char test_state, Register tmp1, Register tmp2) {
735 Assembler::InlineSkippedInstructionsCounter skip_counter(&masm);
736 patchable_jump_if_gc_state(masm, test_state, tmp1, tmp2, entry());
737 __ bind(*continuation());
738 }
739
740 void ShenandoahBarrierStubC2::emit_code(MacroAssembler& masm) {
741 Assembler::InlineSkippedInstructionsCounter skip_counter(&masm);
742 assert(_needs_keep_alive_barrier || _needs_load_ref_barrier, "Why are you here?");
743
744 __ bind(*entry());
745
746 // If we need to load ourselves, do it here.
747 if (_do_load) {
748 if (_narrow) {
749 __ lwu(_obj, _addr);
750 } else {
751 __ ld(_obj, _addr);
752 }
753 }
754
755 // If the object is null, there is no point in applying barriers.
756 maybe_far_jump_if_zero(masm, _obj);
757
758 // We need to make sure that loads done by callers survive across slow-path calls.
759 // For self-loads, we need to care about the case when both KA and LRB are enabled (rare).
760 bool needs_both_barriers = _needs_keep_alive_barrier && _needs_load_ref_barrier;
761 if (!_do_load || needs_both_barriers) {
762 preserve(_obj);
763 }
764
765 // Go for barriers. Barriers can return straight to continuation, as long
766 // as another barrier is not needed and we can reach the fastpath.
767 if (needs_both_barriers) {
768 keepalive(masm, nullptr);
769 lrb(masm);
770 } else if (_needs_keep_alive_barrier) {
771 keepalive(masm, continuation());
772 } else if (_needs_load_ref_barrier) {
773 lrb(masm);
774 } else {
775 ShouldNotReachHere();
776 }
777 }
778
779 void ShenandoahBarrierStubC2::maybe_far_jump_if_zero(MacroAssembler& masm, Register reg) {
780 Label L_short_jump;
781 __ bnez(reg, L_short_jump);
782 __ j(*continuation());
783 __ bind(L_short_jump);
784 }
785
786 void ShenandoahBarrierStubC2::keepalive(MacroAssembler& masm, Label* L_done) {
787 Address index(xthread, in_bytes(ShenandoahThreadLocalData::satb_mark_queue_index_offset()));
788 Address buffer(xthread, in_bytes(ShenandoahThreadLocalData::satb_mark_queue_buffer_offset()));
789 Label L_through, L_slowpath;
790
791 // If another barrier is enabled as well, do a check for a specific barrier.
792 if (_needs_load_ref_barrier) {
793 assert(L_done == nullptr, "Should be");
794 char state_to_check = ShenandoahHeap::MARKING;
795 patchable_jump_if_not_gc_state(masm, state_to_check, _tmp1, _tmp2, &L_through);
796 }
797
798 // Fast-path: put object into buffer.
799 // If buffer is already full, go slow.
800 __ ld(_tmp1, index);
801 __ beqz(_tmp1, L_slowpath);
802 __ subi(_tmp1, _tmp1, wordSize);
803 __ sd(_tmp1, index);
804 __ ld(_tmp2, buffer);
805
806 // Store the object in queue.
807 // If object is narrow, we need to decode it before inserting.
808 __ add(_tmp1, _tmp1, _tmp2);
809 if (_narrow) {
810 __ decode_heap_oop_not_null(_tmp2, _obj);
811 __ sd(_tmp2, Address(_tmp1));
812 } else {
813 __ sd(_obj, Address(_tmp1));
814 }
815
816 // Fast-path exits here.
817 if (L_done != nullptr) {
818 __ j(*L_done);
819 } else {
820 __ j(L_through);
821 }
822
823 // Slow-path: call runtime to handle.
824 __ bind(L_slowpath);
825
826 {
827 SaveLiveRegisters slr(&masm, this);
828
829 // Go to runtime and handle the rest there.
830 __ mv(c_rarg0, _obj);
831 __ la(ra, RuntimeAddress(keepalive_runtime_entry_addr()));
832 __ jalr(ra);
833 }
834 if (L_done != nullptr) {
835 __ j(*L_done);
836 } else {
837 __ bind(L_through);
838 }
839 }
840
841 void ShenandoahBarrierStubC2::lrb(MacroAssembler& masm) {
842 Label L_slow;
843
844 // If another barrier is enabled as well, do a check for a specific barrier.
845 if (_needs_keep_alive_barrier) {
846 char state_to_check = ShenandoahHeap::HAS_FORWARDED | (_needs_load_ref_weak_barrier ? ShenandoahHeap::WEAK_ROOTS : 0);
847 patchable_jump_if_not_gc_state(masm, state_to_check, _tmp1, _tmp2, continuation());
848 }
849
850 // If weak references are being processed, weak/phantom loads need to go slow,
851 // regardless of their cset status.
852 if (_needs_load_ref_weak_barrier) {
853 char state_to_check = ShenandoahHeap::WEAK_ROOTS;
854 patchable_jump_if_gc_state(masm, state_to_check, _tmp1, _tmp2, &L_slow);
855 }
856
857 // Cset-check. Fall-through to slow if in collection set.
858 if (_narrow) {
859 __ decode_heap_oop_not_null(_tmp2, _obj);
860 } else {
861 __ mv(_tmp2, _obj);
862 }
863
864 if (AOTCodeCache::is_on_for_dump()) {
865 __ lwu(_tmp1, ExternalAddress(AOTRuntimeConstants::grain_shift_address()));
866 __ srl(_tmp2, _tmp2, _tmp1);
867 __ ld(_tmp1, ExternalAddress(AOTRuntimeConstants::cset_base_address()));
868 } else {
869 __ mv(_tmp1, ShenandoahHeap::in_cset_fast_test_addr());
870 __ srli(_tmp2, _tmp2, ShenandoahHeapRegion::region_size_bytes_shift_jint());
871 }
872 __ add(_tmp1, _tmp1, _tmp2);
873 __ lbu(_tmp1, Address(_tmp1, 0));
874 maybe_far_jump_if_zero(masm, _tmp1);
875
876 // Slow path
877 __ bind(L_slow);
878
879 // Obj is the result, need to temporarily stop preserving it.
880 bool is_obj_preserved = is_preserved(_obj);
881 if (is_obj_preserved) {
882 dont_preserve(_obj);
883 }
884 {
885 SaveLiveRegisters slr(&masm, this);
886
887 // Shuffle in the arguments. The end result should be:
888 // c_rarg0 <- obj
889 // c_rarg1 <- lea(addr)
890 if (c_rarg0 == _obj) {
891 __ la(c_rarg1, _addr);
892 } else if (c_rarg1 == _obj) {
893 __ mv(_tmp1, c_rarg1);
894 __ la(c_rarg1, _addr);
895 __ mv(c_rarg0, _tmp1);
896 } else {
897 assert_different_registers(c_rarg1, _obj);
898 __ la(c_rarg1, _addr);
899 __ mv(c_rarg0, _obj);
900 }
901
902 // Go to runtime and handle the rest there.
903 __ la(ra, RuntimeAddress(lrb_runtime_entry_addr()));
904 __ jalr(ra);
905
906 // Save the result where needed. Narrow entries return narrowOop (32 bits)
907 // we need to zero the upper 32 bits of x10.
908 if (_narrow) {
909 __ zext(_obj, x10, 32);
910 } else {
911 __ mv(_obj, x10);
912 }
913 }
914 if (is_obj_preserved) {
915 preserve(_obj);
916 }
917
918 __ j(*continuation());
919 }
920
921 int ShenandoahBarrierStubC2::available_gp_registers() {
922 Unimplemented(); // Not used
923 return 0;
924 }
925
926 bool ShenandoahBarrierStubC2::is_special_register(Register r) {
927 Unimplemented(); // Not used
928 return true;
929 }
930
931 void ShenandoahBarrierStubC2::post_init() {
932 // Do nothing.
933 }
934
935 #endif // COMPILER2