1 /*
2 * Copyright (c) 2026, Oracle and/or its affiliates. All rights reserved.
3 * Copyright (c) 2018, 2020, Red Hat, Inc. All rights reserved.
4 * Copyright (c) 2020, 2021, Huawei Technologies Co., Ltd. All rights reserved.
5 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
6 *
7 * This code is free software; you can redistribute it and/or modify it
8 * under the terms of the GNU General Public License version 2 only, as
9 * published by the Free Software Foundation.
10 *
11 * This code is distributed in the hope that it will be useful, but WITHOUT
12 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
13 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
14 * version 2 for more details (a copy is included in the LICENSE file that
15 * accompanied this code).
16 *
17 * You should have received a copy of the GNU General Public License version
18 * 2 along with this work; if not, write to the Free Software Foundation,
19 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
20 *
21 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
22 * or visit www.oracle.com if you need additional information or have any
23 * questions.
24 *
25 */
26
27 #include "code/aotCodeCache.hpp"
28 #include "gc/shenandoah/heuristics/shenandoahHeuristics.hpp"
29 #include "gc/shenandoah/mode/shenandoahMode.hpp"
30 #include "gc/shenandoah/shenandoahBarrierSet.hpp"
31 #include "gc/shenandoah/shenandoahBarrierSetAssembler.hpp"
32 #include "gc/shenandoah/shenandoahHeap.inline.hpp"
33 #include "gc/shenandoah/shenandoahHeapRegion.hpp"
34 #include "gc/shenandoah/shenandoahRuntime.hpp"
35 #include "gc/shenandoah/shenandoahThreadLocalData.hpp"
36 #include "interpreter/interp_masm.hpp"
37 #include "interpreter/interpreter.hpp"
38 #include "runtime/javaThread.hpp"
39 #include "runtime/sharedRuntime.hpp"
40 #ifdef COMPILER1
41 #include "c1/c1_LIRAssembler.hpp"
42 #include "c1/c1_MacroAssembler.hpp"
43 #include "gc/shenandoah/c1/shenandoahBarrierSetC1.hpp"
44 #endif
45 #ifdef COMPILER2
46 #include "gc/shenandoah/c2/shenandoahBarrierSetC2.hpp"
47 #include "opto/output.hpp"
48 #endif
49
50 #define __ masm->
51
52 void ShenandoahBarrierSetAssembler::arraycopy_prologue(MacroAssembler* masm, DecoratorSet decorators, bool is_oop,
53 Register src, Register dst, Register count, RegSet saved_regs) {
54 if (is_oop) {
55 bool dest_uninitialized = (decorators & IS_DEST_UNINITIALIZED) != 0;
56 if ((ShenandoahSATBBarrier && !dest_uninitialized) || ShenandoahLoadRefBarrier) {
57
58 Label done;
59
60 // Avoid calling runtime if count == 0
61 __ beqz(count, done);
62
63 // Is GC active?
64 Address gc_state(xthread, in_bytes(ShenandoahThreadLocalData::gc_state_offset()));
65 assert_different_registers(src, dst, count, t0);
66
67 assert(!saved_regs.contains(t0), "Sanity: about to clobber t0");
68
69 __ lbu(t0, gc_state);
70 if (ShenandoahSATBBarrier && dest_uninitialized) {
71 __ test_bit(t0, t0, ShenandoahHeap::HAS_FORWARDED_BITPOS);
72 __ beqz(t0, done);
73 } else {
74 __ andi(t0, t0, ShenandoahHeap::HAS_FORWARDED | ShenandoahHeap::MARKING);
75 __ beqz(t0, done);
76 }
77
78 __ push_call_clobbered_registers();
79 // If arguments are not in proper places, shuffle them.
80 // Doing this via the stack is the most straight-forward way to avoid
81 // accidentally smashing any register.
82 if (c_rarg0 != src || c_rarg1 != dst || c_rarg2 != count) {
83 __ push_reg(RegSet::of(src), sp);
84 __ push_reg(RegSet::of(dst), sp);
85 __ push_reg(RegSet::of(count), sp);
86 __ pop_reg(RegSet::of(c_rarg2), sp);
87 __ pop_reg(RegSet::of(c_rarg1), sp);
88 __ pop_reg(RegSet::of(c_rarg0), sp);
89 }
90 address target = nullptr;
91 if (UseCompressedOops) {
92 target = CAST_FROM_FN_PTR(address, ShenandoahRuntime::arraycopy_barrier_narrow_oop);
93 } else {
94 target = CAST_FROM_FN_PTR(address, ShenandoahRuntime::arraycopy_barrier_oop);
95 }
96 __ call_VM_leaf(target, 3);
97 __ pop_call_clobbered_registers();
98 __ bind(done);
99 }
100 }
101 }
102
103 void ShenandoahBarrierSetAssembler::arraycopy_epilogue(MacroAssembler* masm, DecoratorSet decorators, bool is_oop,
104 Register start, Register count, Register tmp) {
105 if (ShenandoahCardBarrier && is_oop) {
106 gen_write_ref_array_post_barrier(masm, decorators, start, count, tmp);
107 }
108 }
109
110 void ShenandoahBarrierSetAssembler::satb_barrier(MacroAssembler* masm,
111 Register obj,
112 Register pre_val,
113 Register thread,
114 Register tmp1,
115 Register tmp2) {
116 assert(ShenandoahSATBBarrier, "Should be checked by caller");
117 assert(thread == xthread, "must be");
118
119 Label done;
120 Label runtime;
121
122 assert_different_registers(obj, pre_val, tmp1, tmp2);
123 assert(pre_val != noreg && tmp1 != noreg && tmp2 != noreg, "expecting a register");
124
125 Address index(thread, in_bytes(ShenandoahThreadLocalData::satb_mark_queue_index_offset()));
126 Address buffer(thread, in_bytes(ShenandoahThreadLocalData::satb_mark_queue_buffer_offset()));
127
128 // Is marking active?
129 Address gc_state(xthread, in_bytes(ShenandoahThreadLocalData::gc_state_offset()));
130 __ lbu(t1, gc_state);
131 __ test_bit(t1, t1, ShenandoahHeap::MARKING_BITPOS);
132 __ beqz(t1, done);
133
134 // Do we need to load the previous value?
135 if (obj != noreg) {
136 if (UseCompressedOops) {
137 __ lwu(pre_val, Address(obj, 0));
138 __ decode_heap_oop(pre_val);
139 } else {
140 __ ld(pre_val, Address(obj, 0));
141 }
142 }
143
144 // Is the previous value null?
145 __ beqz(pre_val, done);
146
147 // Can we store original value in the thread's buffer?
148 // Is index == 0?
149 // (The index field is typed as size_t.)
150 __ ld(tmp1, index); // tmp := *index_adr
151 __ beqz(tmp1, runtime); // tmp == 0? If yes, goto runtime
152
153 __ subi(tmp1, tmp1, wordSize); // tmp := tmp - wordSize
154 __ sd(tmp1, index); // *index_adr := tmp
155 __ ld(tmp2, buffer);
156 __ add(tmp1, tmp1, tmp2); // tmp := tmp + *buffer_adr
157
158 // Record the previous value
159 __ sd(pre_val, Address(tmp1, 0));
160 __ j(done);
161
162 // Slow-path call.
163 __ bind(runtime);
164 __ enter();
165 __ push_call_clobbered_registers();
166 if (c_rarg0 != pre_val) {
167 __ mv(c_rarg0, pre_val);
168 }
169 // Calling with super_call_VM_leaf with c_rarg0 bypasses interpreter checks and avoids any moves.
170 __ super_call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::write_barrier_pre), c_rarg0);
171 __ pop_call_clobbered_registers();
172 __ leave();
173
174 __ bind(done);
175 }
176
177 void ShenandoahBarrierSetAssembler::load_reference_barrier(MacroAssembler* masm,
178 Register dst,
179 Address load_addr,
180 DecoratorSet decorators) {
181 assert(ShenandoahLoadRefBarrier, "Should be enabled");
182 assert(dst != t1 && load_addr.base() != t1, "need t1");
183 assert_different_registers(load_addr.base(), t0, t1);
184
185 bool is_strong = ShenandoahBarrierSet::is_strong_access(decorators);
186 bool is_weak = ShenandoahBarrierSet::is_weak_access(decorators);
187 bool is_phantom = ShenandoahBarrierSet::is_phantom_access(decorators);
188 bool is_native = ShenandoahBarrierSet::is_native_access(decorators);
189 bool is_narrow = UseCompressedOops && !is_native;
190
191 Label heap_stable, not_cset;
192 Address gc_state(xthread, in_bytes(ShenandoahThreadLocalData::gc_state_offset()));
193 __ lbu(t1, gc_state);
194
195 // Check for heap stability
196 if (is_strong) {
197 __ test_bit(t1, t1, ShenandoahHeap::HAS_FORWARDED_BITPOS);
198 __ beqz(t1, heap_stable);
199 } else {
200 Label lrb;
201 __ test_bit(t0, t1, ShenandoahHeap::WEAK_ROOTS_BITPOS);
202 __ bnez(t0, lrb);
203 __ test_bit(t0, t1, ShenandoahHeap::HAS_FORWARDED_BITPOS);
204 __ beqz(t0, heap_stable);
205 __ bind(lrb);
206 }
207
208 // use x11 for load address
209 Register result_dst = dst;
210 if (dst == x11) {
211 __ mv(t1, dst);
212 dst = t1;
213 }
214
215 // Save x10 and x11, unless it is an output register
216 RegSet saved_regs = RegSet::of(x10, x11) - result_dst;
217 __ push_reg(saved_regs, sp);
218 __ la(x11, load_addr);
219 __ mv(x10, dst);
220
221 // Test for in-cset
222 if (is_strong) {
223 #if INCLUDE_CDS
224 if (AOTCodeCache::is_on_for_dump()) {
225 __ ld(t1, ExternalAddress(AOTRuntimeConstants::cset_base_address()));
226 __ lwu(t0, ExternalAddress(AOTRuntimeConstants::grain_shift_address()));
227 __ srl(t0, x10, t0);
228 } else
229 #endif
230 {
231 __ mv(t1, ShenandoahHeap::in_cset_fast_test_addr());
232 __ srli(t0, x10, ShenandoahHeapRegion::region_size_bytes_shift_jint());
233 }
234 __ add(t1, t1, t0);
235 __ lbu(t1, Address(t1));
236 __ test_bit(t0, t1, 0);
237 __ beqz(t0, not_cset);
238 }
239
240 // Slow-path call
241 __ enter();
242 __ push_call_clobbered_registers();
243 address target = nullptr;
244 if (is_strong) {
245 if (is_narrow) {
246 target = CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_strong_narrow);
247 } else {
248 target = CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_strong);
249 }
250 } else if (is_weak) {
251 if (is_narrow) {
252 target = CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_weak_narrow);
253 } else {
254 target = CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_weak);
255 }
256 } else {
257 assert(is_phantom, "only remaining strength");
258 assert(!is_narrow, "phantom access cannot be narrow");
259 target = CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_phantom);
260 }
261 // Calling with super_call_VM_leaf with c_rarg0/1 bypasses interpreter checks and avoids any moves.
262 __ super_call_VM_leaf(target, c_rarg0, c_rarg1);
263 __ mv(t0, x10);
264 __ pop_call_clobbered_registers();
265 __ mv(x10, t0);
266 __ leave();
267
268 __ bind(not_cset);
269 __ mv(result_dst, x10);
270 __ pop_reg(saved_regs, sp);
271
272 __ bind(heap_stable);
273 }
274
275 //
276 // Arguments:
277 //
278 // Inputs:
279 // src: oop location to load from, might be clobbered
280 //
281 // Output:
282 // dst: oop loaded from src location
283 //
284 // Kill:
285 // x30 (tmp reg)
286 //
287 // Alias:
288 // dst: x30 (might use x30 as temporary output register to avoid clobbering src)
289 //
290 void ShenandoahBarrierSetAssembler::load_at(MacroAssembler* masm,
291 DecoratorSet decorators,
292 BasicType type,
293 Register dst,
294 Address src,
295 Register tmp1,
296 Register tmp2) {
297 // 1: non-reference load, no additional barrier is needed
298 if (!is_reference_type(type)) {
299 BarrierSetAssembler::load_at(masm, decorators, type, dst, src, tmp1, tmp2);
300 return;
301 }
302
303 // 2: load a reference from src location and apply LRB if needed
304 if (ShenandoahBarrierSet::need_load_reference_barrier(decorators, type)) {
305 Register result_dst = dst;
306
307 // Preserve src location for LRB
308 RegSet saved_regs;
309 if (dst == src.base()) {
310 dst = (src.base() == x28) ? x29 : x28;
311 saved_regs = RegSet::of(dst);
312 __ push_reg(saved_regs, sp);
313 }
314 assert_different_registers(dst, src.base());
315
316 BarrierSetAssembler::load_at(masm, decorators, type, dst, src, tmp1, tmp2);
317
318 load_reference_barrier(masm, dst, src, decorators);
319
320 if (dst != result_dst) {
321 __ mv(result_dst, dst);
322 dst = result_dst;
323 }
324
325 if (saved_regs.bits() != 0) {
326 __ pop_reg(saved_regs, sp);
327 }
328 } else {
329 BarrierSetAssembler::load_at(masm, decorators, type, dst, src, tmp1, tmp2);
330 }
331
332 // 3: apply keep-alive barrier if needed
333 if (ShenandoahBarrierSet::need_keep_alive_barrier(decorators, type)) {
334 satb_barrier(masm /* masm */,
335 noreg /* obj */,
336 dst /* pre_val */,
337 xthread /* thread */,
338 tmp1 /* tmp1 */,
339 tmp2 /* tmp2 */);
340 }
341 }
342
343 void ShenandoahBarrierSetAssembler::card_barrier(MacroAssembler* masm, Register obj, Register tmp1, Register tmp2) {
344 assert(ShenandoahCardBarrier, "Should have been checked by caller");
345 assert(CardTable::dirty_card_val() == 0, "must be");
346 assert_different_registers(obj, tmp1, tmp2);
347
348 __ srli(obj, obj, CardTable::card_shift());
349
350 Address curr_ct_holder_addr(xthread, in_bytes(ShenandoahThreadLocalData::card_table_offset()));
351 __ ld(tmp1, curr_ct_holder_addr);
352 __ add(tmp1, obj, tmp1);
353
354 if (UseCondCardMark) {
355 Label L_already_dirty;
356 __ lbu(tmp2, Address(tmp1));
357 __ beqz(tmp2, L_already_dirty);
358 __ sb(zr, Address(tmp1));
359 __ bind(L_already_dirty);
360 } else {
361 __ sb(zr, Address(tmp1));
362 }
363 }
364
365 void ShenandoahBarrierSetAssembler::store_at(MacroAssembler* masm, DecoratorSet decorators, BasicType type,
366 Address dst, Register val, Register tmp1, Register tmp2, Register tmp3) {
367 // 1: non-reference types require no barriers
368 if (!is_reference_type(type)) {
369 BarrierSetAssembler::store_at(masm, decorators, type, dst, val, tmp1, tmp2, tmp3);
370 return;
371 }
372
373 // Flatten object address right away for simplicity: likely needed by barriers
374 if (dst.offset() == 0) {
375 if (dst.base() != tmp3) {
376 __ mv(tmp3, dst.base());
377 }
378 } else {
379 __ la(tmp3, dst);
380 }
381
382 // 2: pre-barrier: SATB needs the previous value
383 if (ShenandoahBarrierSet::need_satb_barrier(decorators, type)) {
384 satb_barrier(masm,
385 tmp3 /* obj */,
386 tmp2 /* pre_val */,
387 xthread /* thread */,
388 tmp1 /* tmp */,
389 t0 /* tmp2 */);
390 }
391
392 // Store!
393 BarrierSetAssembler::store_at(masm, decorators, type, Address(tmp3, 0), val, noreg, noreg, noreg);
394
395 // 3: post-barrier: card barrier needs store address
396 bool storing_non_null = (val != noreg);
397 if (ShenandoahBarrierSet::need_card_barrier(decorators, type) && storing_non_null) {
398 card_barrier(masm, tmp3, tmp1, tmp2);
399 }
400 }
401
402 void ShenandoahBarrierSetAssembler::try_resolve_jobject_in_native(MacroAssembler* masm, Register jni_env,
403 Register obj, Register tmp, Label& slowpath) {
404 Label done;
405 // Resolve jobject
406 BarrierSetAssembler::try_resolve_jobject_in_native(masm, jni_env, obj, tmp, slowpath);
407
408 // Check for null.
409 __ beqz(obj, done);
410
411 assert(obj != t1, "need t1");
412 Address gc_state(jni_env, ShenandoahThreadLocalData::gc_state_offset() - JavaThread::jni_environment_offset());
413 __ lbu(t1, gc_state);
414
415 // Check for heap in evacuation phase
416 __ test_bit(t0, t1, ShenandoahHeap::EVACUATION_BITPOS);
417 __ bnez(t0, slowpath);
418
419 __ bind(done);
420 }
421
422 void ShenandoahBarrierSetAssembler::try_peek_weak_handle_in_nmethod(MacroAssembler *masm, Register weak_handle,
423 Register obj, Register tmp, Label& slow_path) {
424 assert_different_registers(weak_handle, tmp, noreg);
425 assert_different_registers(obj, tmp, noreg);
426
427
428 Label done;
429
430 // Peek weak handle using the standard implementation.
431 BarrierSetAssembler::try_peek_weak_handle_in_nmethod(masm, weak_handle, obj, tmp, slow_path);
432
433 // Check if the reference is null, and if it is, take the fast path.
434 __ beqz(obj, done);
435
436 Address gc_state(xthread, ShenandoahThreadLocalData::gc_state_offset());
437 __ lbu(tmp, gc_state);
438
439 // Check if the heap is under weak-reference/roots processing, in
440 // which case we need to take the slow path.
441 __ test_bit(tmp, tmp, ShenandoahHeap::WEAK_ROOTS_BITPOS);
442 __ bnez(tmp, slow_path);
443 __ bind(done);
444 }
445
446 void ShenandoahBarrierSetAssembler::check_oop(MacroAssembler* masm, Register obj, Register tmp1, Register tmp2, Label& L_error) {
447 assert_different_registers(obj, tmp1, tmp2);
448 // Check if the oop is in the right area of memory
449 #if INCLUDE_CDS
450 if (AOTCodeCache::is_on_for_dump()) {
451 __ ld(tmp2, ExternalAddress(AOTRuntimeConstants::verify_oop_mask_address()));
452 __ andr(tmp1, obj, tmp2);
453 __ ld(tmp2, ExternalAddress(AOTRuntimeConstants::verify_oop_bits_address()));
454 } else
455 #endif
456 {
457 __ mv(tmp2, (intptr_t) Universe::verify_oop_mask());
458 __ andr(tmp1, obj, tmp2);
459 __ mv(tmp2, (intptr_t) Universe::verify_oop_bits());
460 }
461
462 // Compare tmp1 and tmp2.
463 __ bne(tmp1, tmp2, L_error);
464
465 // This routine is sometimes called before applying GC barriers.
466 // With +COH, loading the klass may end up loading forwarding pointer instead.
467 Label L_skip;
468 if (UseCompactObjectHeaders) {
469 Address gc_state(xthread, ShenandoahThreadLocalData::gc_state_offset());
470 __ lbu(tmp1, gc_state);
471 __ test_bit(tmp1, tmp1, ShenandoahHeap::HAS_FORWARDED_BITPOS);
472 __ bnez(tmp1, L_skip);
473 }
474
475 // Make sure klass is 'reasonable', which is not zero.
476 __ load_narrow_klass(tmp1, obj);
477 __ beqz(tmp1, L_error);
478
479 __ bind(L_skip);
480 }
481
482 void ShenandoahBarrierSetAssembler::gen_write_ref_array_post_barrier(MacroAssembler* masm, DecoratorSet decorators,
483 Register start, Register count, Register tmp) {
484 assert(ShenandoahCardBarrier, "Did you mean to enable ShenandoahCardBarrier?");
485
486 Label L_loop, L_done;
487 const Register end = count;
488
489 // Zero count? Nothing to do.
490 __ beqz(count, L_done);
491
492 // end = start + count << LogBytesPerHeapOop
493 // last element address to make inclusive
494 __ shadd(end, count, start, tmp, LogBytesPerHeapOop);
495 __ subi(end, end, BytesPerHeapOop);
496 __ srli(start, start, CardTable::card_shift());
497 __ srli(end, end, CardTable::card_shift());
498
499 // number of bytes to copy
500 __ sub(count, end, start);
501
502 Address curr_ct_holder_addr(xthread, in_bytes(ShenandoahThreadLocalData::card_table_offset()));
503 __ ld(tmp, curr_ct_holder_addr);
504 __ add(start, start, tmp);
505
506 __ bind(L_loop);
507 __ add(tmp, start, count);
508 __ sb(zr, Address(tmp));
509 __ subi(count, count, 1);
510 __ bgez(count, L_loop);
511 __ bind(L_done);
512 }
513
514 #undef __
515
516 #ifdef COMPILER1
517
518 #define __ ce->masm()->
519
520 void ShenandoahBarrierSetAssembler::keepalive_barrier_c1_stub(LIR_Assembler* ce, ShenandoahKeepaliveBarrierStub* stub) {
521 __ bind(*stub->entry());
522
523 ShenandoahBarrierSetC1* bs = (ShenandoahBarrierSetC1*)BarrierSet::barrier_set()->barrier_set_c1();
524
525 Register obj = stub->obj()->as_register();
526
527 if (stub->do_load()) {
528 ce->mem2reg(stub->addr(), stub->obj(), T_OBJECT, lir_patch_none, nullptr, false /* wide */);
529 }
530 __ beqz(obj, *stub->continuation(), /* is_far */ true);
531
532 ce->store_parameter(obj, 0);
533 __ far_call(RuntimeAddress(bs->keepalive_barrier_stub()));
534 __ j(*stub->continuation());
535 }
536
537 void ShenandoahBarrierSetAssembler::load_reference_barrier_c1_stub(LIR_Assembler* ce, ShenandoahLoadReferenceBarrierStub* stub) {
538 __ bind(*stub->entry());
539
540 ShenandoahBarrierSetC1* bs = (ShenandoahBarrierSetC1*) BarrierSet::barrier_set()->barrier_set_c1();
541
542 Register obj = stub->obj()->as_register();
543 Register addr = stub->addr()->as_pointer_register();
544 Register slow_result = stub->slow_result()->as_register();
545 assert_different_registers(obj, addr, slow_result);
546 assert(slow_result == x10, "C1 must know about our slow call result register");
547
548 ce->store_parameter(obj, 0);
549 ce->store_parameter(addr, 1);
550 __ far_call(RuntimeAddress(bs->load_reference_barrier_stub(stub->decorators())));
551 if (obj != slow_result) {
552 __ mv(obj, slow_result);
553 }
554
555 __ j(*stub->continuation());
556 }
557
558 #undef __
559
560 #define __ sasm->
561
562 void ShenandoahBarrierSetAssembler::keepalive_barrier_c1_runtime_stub(StubAssembler* sasm) {
563 __ prologue("shenandoah_keepalive_barrier", false);
564 const Register tmp_obj = x10;
565 const Register tmp1 = x11;
566 const Register tmp2 = x12;
567 __ push_reg(RegSet::of(tmp1, tmp2, tmp_obj), sp);
568 __ load_parameter(0, tmp_obj);
569 satb_barrier(sasm, noreg, tmp_obj, xthread, tmp1, tmp2);
570 __ pop_reg(RegSet::of(tmp1, tmp2, tmp_obj), sp);
571 __ epilogue();
572 }
573
574 void ShenandoahBarrierSetAssembler::load_reference_barrier_c1_runtime_stub(StubAssembler* sasm, DecoratorSet decorators) {
575 __ prologue("shenandoah_load_reference_barrier", false);
576 const Register tmp_obj = x10;
577 const Register tmp_addr = x11;
578 __ push_reg(RegSet::of(tmp_addr), sp);
579 __ load_parameter(0, tmp_obj);
580 __ load_parameter(1, tmp_addr);
581 load_reference_barrier(sasm, tmp_obj, Address(tmp_addr, 0), decorators);
582 __ pop_reg(RegSet::of(tmp_addr), sp);
583 __ epilogue();
584 }
585
586 #undef __
587
588 #endif // COMPILER1
589
590 #ifdef COMPILER2
591
592 #undef __
593 #define __ masm->
594
595 void ShenandoahBarrierSetAssembler::load_c2(const MachNode* node, MacroAssembler* masm, Register dst, Address src, Register tmp1, Register tmp2, bool is_narrow) {
596 // Do the actual load. This load is the candidate for implicit null check, and MUST come first.
597 if (is_narrow) {
598 __ lwu(dst, src);
599 } else {
600 __ ld(dst, src);
601 }
602
603 ShenandoahBarrierStubC2::load_post(masm, node, dst, src, tmp1, tmp2, is_narrow);
604 }
605
606 void ShenandoahBarrierSetAssembler::store_c2(const MachNode* node, MacroAssembler* masm, Address dst, bool dst_narrow,
607 Register src, bool src_narrow, Register tmp1, Register tmp2, Register tmp3) {
608
609 ShenandoahBarrierStubC2::store_pre(masm, node, dst, tmp1, tmp2, tmp3, dst_narrow);
610
611 // Do the actual store
612 if (dst_narrow) {
613 if (!src_narrow) {
614 // Need to encode into tmp, because we cannot clobber src.
615 assert(tmp1 != noreg, "need temp register");
616 if ((node->barrier_data() & ShenandoahBitNotNull) == 0) {
617 __ encode_heap_oop(tmp1, src);
618 } else {
619 __ encode_heap_oop_not_null(tmp1, src);
620 }
621 src = tmp1;
622 }
623 __ sw(src, dst);
624 } else {
625 __ sd(src, dst);
626 }
627
628 ShenandoahBarrierStubC2::store_post(masm, node, dst, tmp2, tmp3);
629 }
630
631 void ShenandoahBarrierSetAssembler::compare_and_set_c2(const MachNode* node, MacroAssembler* masm, Register res, Register addr,
632 Register oldval, Register newval, Register tmp1, Register tmp2, Register tmp3, bool exchange, bool narrow, bool is_acquire) {
633 const Assembler::Aqrl acquire = is_acquire ? Assembler::aq : Assembler::relaxed;
634 const Assembler::Aqrl release = Assembler::rl;
635 const Assembler::operand_size size = narrow ? Assembler::uint32 : Assembler::int64;
636
637 ShenandoahBarrierStubC2::load_store_pre(masm, node, Address(addr), tmp1, tmp2, tmp3, narrow);
638
639 // CAS!
640 __ cmpxchg(addr, oldval, newval, size, acquire, release, /* result */ res, !exchange /* result_as_bool */);
641
642 ShenandoahBarrierStubC2::load_store_post(masm, node, Address(addr, 0), tmp2, tmp3);
643 }
644
645 void ShenandoahBarrierSetAssembler::get_and_set_c2(const MachNode* node, MacroAssembler* masm, Register preval,
646 Register newval, Register addr, Register tmp1, Register tmp2, Register tmp3, bool is_acquire) {
647 const bool is_narrow = node->bottom_type()->isa_narrowoop();
648
649 ShenandoahBarrierStubC2::load_store_pre(masm, node, Address(addr, 0), tmp1, tmp2, tmp3, is_narrow);
650
651 if (is_narrow) {
652 if (is_acquire) {
653 __ atomic_xchgalwu(preval, newval, addr);
654 } else {
655 __ atomic_xchgwu(preval, newval, addr);
656 }
657 } else {
658 if (is_acquire) {
659 __ atomic_xchgal(preval, newval, addr);
660 } else {
661 __ atomic_xchg(preval, newval, addr);
662 }
663 }
664
665 ShenandoahBarrierStubC2::load_store_post(masm, node, Address(addr, 0), tmp2, tmp3);
666 }
667
668 #undef __
669 #define __ masm.
670
671 void ShenandoahBarrierStubC2::cardtable(MacroAssembler& masm, Address address, Register tmp1, Register tmp2) {
672 assert(CardTable::dirty_card_val() == 0, "must be");
673 Assembler::InlineSkippedInstructionsCounter skip_counter(&masm);
674
675 // tmp1 = card table base (holder)
676 Address curr_ct_holder_addr(xthread, in_bytes(ShenandoahThreadLocalData::card_table_offset()));
677 __ ld(tmp1, curr_ct_holder_addr);
678
679 // tmp1 = effective address
680 __ la(tmp2, address);
681
682 // tmp2 = &card_table[ addr >> CardTable::card_shift() ] ; card index
683 __ srli(tmp2, tmp2, CardTable::card_shift());
684 __ add(tmp2, tmp2, tmp1);
685
686 if (UseCondCardMark) {
687 Label L_already_dirty;
688 __ lbu(tmp1, Address(tmp2));
689 __ beqz(tmp1, L_already_dirty);
690 __ sb(zr, Address(tmp2));
691 __ bind(L_already_dirty);
692 } else {
693 __ sb(zr, Address(tmp2));
694 }
695 }
696
697 void ShenandoahBarrierStubC2::enter_if_gc_state(MacroAssembler& masm, const char test_state, Register tmp) {
698 Assembler::InlineSkippedInstructionsCounter skip_counter(&masm);
699
700 Address gc_state_fast(xthread, in_bytes(ShenandoahThreadLocalData::gc_state_fast_array_offset(test_state)));
701 __ lbu(tmp, gc_state_fast);
702 __ beqz(tmp, *continuation());
703 __ j(*entry());
704
705 // This is were the slowpath stub will return to or the code above will
706 // jump to if the checks are false
707 __ bind(*continuation());
708 }
709
710 void ShenandoahBarrierStubC2::emit_code(MacroAssembler& masm) {
711 Assembler::InlineSkippedInstructionsCounter skip_counter(&masm);
712 assert(_needs_keep_alive_barrier || _needs_load_ref_barrier, "Why are you here?");
713
714 __ bind(*entry());
715
716 // If we need to load ourselves, do it here.
717 if (_do_load) {
718 if (_narrow) {
719 __ lwu(_obj, _addr);
720 } else {
721 __ ld(_obj, _addr);
722 }
723 }
724
725 // If the object is null, there is no point in applying barriers.
726 maybe_far_jump_if_zero(masm, _obj);
727
728 // We need to make sure that loads done by callers survive across slow-path calls.
729 // For self-loads, we need to care about the case when both KA and LRB are enabled (rare).
730 bool needs_both_barriers = _needs_keep_alive_barrier && _needs_load_ref_barrier;
731 if (!_do_load || needs_both_barriers) {
732 preserve(_obj);
733 }
734
735 // Go for barriers. Barriers can return straight to continuation, as long
736 // as another barrier is not needed and we can reach the fastpath.
737 if (needs_both_barriers) {
738 keepalive(masm, nullptr);
739 lrb(masm);
740 } else if (_needs_keep_alive_barrier) {
741 keepalive(masm, continuation());
742 } else if (_needs_load_ref_barrier) {
743 lrb(masm);
744 } else {
745 ShouldNotReachHere();
746 }
747 }
748
749 void ShenandoahBarrierStubC2::maybe_far_jump_if_zero(MacroAssembler& masm, Register reg) {
750 Label L_short_jump;
751 __ bnez(reg, L_short_jump);
752 __ j(*continuation());
753 __ bind(L_short_jump);
754 }
755
756 void ShenandoahBarrierStubC2::keepalive(MacroAssembler& masm, Label* L_done) {
757 Address index(xthread, in_bytes(ShenandoahThreadLocalData::satb_mark_queue_index_offset()));
758 Address buffer(xthread, in_bytes(ShenandoahThreadLocalData::satb_mark_queue_buffer_offset()));
759 Label L_through, L_slowpath;
760
761 // If another barrier is enabled as well, do a runtime check for a specific barrier.
762 if (_needs_load_ref_barrier) {
763 assert(L_done == nullptr, "L_done is always null when _needs_load_ref_barrier is true");
764 Address gc_state_fast(xthread, in_bytes(ShenandoahThreadLocalData::gc_state_fast_array_offset(ShenandoahHeap::MARKING)));
765 __ lbu(_tmp1, gc_state_fast);
766 __ beqz(_tmp1, L_through);
767 }
768
769 // Fast-path: put object into buffer.
770 // If buffer is already full, go slow.
771 __ ld(_tmp1, index);
772 __ beqz(_tmp1, L_slowpath);
773 __ subi(_tmp1, _tmp1, wordSize);
774 __ sd(_tmp1, index);
775 __ ld(_tmp2, buffer);
776
777 // Store the object in queue.
778 // If object is narrow, we need to decode it before inserting.
779 __ add(_tmp1, _tmp1, _tmp2);
780 if (_narrow) {
781 __ decode_heap_oop_not_null(_tmp2, _obj);
782 __ sd(_tmp2, Address(_tmp1));
783 } else {
784 __ sd(_obj, Address(_tmp1));
785 }
786
787 // Fast-path exits here.
788 if (L_done != nullptr) {
789 __ j(*L_done);
790 } else {
791 __ j(L_through);
792 }
793
794 // Slow-path: call runtime to handle.
795 __ bind(L_slowpath);
796
797 {
798 SaveLiveRegisters slr(&masm, this);
799
800 // Go to runtime and handle the rest there.
801 __ mv(c_rarg0, _obj);
802 __ la(ra, RuntimeAddress(keepalive_runtime_entry_addr()));
803 __ jalr(ra);
804 }
805 if (L_done != nullptr) {
806 __ j(*L_done);
807 } else {
808 __ bind(L_through);
809 }
810 }
811
812 void ShenandoahBarrierStubC2::lrb(MacroAssembler& masm) {
813 Label L_slow;
814
815 // If another barrier is enabled as well, do a runtime check for a specific barrier.
816 if (_needs_keep_alive_barrier) {
817 char state_to_check = ShenandoahHeap::HAS_FORWARDED | (_needs_load_ref_weak_barrier ? ShenandoahHeap::WEAK_ROOTS : 0);
818 Address gc_state_fast(xthread, in_bytes(ShenandoahThreadLocalData::gc_state_fast_array_offset(state_to_check)));
819 __ lbu(_tmp1, gc_state_fast);
820 maybe_far_jump_if_zero(masm, _tmp1);
821 }
822
823 // If weak references are being processed, weak/phantom loads need to go slow,
824 // regardless of their cset status.
825 if (_needs_load_ref_weak_barrier) {
826 Address gc_state_fast(xthread, in_bytes(ShenandoahThreadLocalData::gc_state_fast_array_offset(ShenandoahHeap::WEAK_ROOTS)));
827 __ lbu(_tmp1, gc_state_fast);
828 __ bnez(_tmp1, L_slow);
829 }
830
831 // Cset-check. Fall-through to slow if in collection set.
832 if (_narrow) {
833 __ decode_heap_oop_not_null(_tmp2, _obj);
834 } else {
835 __ mv(_tmp2, _obj);
836 }
837
838 if (AOTCodeCache::is_on_for_dump()) {
839 __ lwu(_tmp1, ExternalAddress(AOTRuntimeConstants::grain_shift_address()));
840 __ srl(_tmp2, _tmp2, _tmp1);
841 __ ld(_tmp1, ExternalAddress(AOTRuntimeConstants::cset_base_address()));
842 } else {
843 __ mv(_tmp1, ShenandoahHeap::in_cset_fast_test_addr());
844 __ srli(_tmp2, _tmp2, ShenandoahHeapRegion::region_size_bytes_shift_jint());
845 }
846 __ add(_tmp1, _tmp1, _tmp2);
847 __ lbu(_tmp1, Address(_tmp1, 0));
848 maybe_far_jump_if_zero(masm, _tmp1);
849
850 // Slow path
851 __ bind(L_slow);
852
853 // Obj is the result, need to temporarily stop preserving it.
854 bool is_obj_preserved = is_preserved(_obj);
855 if (is_obj_preserved) {
856 dont_preserve(_obj);
857 }
858 {
859 SaveLiveRegisters slr(&masm, this);
860
861 // Shuffle in the arguments. The end result should be:
862 // c_rarg0 <- obj
863 // c_rarg1 <- lea(addr)
864 if (c_rarg0 == _obj) {
865 __ la(c_rarg1, _addr);
866 } else if (c_rarg1 == _obj) {
867 __ mv(_tmp1, c_rarg1);
868 __ la(c_rarg1, _addr);
869 __ mv(c_rarg0, _tmp1);
870 } else {
871 assert_different_registers(c_rarg1, _obj);
872 __ la(c_rarg1, _addr);
873 __ mv(c_rarg0, _obj);
874 }
875
876 // Go to runtime and handle the rest there.
877 __ la(ra, RuntimeAddress(lrb_runtime_entry_addr()));
878 __ jalr(ra);
879
880 // Save the result where needed. Narrow entries return narrowOop (32 bits)
881 // we need to zero the upper 32 bits of x10.
882 if (_narrow) {
883 __ zext(_obj, x10, 32);
884 } else {
885 __ mv(_obj, x10);
886 }
887 }
888 if (is_obj_preserved) {
889 preserve(_obj);
890 }
891
892 __ j(*continuation());
893 }
894
895 int ShenandoahBarrierStubC2::available_gp_registers() {
896 Unimplemented(); // Not used
897 return 0;
898 }
899
900 bool ShenandoahBarrierStubC2::is_special_register(Register r) {
901 Unimplemented(); // Not used
902 return true;
903 }
904
905 void ShenandoahBarrierStubC2::post_init() {
906 // Do nothing.
907 }
908
909 #endif // COMPILER2