1 /*
2 * Copyright (c) 2026, Oracle and/or its affiliates. All rights reserved.
3 * Copyright (c) 2018, 2020, Red Hat, Inc. All rights reserved.
4 * Copyright (c) 2020, 2021, Huawei Technologies Co., Ltd. All rights reserved.
5 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
6 *
7 * This code is free software; you can redistribute it and/or modify it
8 * under the terms of the GNU General Public License version 2 only, as
9 * published by the Free Software Foundation.
10 *
11 * This code is distributed in the hope that it will be useful, but WITHOUT
12 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
13 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
14 * version 2 for more details (a copy is included in the LICENSE file that
15 * accompanied this code).
16 *
17 * You should have received a copy of the GNU General Public License version
18 * 2 along with this work; if not, write to the Free Software Foundation,
19 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
20 *
21 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
22 * or visit www.oracle.com if you need additional information or have any
23 * questions.
24 *
25 */
26
27 #include "gc/shenandoah/heuristics/shenandoahHeuristics.hpp"
28 #include "gc/shenandoah/mode/shenandoahMode.hpp"
29 #include "gc/shenandoah/shenandoahBarrierSet.hpp"
30 #include "gc/shenandoah/shenandoahBarrierSetAssembler.hpp"
31 #include "gc/shenandoah/shenandoahForwarding.hpp"
32 #include "gc/shenandoah/shenandoahHeap.inline.hpp"
33 #include "gc/shenandoah/shenandoahHeapRegion.hpp"
34 #include "gc/shenandoah/shenandoahRuntime.hpp"
35 #include "gc/shenandoah/shenandoahThreadLocalData.hpp"
36 #include "interpreter/interp_masm.hpp"
37 #include "interpreter/interpreter.hpp"
38 #include "runtime/javaThread.hpp"
39 #include "runtime/sharedRuntime.hpp"
40 #ifdef COMPILER1
41 #include "c1/c1_LIRAssembler.hpp"
42 #include "c1/c1_MacroAssembler.hpp"
43 #include "gc/shenandoah/c1/shenandoahBarrierSetC1.hpp"
44 #endif
45
46 #define __ masm->
47
48 void ShenandoahBarrierSetAssembler::arraycopy_prologue(MacroAssembler* masm, DecoratorSet decorators, bool is_oop,
49 Register src, Register dst, Register count, RegSet saved_regs) {
50 if (is_oop) {
51 bool dest_uninitialized = (decorators & IS_DEST_UNINITIALIZED) != 0;
52 if ((ShenandoahSATBBarrier && !dest_uninitialized) || ShenandoahLoadRefBarrier) {
53
54 Label done;
55
56 // Avoid calling runtime if count == 0
57 __ beqz(count, done);
58
59 // Is GC active?
60 Address gc_state(xthread, in_bytes(ShenandoahThreadLocalData::gc_state_offset()));
61 assert_different_registers(src, dst, count, t0);
62
63 assert(!saved_regs.contains(t0), "Sanity: about to clobber t0");
64
65 __ lbu(t0, gc_state);
66 if (ShenandoahSATBBarrier && dest_uninitialized) {
67 __ test_bit(t0, t0, ShenandoahHeap::HAS_FORWARDED_BITPOS);
68 __ beqz(t0, done);
69 } else {
70 __ andi(t0, t0, ShenandoahHeap::HAS_FORWARDED | ShenandoahHeap::MARKING);
71 __ beqz(t0, done);
72 }
73
74 __ push_call_clobbered_registers();
75 // If arguments are not in proper places, shuffle them.
76 // Doing this via the stack is the most straight-forward way to avoid
77 // accidentally smashing any register.
78 if (c_rarg0 != src || c_rarg1 != dst || c_rarg2 != count) {
79 __ push_reg(RegSet::of(src), sp);
80 __ push_reg(RegSet::of(dst), sp);
81 __ push_reg(RegSet::of(count), sp);
82 __ pop_reg(RegSet::of(c_rarg2), sp);
83 __ pop_reg(RegSet::of(c_rarg1), sp);
84 __ pop_reg(RegSet::of(c_rarg0), sp);
85 }
86 address target = nullptr;
87 if (UseCompressedOops) {
88 target = CAST_FROM_FN_PTR(address, ShenandoahRuntime::arraycopy_barrier_narrow_oop);
89 } else {
90 target = CAST_FROM_FN_PTR(address, ShenandoahRuntime::arraycopy_barrier_oop);
91 }
92 __ call_VM_leaf(target, 3);
93 __ pop_call_clobbered_registers();
94 __ bind(done);
95 }
96 }
97 }
98
99 void ShenandoahBarrierSetAssembler::arraycopy_epilogue(MacroAssembler* masm, DecoratorSet decorators, bool is_oop,
100 Register start, Register count, Register tmp) {
101 if (ShenandoahCardBarrier && is_oop) {
102 gen_write_ref_array_post_barrier(masm, decorators, start, count, tmp);
103 }
104 }
105
106 void ShenandoahBarrierSetAssembler::satb_barrier(MacroAssembler* masm,
107 Register obj,
108 Register pre_val,
109 Register thread,
110 Register tmp1,
111 Register tmp2) {
112 assert(ShenandoahSATBBarrier, "Should be checked by caller");
113 assert(thread == xthread, "must be");
114
115 Label done;
116 Label runtime;
117
118 assert_different_registers(obj, pre_val, tmp1, tmp2);
119 assert(pre_val != noreg && tmp1 != noreg && tmp2 != noreg, "expecting a register");
120
121 Address index(thread, in_bytes(ShenandoahThreadLocalData::satb_mark_queue_index_offset()));
122 Address buffer(thread, in_bytes(ShenandoahThreadLocalData::satb_mark_queue_buffer_offset()));
123
124 // Is marking active?
125 Address gc_state(xthread, in_bytes(ShenandoahThreadLocalData::gc_state_offset()));
126 __ lbu(t1, gc_state);
127 __ test_bit(t1, t1, ShenandoahHeap::MARKING_BITPOS);
128 __ beqz(t1, done);
129
130 // Do we need to load the previous value?
131 if (obj != noreg) {
132 if (UseCompressedOops) {
133 __ lwu(pre_val, Address(obj, 0));
134 __ decode_heap_oop(pre_val);
135 } else {
136 __ ld(pre_val, Address(obj, 0));
137 }
138 }
139
140 // Is the previous value null?
141 __ beqz(pre_val, done);
142
143 // Can we store original value in the thread's buffer?
144 // Is index == 0?
145 // (The index field is typed as size_t.)
146 __ ld(tmp1, index); // tmp := *index_adr
147 __ beqz(tmp1, runtime); // tmp == 0? If yes, goto runtime
148
149 __ subi(tmp1, tmp1, wordSize); // tmp := tmp - wordSize
150 __ sd(tmp1, index); // *index_adr := tmp
151 __ ld(tmp2, buffer);
152 __ add(tmp1, tmp1, tmp2); // tmp := tmp + *buffer_adr
153
154 // Record the previous value
155 __ sd(pre_val, Address(tmp1, 0));
156 __ j(done);
157
158 // Slow-path call.
159 __ bind(runtime);
160 __ enter();
161 __ push_call_clobbered_registers();
162 if (c_rarg0 != pre_val) {
163 __ mv(c_rarg0, pre_val);
164 }
165 // Calling with super_call_VM_leaf with c_rarg0 bypasses interpreter checks and avoids any moves.
166 __ super_call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::write_barrier_pre), c_rarg0);
167 __ pop_call_clobbered_registers();
168 __ leave();
169
170 __ bind(done);
171 }
172
173 void ShenandoahBarrierSetAssembler::resolve_forward_pointer(MacroAssembler* masm, Register dst, Register tmp) {
174 assert(ShenandoahLoadRefBarrier || ShenandoahCASBarrier, "Should be enabled");
175
176 Label is_null;
177 __ beqz(dst, is_null);
178 resolve_forward_pointer_not_null(masm, dst, tmp);
179 __ bind(is_null);
180 }
181
182 // IMPORTANT: This must preserve all registers, even t0 and t1, except those explicitly
183 // passed in.
184 void ShenandoahBarrierSetAssembler::resolve_forward_pointer_not_null(MacroAssembler* masm, Register dst, Register tmp) {
185 assert(ShenandoahLoadRefBarrier || ShenandoahCASBarrier, "Should be enabled");
186 // The below loads the mark word, checks if the lowest two bits are
187 // set, and if so, clear the lowest two bits and copy the result
188 // to dst. Otherwise it leaves dst alone.
189 // Implementing this is surprisingly awkward. I do it here by:
190 // - Inverting the mark word
191 // - Test lowest two bits == 0
192 // - If so, set the lowest two bits
193 // - Invert the result back, and copy to dst
194 RegSet saved_regs = RegSet::of(t2);
195 bool borrow_reg = (tmp == noreg);
196 if (borrow_reg) {
197 // No free registers available. Make one useful.
198 tmp = t0;
199 if (tmp == dst) {
200 tmp = t1;
201 }
202 saved_regs += RegSet::of(tmp);
203 }
204
205 assert_different_registers(tmp, dst, t2);
206 __ push_reg(saved_regs, sp);
207
208 Label done;
209 __ ld(tmp, Address(dst, oopDesc::mark_offset_in_bytes()));
210 __ xori(tmp, tmp, -1); // eon with 0 is equivalent to XOR with -1
211 __ andi(t2, tmp, markWord::lock_mask_in_place);
212 __ bnez(t2, done);
213 __ ori(tmp, tmp, markWord::marked_value);
214 __ xori(dst, tmp, -1); // eon with 0 is equivalent to XOR with -1
215 __ bind(done);
216
217 __ pop_reg(saved_regs, sp);
218 }
219
220 void ShenandoahBarrierSetAssembler::load_reference_barrier(MacroAssembler* masm,
221 Register dst,
222 Address load_addr,
223 DecoratorSet decorators) {
224 assert(ShenandoahLoadRefBarrier, "Should be enabled");
225 assert(dst != t1 && load_addr.base() != t1, "need t1");
226 assert_different_registers(load_addr.base(), t0, t1);
227
228 bool is_strong = ShenandoahBarrierSet::is_strong_access(decorators);
229 bool is_weak = ShenandoahBarrierSet::is_weak_access(decorators);
230 bool is_phantom = ShenandoahBarrierSet::is_phantom_access(decorators);
231 bool is_native = ShenandoahBarrierSet::is_native_access(decorators);
232 bool is_narrow = UseCompressedOops && !is_native;
233
234 Label heap_stable, not_cset;
235 Address gc_state(xthread, in_bytes(ShenandoahThreadLocalData::gc_state_offset()));
236 __ lbu(t1, gc_state);
237
238 // Check for heap stability
239 if (is_strong) {
240 __ test_bit(t1, t1, ShenandoahHeap::HAS_FORWARDED_BITPOS);
241 __ beqz(t1, heap_stable);
242 } else {
243 Label lrb;
244 __ test_bit(t0, t1, ShenandoahHeap::WEAK_ROOTS_BITPOS);
245 __ bnez(t0, lrb);
246 __ test_bit(t0, t1, ShenandoahHeap::HAS_FORWARDED_BITPOS);
247 __ beqz(t0, heap_stable);
248 __ bind(lrb);
249 }
250
251 // use x11 for load address
252 Register result_dst = dst;
253 if (dst == x11) {
254 __ mv(t1, dst);
255 dst = t1;
256 }
257
258 // Save x10 and x11, unless it is an output register
259 RegSet saved_regs = RegSet::of(x10, x11) - result_dst;
260 __ push_reg(saved_regs, sp);
261 __ la(x11, load_addr);
262 __ mv(x10, dst);
263
264 // Test for in-cset
265 if (is_strong) {
266 __ mv(t1, ShenandoahHeap::in_cset_fast_test_addr());
267 __ srli(t0, x10, ShenandoahHeapRegion::region_size_bytes_shift_jint());
268 __ add(t1, t1, t0);
269 __ lbu(t1, Address(t1));
270 __ test_bit(t0, t1, 0);
271 __ beqz(t0, not_cset);
272 }
273
274 // Slow-path call
275 __ enter();
276 __ push_call_clobbered_registers();
277 address target = nullptr;
278 if (is_strong) {
279 if (is_narrow) {
280 target = CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_strong_narrow);
281 } else {
282 target = CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_strong);
283 }
284 } else if (is_weak) {
285 if (is_narrow) {
286 target = CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_weak_narrow);
287 } else {
288 target = CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_weak);
289 }
290 } else {
291 assert(is_phantom, "only remaining strength");
292 assert(!is_narrow, "phantom access cannot be narrow");
293 target = CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_phantom);
294 }
295 // Calling with super_call_VM_leaf with c_rarg0/1 bypasses interpreter checks and avoids any moves.
296 __ super_call_VM_leaf(target, c_rarg0, c_rarg1);
297 __ mv(t0, x10);
298 __ pop_call_clobbered_registers();
299 __ mv(x10, t0);
300 __ leave();
301
302 __ bind(not_cset);
303 __ mv(result_dst, x10);
304 __ pop_reg(saved_regs, sp);
305
306 __ bind(heap_stable);
307 }
308
309 //
310 // Arguments:
311 //
312 // Inputs:
313 // src: oop location to load from, might be clobbered
314 //
315 // Output:
316 // dst: oop loaded from src location
317 //
318 // Kill:
319 // x30 (tmp reg)
320 //
321 // Alias:
322 // dst: x30 (might use x30 as temporary output register to avoid clobbering src)
323 //
324 void ShenandoahBarrierSetAssembler::load_at(MacroAssembler* masm,
325 DecoratorSet decorators,
326 BasicType type,
327 Register dst,
328 Address src,
329 Register tmp1,
330 Register tmp2) {
331 // 1: non-reference load, no additional barrier is needed
332 if (!is_reference_type(type)) {
333 BarrierSetAssembler::load_at(masm, decorators, type, dst, src, tmp1, tmp2);
334 return;
335 }
336
337 // 2: load a reference from src location and apply LRB if needed
338 if (ShenandoahBarrierSet::need_load_reference_barrier(decorators, type)) {
339 Register result_dst = dst;
340
341 // Preserve src location for LRB
342 RegSet saved_regs;
343 if (dst == src.base()) {
344 dst = (src.base() == x28) ? x29 : x28;
345 saved_regs = RegSet::of(dst);
346 __ push_reg(saved_regs, sp);
347 }
348 assert_different_registers(dst, src.base());
349
350 BarrierSetAssembler::load_at(masm, decorators, type, dst, src, tmp1, tmp2);
351
352 load_reference_barrier(masm, dst, src, decorators);
353
354 if (dst != result_dst) {
355 __ mv(result_dst, dst);
356 dst = result_dst;
357 }
358
359 if (saved_regs.bits() != 0) {
360 __ pop_reg(saved_regs, sp);
361 }
362 } else {
363 BarrierSetAssembler::load_at(masm, decorators, type, dst, src, tmp1, tmp2);
364 }
365
366 // 3: apply keep-alive barrier if needed
367 if (ShenandoahBarrierSet::need_keep_alive_barrier(decorators, type)) {
368 satb_barrier(masm /* masm */,
369 noreg /* obj */,
370 dst /* pre_val */,
371 xthread /* thread */,
372 tmp1 /* tmp1 */,
373 tmp2 /* tmp2 */);
374 }
375 }
376
377 void ShenandoahBarrierSetAssembler::card_barrier(MacroAssembler* masm, Register obj) {
378 assert(ShenandoahCardBarrier, "Should have been checked by caller");
379
380 __ srli(obj, obj, CardTable::card_shift());
381
382 assert(CardTable::dirty_card_val() == 0, "must be");
383
384 Address curr_ct_holder_addr(xthread, in_bytes(ShenandoahThreadLocalData::card_table_offset()));
385 __ ld(t1, curr_ct_holder_addr);
386 __ add(t1, obj, t1);
387
388 if (UseCondCardMark) {
389 Label L_already_dirty;
390 __ lbu(t0, Address(t1));
391 __ beqz(t0, L_already_dirty);
392 __ sb(zr, Address(t1));
393 __ bind(L_already_dirty);
394 } else {
395 __ sb(zr, Address(t1));
396 }
397 }
398
399 void ShenandoahBarrierSetAssembler::store_at(MacroAssembler* masm, DecoratorSet decorators, BasicType type,
400 Address dst, Register val, Register tmp1, Register tmp2, Register tmp3) {
401 // 1: non-reference types require no barriers
402 if (!is_reference_type(type)) {
403 BarrierSetAssembler::store_at(masm, decorators, type, dst, val, tmp1, tmp2, tmp3);
404 return;
405 }
406
407 // Flatten object address right away for simplicity: likely needed by barriers
408 if (dst.offset() == 0) {
409 if (dst.base() != tmp3) {
410 __ mv(tmp3, dst.base());
411 }
412 } else {
413 __ la(tmp3, dst);
414 }
415
416 // 2: pre-barrier: SATB needs the previous value
417 if (ShenandoahBarrierSet::need_satb_barrier(decorators, type)) {
418 satb_barrier(masm,
419 tmp3 /* obj */,
420 tmp2 /* pre_val */,
421 xthread /* thread */,
422 tmp1 /* tmp */,
423 t0 /* tmp2 */);
424 }
425
426 // Store!
427 BarrierSetAssembler::store_at(masm, decorators, type, Address(tmp3, 0), val, noreg, noreg, noreg);
428
429 // 3: post-barrier: card barrier needs store address
430 bool storing_non_null = (val != noreg);
431 if (ShenandoahBarrierSet::need_card_barrier(decorators, type) && storing_non_null) {
432 card_barrier(masm, tmp3);
433 }
434 }
435
436 void ShenandoahBarrierSetAssembler::try_resolve_jobject_in_native(MacroAssembler* masm, Register jni_env,
437 Register obj, Register tmp, Label& slowpath) {
438 Label done;
439 // Resolve jobject
440 BarrierSetAssembler::try_resolve_jobject_in_native(masm, jni_env, obj, tmp, slowpath);
441
442 // Check for null.
443 __ beqz(obj, done);
444
445 assert(obj != t1, "need t1");
446 Address gc_state(jni_env, ShenandoahThreadLocalData::gc_state_offset() - JavaThread::jni_environment_offset());
447 __ lbu(t1, gc_state);
448
449 // Check for heap in evacuation phase
450 __ test_bit(t0, t1, ShenandoahHeap::EVACUATION_BITPOS);
451 __ bnez(t0, slowpath);
452
453 __ bind(done);
454 }
455
456 void ShenandoahBarrierSetAssembler::try_peek_weak_handle_in_nmethod(MacroAssembler *masm, Register weak_handle,
457 Register obj, Register tmp, Label& slow_path) {
458 assert_different_registers(weak_handle, tmp, noreg);
459 assert_different_registers(obj, tmp, noreg);
460
461
462 Label done;
463
464 // Peek weak handle using the standard implementation.
465 BarrierSetAssembler::try_peek_weak_handle_in_nmethod(masm, weak_handle, obj, tmp, slow_path);
466
467 // Check if the reference is null, and if it is, take the fast path.
468 __ beqz(obj, done);
469
470 Address gc_state(xthread, ShenandoahThreadLocalData::gc_state_offset());
471 __ lbu(tmp, gc_state);
472
473 // Check if the heap is under weak-reference/roots processing, in
474 // which case we need to take the slow path.
475 __ test_bit(tmp, tmp, ShenandoahHeap::WEAK_ROOTS_BITPOS);
476 __ bnez(tmp, slow_path);
477 __ bind(done);
478 }
479
480 // Special Shenandoah CAS implementation that handles false negatives due
481 // to concurrent evacuation. The service is more complex than a
482 // traditional CAS operation because the CAS operation is intended to
483 // succeed if the reference at addr exactly matches expected or if the
484 // reference at addr holds a pointer to a from-space object that has
485 // been relocated to the location named by expected. There are two
486 // races that must be addressed:
487 // a) A parallel thread may mutate the contents of addr so that it points
488 // to a different object. In this case, the CAS operation should fail.
489 // b) A parallel thread may heal the contents of addr, replacing a
490 // from-space pointer held in addr with the to-space pointer
491 // representing the new location of the object.
492 // Upon entry to cmpxchg_oop, it is assured that new_val equals null
493 // or it refers to an object that is not being evacuated out of
494 // from-space, or it refers to the to-space version of an object that
495 // is being evacuated out of from-space.
496 //
497 // By default the value held in the result register following execution
498 // of the generated code sequence is 0 to indicate failure of CAS,
499 // non-zero to indicate success. If is_cae, the result is the value most
500 // recently fetched from addr rather than a boolean success indicator.
501 //
502 // Clobbers t0, t1
503 void ShenandoahBarrierSetAssembler::cmpxchg_oop(MacroAssembler* masm,
504 Register addr,
505 Register expected,
506 Register new_val,
507 Assembler::Aqrl acquire,
508 Assembler::Aqrl release,
509 bool is_cae,
510 Register result) {
511 bool is_narrow = UseCompressedOops;
512 Assembler::operand_size size = is_narrow ? Assembler::uint32 : Assembler::int64;
513
514 assert_different_registers(addr, expected, t0, t1);
515 assert_different_registers(addr, new_val, t0, t1);
516
517 Label retry, success, fail, done;
518
519 __ bind(retry);
520
521 // Step1: Try to CAS.
522 __ cmpxchg(addr, expected, new_val, size, acquire, release, /* result */ t1);
523
524 // If success, then we are done.
525 __ beq(expected, t1, success);
526
527 // Step2: CAS failed, check the forwarded pointer.
528 __ mv(t0, t1);
529
530 if (is_narrow) {
531 __ decode_heap_oop(t0, t0);
532 }
533 resolve_forward_pointer(masm, t0);
534
535 __ encode_heap_oop(t0, t0);
536
537 // Report failure when the forwarded oop was not expected.
538 __ bne(t0, expected, fail);
539
540 // Step 3: CAS again using the forwarded oop.
541 __ cmpxchg(addr, t1, new_val, size, acquire, release, /* result */ t0);
542
543 // Retry when failed.
544 __ bne(t0, t1, retry);
545
546 __ bind(success);
547 if (is_cae) {
548 __ mv(result, expected);
549 } else {
550 __ mv(result, 1);
551 }
552 __ j(done);
553
554 __ bind(fail);
555 if (is_cae) {
556 __ mv(result, t0);
557 } else {
558 __ mv(result, zr);
559 }
560
561 __ bind(done);
562 }
563
564 void ShenandoahBarrierSetAssembler::gen_write_ref_array_post_barrier(MacroAssembler* masm, DecoratorSet decorators,
565 Register start, Register count, Register tmp) {
566 assert(ShenandoahCardBarrier, "Did you mean to enable ShenandoahCardBarrier?");
567
568 Label L_loop, L_done;
569 const Register end = count;
570
571 // Zero count? Nothing to do.
572 __ beqz(count, L_done);
573
574 // end = start + count << LogBytesPerHeapOop
575 // last element address to make inclusive
576 __ shadd(end, count, start, tmp, LogBytesPerHeapOop);
577 __ subi(end, end, BytesPerHeapOop);
578 __ srli(start, start, CardTable::card_shift());
579 __ srli(end, end, CardTable::card_shift());
580
581 // number of bytes to copy
582 __ sub(count, end, start);
583
584 Address curr_ct_holder_addr(xthread, in_bytes(ShenandoahThreadLocalData::card_table_offset()));
585 __ ld(tmp, curr_ct_holder_addr);
586 __ add(start, start, tmp);
587
588 __ bind(L_loop);
589 __ add(tmp, start, count);
590 __ sb(zr, Address(tmp));
591 __ subi(count, count, 1);
592 __ bgez(count, L_loop);
593 __ bind(L_done);
594 }
595
596 #undef __
597
598 #ifdef COMPILER1
599
600 #define __ ce->masm()->
601
602 void ShenandoahBarrierSetAssembler::gen_pre_barrier_stub(LIR_Assembler* ce, ShenandoahPreBarrierStub* stub) {
603 ShenandoahBarrierSetC1* bs = (ShenandoahBarrierSetC1*)BarrierSet::barrier_set()->barrier_set_c1();
604 // At this point we know that marking is in progress.
605 // If do_load() is true then we have to emit the
606 // load of the previous value; otherwise it has already
607 // been loaded into _pre_val.
608 __ bind(*stub->entry());
609
610 assert(stub->pre_val()->is_register(), "Precondition.");
611
612 Register pre_val_reg = stub->pre_val()->as_register();
613
614 if (stub->do_load()) {
615 ce->mem2reg(stub->addr(), stub->pre_val(), T_OBJECT, stub->patch_code(), stub->info(), false /* wide */);
616 }
617 __ beqz(pre_val_reg, *stub->continuation(), /* is_far */ true);
618 ce->store_parameter(stub->pre_val()->as_register(), 0);
619 __ far_call(RuntimeAddress(bs->pre_barrier_c1_runtime_code_blob()->code_begin()));
620 __ j(*stub->continuation());
621 }
622
623 void ShenandoahBarrierSetAssembler::gen_load_reference_barrier_stub(LIR_Assembler* ce,
624 ShenandoahLoadReferenceBarrierStub* stub) {
625 ShenandoahBarrierSetC1* bs = (ShenandoahBarrierSetC1*)BarrierSet::barrier_set()->barrier_set_c1();
626 __ bind(*stub->entry());
627
628 DecoratorSet decorators = stub->decorators();
629 bool is_strong = ShenandoahBarrierSet::is_strong_access(decorators);
630 bool is_weak = ShenandoahBarrierSet::is_weak_access(decorators);
631 bool is_phantom = ShenandoahBarrierSet::is_phantom_access(decorators);
632 bool is_native = ShenandoahBarrierSet::is_native_access(decorators);
633
634 Register obj = stub->obj()->as_register();
635 Register res = stub->result()->as_register();
636 Register addr = stub->addr()->as_pointer_register();
637 Register tmp1 = stub->tmp1()->as_register();
638 Register tmp2 = stub->tmp2()->as_register();
639
640 assert(res == x10, "result must arrive in x10");
641 assert_different_registers(tmp1, tmp2, t0);
642
643 if (res != obj) {
644 __ mv(res, obj);
645 }
646
647 if (is_strong) {
648 // Check for object in cset.
649 __ mv(tmp2, ShenandoahHeap::in_cset_fast_test_addr());
650 __ srli(tmp1, res, ShenandoahHeapRegion::region_size_bytes_shift_jint());
651 __ add(tmp2, tmp2, tmp1);
652 __ lbu(tmp2, Address(tmp2));
653 __ beqz(tmp2, *stub->continuation(), true /* is_far */);
654 }
655
656 ce->store_parameter(res, 0);
657 ce->store_parameter(addr, 1);
658
659 if (is_strong) {
660 if (is_native) {
661 __ far_call(RuntimeAddress(bs->load_reference_barrier_strong_native_rt_code_blob()->code_begin()));
662 } else {
663 __ far_call(RuntimeAddress(bs->load_reference_barrier_strong_rt_code_blob()->code_begin()));
664 }
665 } else if (is_weak) {
666 __ far_call(RuntimeAddress(bs->load_reference_barrier_weak_rt_code_blob()->code_begin()));
667 } else {
668 assert(is_phantom, "only remaining strength");
669 __ far_call(RuntimeAddress(bs->load_reference_barrier_phantom_rt_code_blob()->code_begin()));
670 }
671
672 __ j(*stub->continuation());
673 }
674
675 #undef __
676
677 #define __ sasm->
678
679 void ShenandoahBarrierSetAssembler::generate_c1_pre_barrier_runtime_stub(StubAssembler* sasm) {
680 __ prologue("shenandoah_pre_barrier", false);
681
682 // arg0 : previous value of memory
683
684 BarrierSet* bs = BarrierSet::barrier_set();
685
686 const Register pre_val = x10;
687 const Register thread = xthread;
688 const Register tmp = t0;
689
690 Address queue_index(thread, in_bytes(ShenandoahThreadLocalData::satb_mark_queue_index_offset()));
691 Address buffer(thread, in_bytes(ShenandoahThreadLocalData::satb_mark_queue_buffer_offset()));
692
693 Label done;
694 Label runtime;
695
696 // Is marking still active?
697 Address gc_state(thread, in_bytes(ShenandoahThreadLocalData::gc_state_offset()));
698 __ lb(tmp, gc_state);
699 __ test_bit(tmp, tmp, ShenandoahHeap::MARKING_BITPOS);
700 __ beqz(tmp, done);
701
702 // Can we store original value in the thread's buffer?
703 __ ld(tmp, queue_index);
704 __ beqz(tmp, runtime);
705
706 __ subi(tmp, tmp, wordSize);
707 __ sd(tmp, queue_index);
708 __ ld(t1, buffer);
709 __ add(tmp, tmp, t1);
710 __ load_parameter(0, t1);
711 __ sd(t1, Address(tmp, 0));
712 __ j(done);
713
714 __ bind(runtime);
715 __ push_call_clobbered_registers();
716 __ load_parameter(0, pre_val);
717 __ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::write_barrier_pre), pre_val);
718 __ pop_call_clobbered_registers();
719 __ bind(done);
720
721 __ epilogue();
722 }
723
724 void ShenandoahBarrierSetAssembler::generate_c1_load_reference_barrier_runtime_stub(StubAssembler* sasm,
725 DecoratorSet decorators) {
726 __ prologue("shenandoah_load_reference_barrier", false);
727 // arg0 : object to be resolved
728
729 __ push_call_clobbered_registers();
730 __ load_parameter(0, x10);
731 __ load_parameter(1, x11);
732
733 bool is_strong = ShenandoahBarrierSet::is_strong_access(decorators);
734 bool is_weak = ShenandoahBarrierSet::is_weak_access(decorators);
735 bool is_phantom = ShenandoahBarrierSet::is_phantom_access(decorators);
736 bool is_native = ShenandoahBarrierSet::is_native_access(decorators);
737 address target = nullptr;
738 if (is_strong) {
739 if (is_native) {
740 target = CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_strong);
741 } else {
742 if (UseCompressedOops) {
743 target = CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_strong_narrow);
744 } else {
745 target = CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_strong);
746 }
747 }
748 } else if (is_weak) {
749 assert(!is_native, "weak must not be called off-heap");
750 if (UseCompressedOops) {
751 target = CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_weak_narrow);
752 } else {
753 target = CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_weak);
754 }
755 } else {
756 assert(is_phantom, "only remaining strength");
757 assert(is_native, "phantom must only be called off-heap");
758 target = CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_phantom);
759 }
760 __ rt_call(target);
761 __ mv(t0, x10);
762 __ pop_call_clobbered_registers();
763 __ mv(x10, t0);
764
765 __ epilogue();
766 }
767
768 #undef __
769
770 #endif // COMPILER1