1 /*
2 * Copyright (c) 2026, Oracle and/or its affiliates. All rights reserved.
3 * Copyright (c) 2018, 2020, Red Hat, Inc. All rights reserved.
4 * Copyright (c) 2020, 2021, Huawei Technologies Co., Ltd. All rights reserved.
5 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
6 *
7 * This code is free software; you can redistribute it and/or modify it
8 * under the terms of the GNU General Public License version 2 only, as
9 * published by the Free Software Foundation.
10 *
11 * This code is distributed in the hope that it will be useful, but WITHOUT
12 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
13 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
14 * version 2 for more details (a copy is included in the LICENSE file that
15 * accompanied this code).
16 *
17 * You should have received a copy of the GNU General Public License version
18 * 2 along with this work; if not, write to the Free Software Foundation,
19 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
20 *
21 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
22 * or visit www.oracle.com if you need additional information or have any
23 * questions.
24 *
25 */
26
27 #include "gc/shenandoah/heuristics/shenandoahHeuristics.hpp"
28 #include "gc/shenandoah/mode/shenandoahMode.hpp"
29 #include "gc/shenandoah/shenandoahBarrierSet.hpp"
30 #include "gc/shenandoah/shenandoahBarrierSetAssembler.hpp"
31 #include "gc/shenandoah/shenandoahForwarding.hpp"
32 #include "gc/shenandoah/shenandoahHeap.inline.hpp"
33 #include "gc/shenandoah/shenandoahHeapRegion.hpp"
34 #include "gc/shenandoah/shenandoahRuntime.hpp"
35 #include "gc/shenandoah/shenandoahThreadLocalData.hpp"
36 #include "interpreter/interp_masm.hpp"
37 #include "interpreter/interpreter.hpp"
38 #include "runtime/javaThread.hpp"
39 #include "runtime/sharedRuntime.hpp"
40 #ifdef COMPILER1
41 #include "c1/c1_LIRAssembler.hpp"
42 #include "c1/c1_MacroAssembler.hpp"
43 #include "gc/shenandoah/c1/shenandoahBarrierSetC1.hpp"
44 #endif
45
46 #define __ masm->
47
48 void ShenandoahBarrierSetAssembler::arraycopy_prologue(MacroAssembler* masm, DecoratorSet decorators, bool is_oop,
49 Register src, Register dst, Register count, RegSet saved_regs) {
50 if (is_oop) {
51 bool dest_uninitialized = (decorators & IS_DEST_UNINITIALIZED) != 0;
52 if ((ShenandoahSATBBarrier && !dest_uninitialized) || ShenandoahLoadRefBarrier) {
53
54 Label done;
55
56 // Avoid calling runtime if count == 0
57 __ beqz(count, done);
58
59 // Is GC active?
60 Address gc_state(xthread, in_bytes(ShenandoahThreadLocalData::gc_state_offset()));
61 assert_different_registers(src, dst, count, t0);
62
63 __ lbu(t0, gc_state);
64 if (ShenandoahSATBBarrier && dest_uninitialized) {
65 __ test_bit(t0, t0, ShenandoahHeap::HAS_FORWARDED_BITPOS);
66 __ beqz(t0, done);
67 } else {
68 __ andi(t0, t0, ShenandoahHeap::HAS_FORWARDED | ShenandoahHeap::MARKING);
69 __ beqz(t0, done);
70 }
71
72 __ push_reg(saved_regs, sp);
73 if (UseCompressedOops) {
74 __ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::arraycopy_barrier_narrow_oop),
75 src, dst, count);
76 } else {
77 __ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::arraycopy_barrier_oop), src, dst, count);
78 }
79 __ pop_reg(saved_regs, sp);
80 __ bind(done);
81 }
82 }
83 }
84
85 void ShenandoahBarrierSetAssembler::arraycopy_epilogue(MacroAssembler* masm, DecoratorSet decorators, bool is_oop,
86 Register start, Register count, Register tmp) {
87 if (ShenandoahCardBarrier && is_oop) {
88 gen_write_ref_array_post_barrier(masm, decorators, start, count, tmp);
89 }
90 }
91
92 void ShenandoahBarrierSetAssembler::satb_barrier(MacroAssembler* masm,
93 Register obj,
94 Register pre_val,
95 Register thread,
96 Register tmp1,
97 Register tmp2,
98 bool tosca_live,
99 bool expand_call) {
100 assert(ShenandoahSATBBarrier, "Should be checked by caller");
101
102 // If expand_call is true then we expand the call_VM_leaf macro
103 // directly to skip generating the check by
104 // InterpreterMacroAssembler::call_VM_leaf_base that checks _last_sp.
105 assert(thread == xthread, "must be");
106
107 Label done;
108 Label runtime;
109
110 assert_different_registers(obj, pre_val, tmp1, tmp2);
111 assert(pre_val != noreg && tmp1 != noreg && tmp2 != noreg, "expecting a register");
112
113 Address index(thread, in_bytes(ShenandoahThreadLocalData::satb_mark_queue_index_offset()));
114 Address buffer(thread, in_bytes(ShenandoahThreadLocalData::satb_mark_queue_buffer_offset()));
115
116 // Is marking active?
117 Address gc_state(xthread, in_bytes(ShenandoahThreadLocalData::gc_state_offset()));
118 __ lbu(t1, gc_state);
119 __ test_bit(t1, t1, ShenandoahHeap::MARKING_BITPOS);
120 __ beqz(t1, done);
121
122 // Do we need to load the previous value?
123 if (obj != noreg) {
124 __ load_heap_oop(pre_val, Address(obj, 0), noreg, noreg, AS_RAW);
125 }
126
127 // Is the previous value null?
128 __ beqz(pre_val, done);
129
130 // Can we store original value in the thread's buffer?
131 // Is index == 0?
132 // (The index field is typed as size_t.)
133 __ ld(tmp1, index); // tmp := *index_adr
134 __ beqz(tmp1, runtime); // tmp == 0? If yes, goto runtime
135
136 __ subi(tmp1, tmp1, wordSize); // tmp := tmp - wordSize
137 __ sd(tmp1, index); // *index_adr := tmp
138 __ ld(tmp2, buffer);
139 __ add(tmp1, tmp1, tmp2); // tmp := tmp + *buffer_adr
140
141 // Record the previous value
142 __ sd(pre_val, Address(tmp1, 0));
143 __ j(done);
144
145 __ bind(runtime);
146 // save the live input values
147 RegSet saved = RegSet::of(pre_val);
148 if (tosca_live) saved += RegSet::of(x10);
149 if (obj != noreg) saved += RegSet::of(obj);
150
151 __ push_reg(saved, sp);
152
153 // Calling the runtime using the regular call_VM_leaf mechanism generates
154 // code (generated by InterpreterMacroAssember::call_VM_leaf_base)
155 // that checks that the *(rfp+frame::interpreter_frame_last_sp) is null.
156 //
157 // If we care generating the pre-barrier without a frame (e.g. in the
158 // intrinsified Reference.get() routine) then ebp might be pointing to
159 // the caller frame and so this check will most likely fail at runtime.
160 //
161 // Expanding the call directly bypasses the generation of the check.
162 // So when we do not have have a full interpreter frame on the stack
163 // expand_call should be passed true.
164 if (expand_call) {
165 assert(pre_val != c_rarg1, "smashed arg");
166 __ super_call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::write_barrier_pre), pre_val);
167 } else {
168 __ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::write_barrier_pre), pre_val);
169 }
170
171 __ pop_reg(saved, sp);
172
173 __ bind(done);
174 }
175
176 void ShenandoahBarrierSetAssembler::resolve_forward_pointer(MacroAssembler* masm, Register dst, Register tmp) {
177 assert(ShenandoahLoadRefBarrier || ShenandoahCASBarrier, "Should be enabled");
178
179 Label is_null;
180 __ beqz(dst, is_null);
181 resolve_forward_pointer_not_null(masm, dst, tmp);
182 __ bind(is_null);
183 }
184
185 // IMPORTANT: This must preserve all registers, even t0 and t1, except those explicitly
186 // passed in.
187 void ShenandoahBarrierSetAssembler::resolve_forward_pointer_not_null(MacroAssembler* masm, Register dst, Register tmp) {
188 assert(ShenandoahLoadRefBarrier || ShenandoahCASBarrier, "Should be enabled");
189 // The below loads the mark word, checks if the lowest two bits are
190 // set, and if so, clear the lowest two bits and copy the result
191 // to dst. Otherwise it leaves dst alone.
192 // Implementing this is surprisingly awkward. I do it here by:
193 // - Inverting the mark word
194 // - Test lowest two bits == 0
195 // - If so, set the lowest two bits
196 // - Invert the result back, and copy to dst
197 RegSet saved_regs = RegSet::of(t2);
198 bool borrow_reg = (tmp == noreg);
199 if (borrow_reg) {
200 // No free registers available. Make one useful.
201 tmp = t0;
202 if (tmp == dst) {
203 tmp = t1;
204 }
205 saved_regs += RegSet::of(tmp);
206 }
207
208 assert_different_registers(tmp, dst, t2);
209 __ push_reg(saved_regs, sp);
210
211 Label done;
212 __ ld(tmp, Address(dst, oopDesc::mark_offset_in_bytes()));
213 __ xori(tmp, tmp, -1); // eon with 0 is equivalent to XOR with -1
214 __ andi(t2, tmp, markWord::lock_mask_in_place);
215 __ bnez(t2, done);
216 __ ori(tmp, tmp, markWord::marked_value);
217 __ xori(dst, tmp, -1); // eon with 0 is equivalent to XOR with -1
218 __ bind(done);
219
220 __ pop_reg(saved_regs, sp);
221 }
222
223 void ShenandoahBarrierSetAssembler::load_reference_barrier(MacroAssembler* masm,
224 Register dst,
225 Address load_addr,
226 DecoratorSet decorators) {
227 assert(ShenandoahLoadRefBarrier, "Should be enabled");
228 assert(dst != t1 && load_addr.base() != t1, "need t1");
229 assert_different_registers(load_addr.base(), t0, t1);
230
231 bool is_strong = ShenandoahBarrierSet::is_strong_access(decorators);
232 bool is_weak = ShenandoahBarrierSet::is_weak_access(decorators);
233 bool is_phantom = ShenandoahBarrierSet::is_phantom_access(decorators);
234 bool is_native = ShenandoahBarrierSet::is_native_access(decorators);
235 bool is_narrow = UseCompressedOops && !is_native;
236
237 Label heap_stable, not_cset;
238 __ enter();
239 Address gc_state(xthread, in_bytes(ShenandoahThreadLocalData::gc_state_offset()));
240 __ lbu(t1, gc_state);
241
242 // Check for heap stability
243 if (is_strong) {
244 __ test_bit(t1, t1, ShenandoahHeap::HAS_FORWARDED_BITPOS);
245 __ beqz(t1, heap_stable);
246 } else {
247 Label lrb;
248 __ test_bit(t0, t1, ShenandoahHeap::WEAK_ROOTS_BITPOS);
249 __ bnez(t0, lrb);
250 __ test_bit(t0, t1, ShenandoahHeap::HAS_FORWARDED_BITPOS);
251 __ beqz(t0, heap_stable);
252 __ bind(lrb);
253 }
254
255 // use x11 for load address
256 Register result_dst = dst;
257 if (dst == x11) {
258 __ mv(t1, dst);
259 dst = t1;
260 }
261
262 // Save x10 and x11, unless it is an output register
263 RegSet saved_regs = RegSet::of(x10, x11) - result_dst;
264 __ push_reg(saved_regs, sp);
265 __ la(x11, load_addr);
266 __ mv(x10, dst);
267
268 // Test for in-cset
269 if (is_strong) {
270 __ mv(t1, ShenandoahHeap::in_cset_fast_test_addr());
271 __ srli(t0, x10, ShenandoahHeapRegion::region_size_bytes_shift_jint());
272 __ add(t1, t1, t0);
273 __ lbu(t1, Address(t1));
274 __ test_bit(t0, t1, 0);
275 __ beqz(t0, not_cset);
276 }
277
278 __ push_call_clobbered_registers();
279 address target = nullptr;
280 if (is_strong) {
281 if (is_narrow) {
282 target = CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_strong_narrow);
283 } else {
284 target = CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_strong);
285 }
286 } else if (is_weak) {
287 if (is_narrow) {
288 target = CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_weak_narrow);
289 } else {
290 target = CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_weak);
291 }
292 } else {
293 assert(is_phantom, "only remaining strength");
294 assert(!is_narrow, "phantom access cannot be narrow");
295 target = CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_weak);
296 }
297 __ rt_call(target);
298 __ mv(t0, x10);
299 __ pop_call_clobbered_registers();
300 __ mv(x10, t0);
301 __ bind(not_cset);
302 __ mv(result_dst, x10);
303 __ pop_reg(saved_regs, sp);
304
305 __ bind(heap_stable);
306 __ leave();
307 }
308
309 //
310 // Arguments:
311 //
312 // Inputs:
313 // src: oop location to load from, might be clobbered
314 //
315 // Output:
316 // dst: oop loaded from src location
317 //
318 // Kill:
319 // x30 (tmp reg)
320 //
321 // Alias:
322 // dst: x30 (might use x30 as temporary output register to avoid clobbering src)
323 //
324 void ShenandoahBarrierSetAssembler::load_at(MacroAssembler* masm,
325 DecoratorSet decorators,
326 BasicType type,
327 Register dst,
328 Address src,
329 Register tmp1,
330 Register tmp2) {
331 // 1: non-reference load, no additional barrier is needed
332 if (!is_reference_type(type)) {
333 BarrierSetAssembler::load_at(masm, decorators, type, dst, src, tmp1, tmp2);
334 return;
335 }
336
337 // 2: load a reference from src location and apply LRB if needed
338 if (ShenandoahBarrierSet::need_load_reference_barrier(decorators, type)) {
339 Register result_dst = dst;
340
341 // Preserve src location for LRB
342 RegSet saved_regs;
343 if (dst == src.base()) {
344 dst = (src.base() == x28) ? x29 : x28;
345 saved_regs = RegSet::of(dst);
346 __ push_reg(saved_regs, sp);
347 }
348 assert_different_registers(dst, src.base());
349
350 BarrierSetAssembler::load_at(masm, decorators, type, dst, src, tmp1, tmp2);
351
352 load_reference_barrier(masm, dst, src, decorators);
353
354 if (dst != result_dst) {
355 __ mv(result_dst, dst);
356 dst = result_dst;
357 }
358
359 if (saved_regs.bits() != 0) {
360 __ pop_reg(saved_regs, sp);
361 }
362 } else {
363 BarrierSetAssembler::load_at(masm, decorators, type, dst, src, tmp1, tmp2);
364 }
365
366 // 3: apply keep-alive barrier if needed
367 if (ShenandoahBarrierSet::need_keep_alive_barrier(decorators, type)) {
368 __ enter();
369 __ push_call_clobbered_registers();
370 satb_barrier(masm /* masm */,
371 noreg /* obj */,
372 dst /* pre_val */,
373 xthread /* thread */,
374 tmp1 /* tmp1 */,
375 tmp2 /* tmp2 */,
376 true /* tosca_live */,
377 true /* expand_call */);
378 __ pop_call_clobbered_registers();
379 __ leave();
380 }
381 }
382
383 void ShenandoahBarrierSetAssembler::card_barrier(MacroAssembler* masm, Register obj) {
384 assert(ShenandoahCardBarrier, "Should have been checked by caller");
385
386 __ srli(obj, obj, CardTable::card_shift());
387
388 assert(CardTable::dirty_card_val() == 0, "must be");
389
390 Address curr_ct_holder_addr(xthread, in_bytes(ShenandoahThreadLocalData::card_table_offset()));
391 __ ld(t1, curr_ct_holder_addr);
392 __ add(t1, obj, t1);
393
394 if (UseCondCardMark) {
395 Label L_already_dirty;
396 __ lbu(t0, Address(t1));
397 __ beqz(t0, L_already_dirty);
398 __ sb(zr, Address(t1));
399 __ bind(L_already_dirty);
400 } else {
401 __ sb(zr, Address(t1));
402 }
403 }
404
405 void ShenandoahBarrierSetAssembler::store_at(MacroAssembler* masm, DecoratorSet decorators, BasicType type,
406 Address dst, Register val, Register tmp1, Register tmp2, Register tmp3) {
407 // 1: non-reference types require no barriers
408 if (!is_reference_type(type)) {
409 BarrierSetAssembler::store_at(masm, decorators, type, dst, val, tmp1, tmp2, tmp3);
410 return;
411 }
412
413 // Flatten object address right away for simplicity: likely needed by barriers
414 if (dst.offset() == 0) {
415 if (dst.base() != tmp3) {
416 __ mv(tmp3, dst.base());
417 }
418 } else {
419 __ la(tmp3, dst);
420 }
421
422 bool storing_non_null = (val != noreg);
423
424 // 2: pre-barrier: SATB needs the previous value
425 if (ShenandoahBarrierSet::need_satb_barrier(decorators, type)) {
426 satb_barrier(masm,
427 tmp3 /* obj */,
428 tmp2 /* pre_val */,
429 xthread /* thread */,
430 tmp1 /* tmp */,
431 t0 /* tmp2 */,
432 storing_non_null /* tosca_live */,
433 false /* expand_call */);
434 }
435
436 // Store!
437 BarrierSetAssembler::store_at(masm, decorators, type, Address(tmp3, 0), val, noreg, noreg, noreg);
438
439 // 3: post-barrier: card barrier needs store address
440 if (ShenandoahBarrierSet::need_card_barrier(decorators, type) && storing_non_null) {
441 card_barrier(masm, tmp3);
442 }
443 }
444
445 void ShenandoahBarrierSetAssembler::try_resolve_jobject_in_native(MacroAssembler* masm, Register jni_env,
446 Register obj, Register tmp, Label& slowpath) {
447 Label done;
448 // Resolve jobject
449 BarrierSetAssembler::try_resolve_jobject_in_native(masm, jni_env, obj, tmp, slowpath);
450
451 // Check for null.
452 __ beqz(obj, done);
453
454 assert(obj != t1, "need t1");
455 Address gc_state(jni_env, ShenandoahThreadLocalData::gc_state_offset() - JavaThread::jni_environment_offset());
456 __ lbu(t1, gc_state);
457
458 // Check for heap in evacuation phase
459 __ test_bit(t0, t1, ShenandoahHeap::EVACUATION_BITPOS);
460 __ bnez(t0, slowpath);
461
462 __ bind(done);
463 }
464
465 #ifdef COMPILER2
466 void ShenandoahBarrierSetAssembler::try_resolve_weak_handle_in_c2(MacroAssembler *masm, Register obj,
467 Register tmp, Label& slow_path) {
468 assert_different_registers(obj, tmp);
469
470 Label done;
471
472 // Resolve weak handle using the standard implementation.
473 BarrierSetAssembler::try_resolve_weak_handle_in_c2(masm, obj, tmp, slow_path);
474
475 // Check if the reference is null, and if it is, take the fast path.
476 __ beqz(obj, done);
477
478 Address gc_state(xthread, ShenandoahThreadLocalData::gc_state_offset());
479 __ lbu(tmp, gc_state);
480
481 // Check if the heap is under weak-reference/roots processing, in
482 // which case we need to take the slow path.
483 __ test_bit(tmp, tmp, ShenandoahHeap::WEAK_ROOTS_BITPOS);
484 __ bnez(tmp, slow_path);
485 __ bind(done);
486 }
487 #endif
488
489 // Special Shenandoah CAS implementation that handles false negatives due
490 // to concurrent evacuation. The service is more complex than a
491 // traditional CAS operation because the CAS operation is intended to
492 // succeed if the reference at addr exactly matches expected or if the
493 // reference at addr holds a pointer to a from-space object that has
494 // been relocated to the location named by expected. There are two
495 // races that must be addressed:
496 // a) A parallel thread may mutate the contents of addr so that it points
497 // to a different object. In this case, the CAS operation should fail.
498 // b) A parallel thread may heal the contents of addr, replacing a
499 // from-space pointer held in addr with the to-space pointer
500 // representing the new location of the object.
501 // Upon entry to cmpxchg_oop, it is assured that new_val equals null
502 // or it refers to an object that is not being evacuated out of
503 // from-space, or it refers to the to-space version of an object that
504 // is being evacuated out of from-space.
505 //
506 // By default the value held in the result register following execution
507 // of the generated code sequence is 0 to indicate failure of CAS,
508 // non-zero to indicate success. If is_cae, the result is the value most
509 // recently fetched from addr rather than a boolean success indicator.
510 //
511 // Clobbers t0, t1
512 void ShenandoahBarrierSetAssembler::cmpxchg_oop(MacroAssembler* masm,
513 Register addr,
514 Register expected,
515 Register new_val,
516 Assembler::Aqrl acquire,
517 Assembler::Aqrl release,
518 bool is_cae,
519 Register result) {
520 bool is_narrow = UseCompressedOops;
521 Assembler::operand_size size = is_narrow ? Assembler::uint32 : Assembler::int64;
522
523 assert_different_registers(addr, expected, t0, t1);
524 assert_different_registers(addr, new_val, t0, t1);
525
526 Label retry, success, fail, done;
527
528 __ bind(retry);
529
530 // Step1: Try to CAS.
531 __ cmpxchg(addr, expected, new_val, size, acquire, release, /* result */ t1);
532
533 // If success, then we are done.
534 __ beq(expected, t1, success);
535
536 // Step2: CAS failed, check the forwarded pointer.
537 __ mv(t0, t1);
538
539 if (is_narrow) {
540 __ decode_heap_oop(t0, t0);
541 }
542 resolve_forward_pointer(masm, t0);
543
544 __ encode_heap_oop(t0, t0);
545
546 // Report failure when the forwarded oop was not expected.
547 __ bne(t0, expected, fail);
548
549 // Step 3: CAS again using the forwarded oop.
550 __ cmpxchg(addr, t1, new_val, size, acquire, release, /* result */ t0);
551
552 // Retry when failed.
553 __ bne(t0, t1, retry);
554
555 __ bind(success);
556 if (is_cae) {
557 __ mv(result, expected);
558 } else {
559 __ mv(result, 1);
560 }
561 __ j(done);
562
563 __ bind(fail);
564 if (is_cae) {
565 __ mv(result, t0);
566 } else {
567 __ mv(result, zr);
568 }
569
570 __ bind(done);
571 }
572
573 void ShenandoahBarrierSetAssembler::gen_write_ref_array_post_barrier(MacroAssembler* masm, DecoratorSet decorators,
574 Register start, Register count, Register tmp) {
575 assert(ShenandoahCardBarrier, "Did you mean to enable ShenandoahCardBarrier?");
576
577 Label L_loop, L_done;
578 const Register end = count;
579
580 // Zero count? Nothing to do.
581 __ beqz(count, L_done);
582
583 // end = start + count << LogBytesPerHeapOop
584 // last element address to make inclusive
585 __ shadd(end, count, start, tmp, LogBytesPerHeapOop);
586 __ subi(end, end, BytesPerHeapOop);
587 __ srli(start, start, CardTable::card_shift());
588 __ srli(end, end, CardTable::card_shift());
589
590 // number of bytes to copy
591 __ sub(count, end, start);
592
593 Address curr_ct_holder_addr(xthread, in_bytes(ShenandoahThreadLocalData::card_table_offset()));
594 __ ld(tmp, curr_ct_holder_addr);
595 __ add(start, start, tmp);
596
597 __ bind(L_loop);
598 __ add(tmp, start, count);
599 __ sb(zr, Address(tmp));
600 __ subi(count, count, 1);
601 __ bgez(count, L_loop);
602 __ bind(L_done);
603 }
604
605 #undef __
606
607 #ifdef COMPILER1
608
609 #define __ ce->masm()->
610
611 void ShenandoahBarrierSetAssembler::gen_pre_barrier_stub(LIR_Assembler* ce, ShenandoahPreBarrierStub* stub) {
612 ShenandoahBarrierSetC1* bs = (ShenandoahBarrierSetC1*)BarrierSet::barrier_set()->barrier_set_c1();
613 // At this point we know that marking is in progress.
614 // If do_load() is true then we have to emit the
615 // load of the previous value; otherwise it has already
616 // been loaded into _pre_val.
617 __ bind(*stub->entry());
618
619 assert(stub->pre_val()->is_register(), "Precondition.");
620
621 Register pre_val_reg = stub->pre_val()->as_register();
622
623 if (stub->do_load()) {
624 ce->mem2reg(stub->addr(), stub->pre_val(), T_OBJECT, stub->patch_code(), stub->info(), false /* wide */);
625 }
626 __ beqz(pre_val_reg, *stub->continuation(), /* is_far */ true);
627 ce->store_parameter(stub->pre_val()->as_register(), 0);
628 __ far_call(RuntimeAddress(bs->pre_barrier_c1_runtime_code_blob()->code_begin()));
629 __ j(*stub->continuation());
630 }
631
632 void ShenandoahBarrierSetAssembler::gen_load_reference_barrier_stub(LIR_Assembler* ce,
633 ShenandoahLoadReferenceBarrierStub* stub) {
634 ShenandoahBarrierSetC1* bs = (ShenandoahBarrierSetC1*)BarrierSet::barrier_set()->barrier_set_c1();
635 __ bind(*stub->entry());
636
637 DecoratorSet decorators = stub->decorators();
638 bool is_strong = ShenandoahBarrierSet::is_strong_access(decorators);
639 bool is_weak = ShenandoahBarrierSet::is_weak_access(decorators);
640 bool is_phantom = ShenandoahBarrierSet::is_phantom_access(decorators);
641 bool is_native = ShenandoahBarrierSet::is_native_access(decorators);
642
643 Register obj = stub->obj()->as_register();
644 Register res = stub->result()->as_register();
645 Register addr = stub->addr()->as_pointer_register();
646 Register tmp1 = stub->tmp1()->as_register();
647 Register tmp2 = stub->tmp2()->as_register();
648
649 assert(res == x10, "result must arrive in x10");
650 assert_different_registers(tmp1, tmp2, t0);
651
652 if (res != obj) {
653 __ mv(res, obj);
654 }
655
656 if (is_strong) {
657 // Check for object in cset.
658 __ mv(tmp2, ShenandoahHeap::in_cset_fast_test_addr());
659 __ srli(tmp1, res, ShenandoahHeapRegion::region_size_bytes_shift_jint());
660 __ add(tmp2, tmp2, tmp1);
661 __ lbu(tmp2, Address(tmp2));
662 __ beqz(tmp2, *stub->continuation(), true /* is_far */);
663 }
664
665 ce->store_parameter(res, 0);
666 ce->store_parameter(addr, 1);
667
668 if (is_strong) {
669 if (is_native) {
670 __ far_call(RuntimeAddress(bs->load_reference_barrier_strong_native_rt_code_blob()->code_begin()));
671 } else {
672 __ far_call(RuntimeAddress(bs->load_reference_barrier_strong_rt_code_blob()->code_begin()));
673 }
674 } else if (is_weak) {
675 __ far_call(RuntimeAddress(bs->load_reference_barrier_weak_rt_code_blob()->code_begin()));
676 } else {
677 assert(is_phantom, "only remaining strength");
678 __ far_call(RuntimeAddress(bs->load_reference_barrier_phantom_rt_code_blob()->code_begin()));
679 }
680
681 __ j(*stub->continuation());
682 }
683
684 #undef __
685
686 #define __ sasm->
687
688 void ShenandoahBarrierSetAssembler::generate_c1_pre_barrier_runtime_stub(StubAssembler* sasm) {
689 __ prologue("shenandoah_pre_barrier", false);
690
691 // arg0 : previous value of memory
692
693 BarrierSet* bs = BarrierSet::barrier_set();
694
695 const Register pre_val = x10;
696 const Register thread = xthread;
697 const Register tmp = t0;
698
699 Address queue_index(thread, in_bytes(ShenandoahThreadLocalData::satb_mark_queue_index_offset()));
700 Address buffer(thread, in_bytes(ShenandoahThreadLocalData::satb_mark_queue_buffer_offset()));
701
702 Label done;
703 Label runtime;
704
705 // Is marking still active?
706 Address gc_state(thread, in_bytes(ShenandoahThreadLocalData::gc_state_offset()));
707 __ lb(tmp, gc_state);
708 __ test_bit(tmp, tmp, ShenandoahHeap::MARKING_BITPOS);
709 __ beqz(tmp, done);
710
711 // Can we store original value in the thread's buffer?
712 __ ld(tmp, queue_index);
713 __ beqz(tmp, runtime);
714
715 __ subi(tmp, tmp, wordSize);
716 __ sd(tmp, queue_index);
717 __ ld(t1, buffer);
718 __ add(tmp, tmp, t1);
719 __ load_parameter(0, t1);
720 __ sd(t1, Address(tmp, 0));
721 __ j(done);
722
723 __ bind(runtime);
724 __ push_call_clobbered_registers();
725 __ load_parameter(0, pre_val);
726 __ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::write_barrier_pre), pre_val);
727 __ pop_call_clobbered_registers();
728 __ bind(done);
729
730 __ epilogue();
731 }
732
733 void ShenandoahBarrierSetAssembler::generate_c1_load_reference_barrier_runtime_stub(StubAssembler* sasm,
734 DecoratorSet decorators) {
735 __ prologue("shenandoah_load_reference_barrier", false);
736 // arg0 : object to be resolved
737
738 __ push_call_clobbered_registers();
739 __ load_parameter(0, x10);
740 __ load_parameter(1, x11);
741
742 bool is_strong = ShenandoahBarrierSet::is_strong_access(decorators);
743 bool is_weak = ShenandoahBarrierSet::is_weak_access(decorators);
744 bool is_phantom = ShenandoahBarrierSet::is_phantom_access(decorators);
745 bool is_native = ShenandoahBarrierSet::is_native_access(decorators);
746 address target = nullptr;
747 if (is_strong) {
748 if (is_native) {
749 target = CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_strong);
750 } else {
751 if (UseCompressedOops) {
752 target = CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_strong_narrow);
753 } else {
754 target = CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_strong);
755 }
756 }
757 } else if (is_weak) {
758 assert(!is_native, "weak must not be called off-heap");
759 if (UseCompressedOops) {
760 target = CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_weak_narrow);
761 } else {
762 target = CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_weak);
763 }
764 } else {
765 assert(is_phantom, "only remaining strength");
766 assert(is_native, "phantom must only be called off-heap");
767 target = CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_phantom);
768 }
769 __ rt_call(target);
770 __ mv(t0, x10);
771 __ pop_call_clobbered_registers();
772 __ mv(x10, t0);
773
774 __ epilogue();
775 }
776
777 #undef __
778
779 #endif // COMPILER1