1 /*
2 * Copyright (c) 2026, Oracle and/or its affiliates. All rights reserved.
3 * Copyright (c) 2018, 2022, Red Hat, Inc. All rights reserved.
4 * Copyright Amazon.com Inc. or its affiliates. All Rights Reserved.
5 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
6 *
7 * This code is free software; you can redistribute it and/or modify it
8 * under the terms of the GNU General Public License version 2 only, as
9 * published by the Free Software Foundation.
10 *
11 * This code is distributed in the hope that it will be useful, but WITHOUT
12 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
13 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
14 * version 2 for more details (a copy is included in the LICENSE file that
15 * accompanied this code).
16 *
17 * You should have received a copy of the GNU General Public License version
18 * 2 along with this work; if not, write to the Free Software Foundation,
19 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
20 *
21 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
22 * or visit www.oracle.com if you need additional information or have any
23 * questions.
24 *
25 */
26
27 #include "code/aotCodeCache.hpp"
28 #include "gc/shenandoah/heuristics/shenandoahHeuristics.hpp"
29 #include "gc/shenandoah/mode/shenandoahMode.hpp"
30 #include "gc/shenandoah/shenandoahBarrierSet.hpp"
31 #include "gc/shenandoah/shenandoahBarrierSetAssembler.hpp"
32 #include "gc/shenandoah/shenandoahHeap.inline.hpp"
33 #include "gc/shenandoah/shenandoahHeapRegion.hpp"
34 #include "gc/shenandoah/shenandoahNMethod.inline.hpp"
35 #include "gc/shenandoah/shenandoahRuntime.hpp"
36 #include "gc/shenandoah/shenandoahThreadLocalData.hpp"
37 #include "interpreter/interp_masm.hpp"
38 #include "interpreter/interpreter.hpp"
39 #include "nativeInst_aarch64.hpp"
40 #include "runtime/javaThread.hpp"
41 #include "runtime/sharedRuntime.hpp"
42 #ifdef COMPILER1
43 #include "c1/c1_LIRAssembler.hpp"
44 #include "c1/c1_MacroAssembler.hpp"
45 #include "gc/shenandoah/c1/shenandoahBarrierSetC1.hpp"
46 #endif
47 #ifdef COMPILER2
48 #include "gc/shenandoah/c2/shenandoahBarrierSetC2.hpp"
49 #include "opto/output.hpp"
50 #endif
51
52 #define __ masm->
53
54 void ShenandoahBarrierSetAssembler::arraycopy_prologue(MacroAssembler* masm, DecoratorSet decorators, bool is_oop,
55 Register src, Register dst, Register count, RegSet saved_regs) {
56 if (is_oop) {
57 bool dest_uninitialized = (decorators & IS_DEST_UNINITIALIZED) != 0;
58 if ((ShenandoahSATBBarrier && !dest_uninitialized) || ShenandoahLoadRefBarrier) {
59
60 Label done;
61
62 // Avoid calling runtime if count == 0
63 __ cbz(count, done);
64
65 // Is GC active?
66 assert(!saved_regs.contains(rscratch1), "Sanity: about to clobber rscratch1");
67 assert(!saved_regs.contains(rscratch2), "Sanity: about to clobber rscratch2");
68 Address gc_state(rthread, in_bytes(ShenandoahThreadLocalData::gc_state_offset()));
69 __ ldrb(rscratch1, gc_state);
70 if (ShenandoahSATBBarrier && dest_uninitialized) {
71 __ tbz(rscratch1, ShenandoahHeap::HAS_FORWARDED_BITPOS, done);
72 } else {
73 __ mov(rscratch2, ShenandoahHeap::HAS_FORWARDED | ShenandoahHeap::MARKING);
74 __ tst(rscratch1, rscratch2);
75 __ br(Assembler::EQ, done);
76 }
77
78 __ push_call_clobbered_registers();
79 // If arguments are not in proper places, shuffle them.
80 // Doing this via the stack is the most straight-forward way to avoid
81 // accidentally smashing any register.
82 if (c_rarg0 != src || c_rarg1 != dst || c_rarg2 != count) {
83 __ push(RegSet::of(src), sp);
84 __ push(RegSet::of(dst), sp);
85 __ push(RegSet::of(count), sp);
86 __ pop(RegSet::of(c_rarg2), sp);
87 __ pop(RegSet::of(c_rarg1), sp);
88 __ pop(RegSet::of(c_rarg0), sp);
89 }
90 address target = nullptr;
91 if (UseCompressedOops) {
92 target = CAST_FROM_FN_PTR(address, ShenandoahRuntime::arraycopy_barrier_narrow_oop);
93 } else {
94 target = CAST_FROM_FN_PTR(address, ShenandoahRuntime::arraycopy_barrier_oop);
95 }
96 __ call_VM_leaf(target, 3);
97 __ pop_call_clobbered_registers();
98 __ bind(done);
99 }
100 }
101 }
102
103 void ShenandoahBarrierSetAssembler::arraycopy_epilogue(MacroAssembler* masm, DecoratorSet decorators, bool is_oop,
104 Register start, Register count, Register tmp) {
105 if (ShenandoahCardBarrier && is_oop) {
106 gen_write_ref_array_post_barrier(masm, decorators, start, count, tmp);
107 }
108 }
109
110 void ShenandoahBarrierSetAssembler::satb_barrier(MacroAssembler* masm,
111 Register obj,
112 Register pre_val,
113 Register thread,
114 Register tmp1,
115 Register tmp2) {
116 assert(ShenandoahSATBBarrier, "Should be checked by caller");
117 assert(thread == rthread, "must be");
118
119 Label done;
120 Label runtime;
121
122 assert_different_registers(obj, pre_val, tmp1, tmp2);
123 assert(pre_val != noreg && tmp1 != noreg && tmp2 != noreg, "expecting a register");
124
125 Address index(thread, in_bytes(ShenandoahThreadLocalData::satb_mark_queue_index_offset()));
126 Address buffer(thread, in_bytes(ShenandoahThreadLocalData::satb_mark_queue_buffer_offset()));
127
128 // Is marking active?
129 Address gc_state(thread, in_bytes(ShenandoahThreadLocalData::gc_state_offset()));
130 __ ldrb(tmp1, gc_state);
131 __ tbz(tmp1, ShenandoahHeap::MARKING_BITPOS, done);
132
133 // Do we need to load the previous value?
134 if (obj != noreg) {
135 if (UseCompressedOops) {
136 __ ldrw(pre_val, Address(obj, 0));
137 __ decode_heap_oop(pre_val);
138 } else {
139 __ ldr(pre_val, Address(obj, 0));
140 }
141 }
142
143 // Is the previous value null?
144 __ cbz(pre_val, done);
145
146 // Can we store original value in the thread's buffer?
147 // Is index == 0?
148 // (The index field is typed as size_t.)
149
150 __ ldr(tmp1, index); // tmp := *index_adr
151 __ cbz(tmp1, runtime); // tmp == 0?
152 // If yes, goto runtime
153
154 __ sub(tmp1, tmp1, wordSize); // tmp := tmp - wordSize
155 __ str(tmp1, index); // *index_adr := tmp
156 __ ldr(tmp2, buffer);
157 __ add(tmp1, tmp1, tmp2); // tmp := tmp + *buffer_adr
158
159 // Record the previous value
160 __ str(pre_val, Address(tmp1, 0));
161 __ b(done);
162
163 __ bind(runtime);
164
165 // Slow-path call
166 __ enter(/* strip_ret_addr = */ true);
167 __ push_call_clobbered_registers();
168 if (c_rarg0 != pre_val) {
169 __ mov(c_rarg0, pre_val);
170 }
171 // Calling with super_call_VM_leaf with c_rarg0 bypasses interpreter checks and avoids any moves.
172 __ super_call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::write_barrier_pre), c_rarg0);
173 __ pop_call_clobbered_registers();
174 __ leave();
175
176 __ bind(done);
177 }
178
179 void ShenandoahBarrierSetAssembler::load_reference_barrier(MacroAssembler* masm, Register dst, Address load_addr, DecoratorSet decorators) {
180 assert(ShenandoahLoadRefBarrier, "Should be enabled");
181 assert(dst != rscratch2, "need rscratch2");
182 assert_different_registers(load_addr.base(), load_addr.index(), rscratch1, rscratch2);
183
184 bool is_strong = ShenandoahBarrierSet::is_strong_access(decorators);
185 bool is_weak = ShenandoahBarrierSet::is_weak_access(decorators);
186 bool is_phantom = ShenandoahBarrierSet::is_phantom_access(decorators);
187 bool is_native = ShenandoahBarrierSet::is_native_access(decorators);
188 bool is_narrow = UseCompressedOops && !is_native;
189
190 Label heap_stable, not_cset;
191 Address gc_state(rthread, in_bytes(ShenandoahThreadLocalData::gc_state_offset()));
192 __ ldrb(rscratch2, gc_state);
193
194 // Check for heap stability
195 if (is_strong) {
196 __ tbz(rscratch2, ShenandoahHeap::HAS_FORWARDED_BITPOS, heap_stable);
197 } else {
198 Label lrb;
199 __ tbnz(rscratch2, ShenandoahHeap::WEAK_ROOTS_BITPOS, lrb);
200 __ tbz(rscratch2, ShenandoahHeap::HAS_FORWARDED_BITPOS, heap_stable);
201 __ bind(lrb);
202 }
203
204 // use r1 for load address
205 Register result_dst = dst;
206 if (dst == r1) {
207 __ mov(rscratch1, dst);
208 dst = rscratch1;
209 }
210
211 // Save r0 and r1, unless it is an output register
212 RegSet to_save = RegSet::of(r0, r1) - result_dst;
213 __ push(to_save, sp);
214 __ lea(r1, load_addr);
215 __ mov(r0, dst);
216
217 // Test for in-cset
218 if (is_strong) {
219 #if INCLUDE_CDS
220 if (AOTCodeCache::is_on_for_dump()) {
221 __ lea(rscratch2, ExternalAddress(AOTRuntimeConstants::cset_base_address()));
222 __ ldr(rscratch2, Address(rscratch2));
223 __ lea(rscratch1, ExternalAddress(AOTRuntimeConstants::grain_shift_address()));
224 __ ldrw(rscratch1, Address(rscratch1));
225 __ lsrv(rscratch1, r0, rscratch1);
226 } else
227 #endif
228 {
229 __ mov(rscratch2, ShenandoahHeap::in_cset_fast_test_addr());
230 __ lsr(rscratch1, r0, ShenandoahHeapRegion::region_size_bytes_shift_jint());
231 }
232 __ ldrb(rscratch2, Address(rscratch2, rscratch1));
233 __ tbz(rscratch2, 0, not_cset);
234 }
235
236 // Slow-path call
237 __ enter(/* strip_ret_addr = */ true);
238 __ push_call_clobbered_registers();
239 address target = nullptr;
240 if (is_strong) {
241 if (is_narrow) {
242 target = CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_strong_narrow);
243 } else {
244 target = CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_strong);
245 }
246 } else if (is_weak) {
247 if (is_narrow) {
248 target = CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_weak_narrow);
249 } else {
250 target = CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_weak);
251 }
252 } else {
253 assert(is_phantom, "only remaining strength");
254 assert(!is_narrow, "phantom access cannot be narrow");
255 target = CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_phantom);
256 }
257 // Calling with super_call_VM_leaf with c_rarg0/1 bypasses interpreter checks and avoids any moves.
258 __ super_call_VM_leaf(target, c_rarg0, c_rarg1);
259 __ mov(rscratch1, r0);
260 __ pop_call_clobbered_registers();
261 __ mov(r0, rscratch1);
262 __ leave();
263
264 __ bind(not_cset);
265
266 __ mov(result_dst, r0);
267 __ pop(to_save, sp);
268
269 __ bind(heap_stable);
270 }
271
272 //
273 // Arguments:
274 //
275 // Inputs:
276 // src: oop location to load from, might be clobbered
277 //
278 // Output:
279 // dst: oop loaded from src location
280 //
281 // Kill:
282 // rscratch1 (scratch reg)
283 //
284 // Alias:
285 // dst: rscratch1 (might use rscratch1 as temporary output register to avoid clobbering src)
286 //
287 void ShenandoahBarrierSetAssembler::load_at(MacroAssembler* masm, DecoratorSet decorators, BasicType type,
288 Register dst, Address src, Register tmp1, Register tmp2) {
289 // 1: non-reference load, no additional barrier is needed
290 if (!is_reference_type(type)) {
291 BarrierSetAssembler::load_at(masm, decorators, type, dst, src, tmp1, tmp2);
292 return;
293 }
294
295 // 2: load a reference from src location and apply LRB if needed
296 if (ShenandoahBarrierSet::need_load_reference_barrier(decorators, type)) {
297 Register result_dst = dst;
298
299 // Preserve src location for LRB
300 if (dst == src.base() || dst == src.index()) {
301 dst = rscratch1;
302 }
303 assert_different_registers(dst, src.base(), src.index());
304
305 BarrierSetAssembler::load_at(masm, decorators, type, dst, src, tmp1, tmp2);
306
307 load_reference_barrier(masm, dst, src, decorators);
308
309 if (dst != result_dst) {
310 __ mov(result_dst, dst);
311 dst = result_dst;
312 }
313 } else {
314 BarrierSetAssembler::load_at(masm, decorators, type, dst, src, tmp1, tmp2);
315 }
316
317 // 3: apply keep-alive barrier if needed
318 if (ShenandoahBarrierSet::need_keep_alive_barrier(decorators, type)) {
319 satb_barrier(masm /* masm */,
320 noreg /* obj */,
321 dst /* pre_val */,
322 rthread /* thread */,
323 tmp1 /* tmp1 */,
324 tmp2 /* tmp2 */);
325 }
326 }
327
328 void ShenandoahBarrierSetAssembler::card_barrier(MacroAssembler* masm, Register obj, Register tmp1, Register tmp2) {
329 assert(ShenandoahCardBarrier, "Should have been checked by caller");
330 assert_different_registers(obj, tmp1, tmp2);
331 assert(CardTable::dirty_card_val() == 0, "must be");
332
333 __ lsr(obj, obj, CardTable::card_shift());
334
335 Address curr_ct_holder_addr(rthread, in_bytes(ShenandoahThreadLocalData::card_table_offset()));
336 __ ldr(tmp1, curr_ct_holder_addr);
337
338 if (UseCondCardMark) {
339 Label L_already_dirty;
340 __ ldrb(tmp2, Address(obj, tmp1));
341 __ cbz(tmp2, L_already_dirty);
342 __ strb(zr, Address(obj, tmp1));
343 __ bind(L_already_dirty);
344 } else {
345 __ strb(zr, Address(obj, tmp1));
346 }
347 }
348
349 void ShenandoahBarrierSetAssembler::store_at(MacroAssembler* masm, DecoratorSet decorators, BasicType type,
350 Address dst, Register val, Register tmp1, Register tmp2, Register tmp3) {
351 // 1: non-reference types require no barriers
352 if (!is_reference_type(type)) {
353 BarrierSetAssembler::store_at(masm, decorators, type, dst, val, tmp1, tmp2, tmp3);
354 return;
355 }
356
357 // Flatten object address right away for simplicity: likely needed by barriers
358 if (dst.index() == noreg && dst.offset() == 0) {
359 if (dst.base() != tmp3) {
360 __ mov(tmp3, dst.base());
361 }
362 } else {
363 __ lea(tmp3, dst);
364 }
365
366 // 2: pre-barrier: SATB needs the previous value
367 if (ShenandoahBarrierSet::need_satb_barrier(decorators, type)) {
368 satb_barrier(masm,
369 tmp3 /* obj */,
370 tmp2 /* pre_val */,
371 rthread /* thread */,
372 tmp1 /* tmp */,
373 rscratch1 /* tmp2 */);
374 }
375
376 // Store!
377 BarrierSetAssembler::store_at(masm, decorators, type, Address(tmp3, 0), val, noreg, noreg, noreg);
378
379 // 3: post-barrier: card barrier needs store address
380 bool storing_non_null = (val != noreg);
381 if (ShenandoahBarrierSet::need_card_barrier(decorators, type) && storing_non_null) {
382 card_barrier(masm, tmp3, tmp1, tmp2);
383 }
384 }
385
386 void ShenandoahBarrierSetAssembler::try_resolve_jobject_in_native(MacroAssembler* masm, Register jni_env,
387 Register obj, Register tmp, Label& slowpath) {
388 Label done;
389 // Resolve jobject
390 BarrierSetAssembler::try_resolve_jobject_in_native(masm, jni_env, obj, tmp, slowpath);
391
392 // Check for null.
393 __ cbz(obj, done);
394
395 assert(obj != rscratch2, "need rscratch2");
396 Address gc_state(jni_env, ShenandoahThreadLocalData::gc_state_offset() - JavaThread::jni_environment_offset());
397 __ lea(rscratch2, gc_state);
398 __ ldrb(rscratch2, Address(rscratch2));
399
400 // Check for heap in evacuation phase
401 __ tbnz(rscratch2, ShenandoahHeap::EVACUATION_BITPOS, slowpath);
402
403 __ bind(done);
404 }
405
406 void ShenandoahBarrierSetAssembler::try_peek_weak_handle_in_nmethod(MacroAssembler* masm, Register weak_handle, Register obj,
407 Register tmp, Label& slow_path) {
408 assert_different_registers(weak_handle, tmp, noreg);
409 assert_different_registers(obj, tmp, noreg);
410
411 Label done;
412
413 // Peek weak handle using the standard implementation.
414 BarrierSetAssembler::try_peek_weak_handle_in_nmethod(masm, weak_handle, obj, tmp, slow_path);
415
416 // Check if the reference is null, and if it is, take the fast path.
417 __ cbz(obj, done);
418
419 Address gc_state(rthread, ShenandoahThreadLocalData::gc_state_offset());
420 __ lea(tmp, gc_state);
421 __ ldrb(tmp, __ legitimize_address(gc_state, 1, tmp));
422
423 // Check if the heap is under weak-reference/roots processing, in
424 // which case we need to take the slow path.
425 __ tbnz(tmp, ShenandoahHeap::WEAK_ROOTS_BITPOS, slow_path);
426 __ bind(done);
427 }
428
429 void ShenandoahBarrierSetAssembler::check_oop(MacroAssembler* masm, Register obj, Register tmp1, Register tmp2, Label& L_error) {
430 assert_different_registers(obj, tmp1, tmp2);
431 // Check if the oop is in the right area of memory
432 #if INCLUDE_CDS
433 if (AOTCodeCache::is_on_for_dump()) {
434 __ lea(tmp2, ExternalAddress(AOTRuntimeConstants::verify_oop_mask_address()));
435 __ ldr(tmp2, Address(tmp2));
436 __ andr(tmp1, obj, tmp2);
437 __ lea(tmp2, ExternalAddress(AOTRuntimeConstants::verify_oop_bits_address()));
438 __ ldr(tmp2, Address(tmp2));
439 } else
440 #endif
441 {
442 __ mov(tmp2, (intptr_t) Universe::verify_oop_mask());
443 __ andr(tmp1, obj, tmp2);
444 __ mov(tmp2, (intptr_t) Universe::verify_oop_bits());
445 }
446 // Compare tmp1 and tmp2. We don't use a compare
447 // instruction here because the flags register is live.
448 __ eor(tmp1, tmp1, tmp2);
449 __ cbnz(tmp1, L_error);
450
451 // This routine is sometimes called before applying GC barriers.
452 // With +COH, loading the klass may end up loading forwarding pointer instead.
453 Label L_skip;
454 if (UseCompactObjectHeaders) {
455 Address gc_state(rthread, in_bytes(ShenandoahThreadLocalData::gc_state_offset()));
456 __ ldrb(tmp1, gc_state);
457 __ tbnz(tmp1, ShenandoahHeap::HAS_FORWARDED_BITPOS, L_skip);
458 }
459
460 // Make sure klass is 'reasonable', which is not zero.
461 __ load_narrow_klass(tmp1, obj);
462 __ cbz(tmp1, L_error);
463 __ bind(L_skip);
464 }
465
466 void ShenandoahBarrierSetAssembler::gen_write_ref_array_post_barrier(MacroAssembler* masm, DecoratorSet decorators,
467 Register start, Register count, Register scratch) {
468 assert(ShenandoahCardBarrier, "Should have been checked by caller");
469
470 Label L_loop, L_done;
471 const Register end = count;
472
473 // Zero count? Nothing to do.
474 __ cbz(count, L_done);
475
476 // end = start + count << LogBytesPerHeapOop
477 // last element address to make inclusive
478 __ lea(end, Address(start, count, Address::lsl(LogBytesPerHeapOop)));
479 __ sub(end, end, BytesPerHeapOop);
480 __ lsr(start, start, CardTable::card_shift());
481 __ lsr(end, end, CardTable::card_shift());
482
483 // number of bytes to copy
484 __ sub(count, end, start);
485
486 Address curr_ct_holder_addr(rthread, in_bytes(ShenandoahThreadLocalData::card_table_offset()));
487 __ ldr(scratch, curr_ct_holder_addr);
488 __ add(start, start, scratch);
489 __ bind(L_loop);
490 __ strb(zr, Address(start, count));
491 __ subs(count, count, 1);
492 __ br(Assembler::GE, L_loop);
493 __ bind(L_done);
494 }
495
496 #undef __
497
498 address ShenandoahBarrierSetAssembler::parse_jump_address(address pc) {
499 NativeInstruction* ni = nativeInstruction_at(pc);
500 assert(ni->is_jump(), "Initial code version: GC barrier fastpath must be a jump");
501 NativeJump* jmp = nativeJump_at(pc);
502 return jmp->jump_destination();
503 }
504
505 static uint32_t encode_patchable_nop() {
506 return 0xD503201F;
507 }
508
509 static uint32_t encode_patchable_jump(address pc, address target_pc) {
510 int32_t disp = checked_cast<int32_t>((intptr_t)target_pc - (intptr_t)pc);
511 int64_t imm26 = disp >> 2;
512 guarantee(Assembler::is_simm(imm26, 26), "Maximum offset is 128MiB");
513 return 0x14000000 | (imm26 & 0x03FFFFFF);
514 }
515
516 void ShenandoahBarrierSetAssembler::insert_patchable_nop(address pc) {
517 *((uint32_t*)pc) = encode_patchable_nop();
518 assert(nativeInstruction_at(pc)->is_nop(), "Sanity");
519 }
520
521 void ShenandoahBarrierSetAssembler::insert_patchable_jump(address pc, address target_pc) {
522 *((uint32_t*)pc) = encode_patchable_jump(pc, target_pc);
523 }
524
525 bool ShenandoahBarrierSetAssembler::is_patchable_nop(address pc) {
526 return *((uint32_t*)pc) == encode_patchable_nop();
527 }
528
529 bool ShenandoahBarrierSetAssembler::is_patchable_jump(address pc, address target_pc) {
530 return *((uint32_t*)pc) == encode_patchable_jump(pc, target_pc);
531 }
532
533 #ifdef COMPILER1
534
535 #define __ ce->masm()->
536
537 void ShenandoahBarrierSetAssembler::keepalive_barrier_c1_stub(LIR_Assembler* ce, ShenandoahKeepaliveBarrierStub* stub) {
538 __ bind(*stub->entry());
539
540 ShenandoahBarrierSetC1* bs = (ShenandoahBarrierSetC1*)BarrierSet::barrier_set()->barrier_set_c1();
541
542 Register obj = stub->obj()->as_register();
543
544 if (stub->do_load()) {
545 ce->mem2reg(stub->addr(), stub->obj(), T_OBJECT, lir_patch_none, nullptr, /* wide = */ false);
546 }
547 __ cbz(obj, *stub->continuation());
548 ce->store_parameter(obj, 0);
549 __ far_call(RuntimeAddress(bs->keepalive_barrier_stub()));
550 __ b(*stub->continuation());
551 }
552
553 void ShenandoahBarrierSetAssembler::load_reference_barrier_c1_stub(LIR_Assembler* ce, ShenandoahLoadReferenceBarrierStub* stub) {
554 __ bind(*stub->entry());
555
556 ShenandoahBarrierSetC1* bs = (ShenandoahBarrierSetC1*)BarrierSet::barrier_set()->barrier_set_c1();
557
558 Register obj = stub->obj()->as_register();
559 Register addr = stub->addr()->as_pointer_register();
560 Register slow_result = stub->slow_result()->as_register();
561 assert_different_registers(obj, addr, slow_result);
562 assert(slow_result == r0, "C1 must know about our slow call result register");
563
564 ce->store_parameter(obj, 0);
565 ce->store_parameter(addr, 1);
566 __ far_call(RuntimeAddress(bs->load_reference_barrier_stub(stub->decorators())));
567 if (obj != slow_result) {
568 __ mov(obj, slow_result);
569 }
570
571 __ b(*stub->continuation());
572 }
573
574 #undef __
575
576 #define __ sasm->
577
578 void ShenandoahBarrierSetAssembler::keepalive_barrier_c1_runtime_stub(StubAssembler* sasm) {
579 __ prologue("shenandoah_keepalive_barrier", false);
580 const Register tmp_obj = r0;
581 const Register tmp1 = r1;
582 const Register tmp2 = r2;
583 __ push(RegSet::of(tmp1, tmp2, tmp_obj), sp);
584 __ load_parameter(0, tmp_obj);
585 satb_barrier(sasm, noreg, tmp_obj, rthread, tmp1, tmp2);
586 __ pop(RegSet::of(tmp1, tmp2, tmp_obj), sp);
587 __ epilogue();
588 }
589
590 void ShenandoahBarrierSetAssembler::load_reference_barrier_c1_runtime_stub(StubAssembler* sasm, DecoratorSet decorators) {
591 __ prologue("shenandoah_load_reference_barrier", false);
592 const Register tmp_obj = r0;
593 const Register tmp_addr = r1;
594 __ push(RegSet::of(tmp_addr), sp);
595 __ load_parameter(0, tmp_obj);
596 __ load_parameter(1, tmp_addr);
597 load_reference_barrier(sasm, tmp_obj, Address(tmp_addr, 0), decorators);
598 __ pop(RegSet::of(tmp_addr), sp);
599 __ epilogue();
600 }
601
602 #undef __
603
604 #endif // COMPILER1
605
606 #ifdef COMPILER2
607
608 #undef __
609 #define __ masm->
610
611
612 void ShenandoahBarrierSetAssembler::load_c2(const MachNode* node, MacroAssembler* masm, Register dst, Address src, Register tmp1, Register tmp2, bool is_narrow, bool is_acquire) {
613 // Do the actual load. This load is the candidate for implicit null check, and MUST come first.
614 if (is_narrow) {
615 if (is_acquire) {
616 assert(src.getMode() == Address::base_plus_offset && src.offset() == 0,
617 "is_acquire path requires address to be base-only");
618 __ ldarw(dst, src.base());
619 } else {
620 __ ldrw(dst, src);
621 }
622 } else {
623 if (is_acquire) {
624 assert(src.getMode() == Address::base_plus_offset && src.offset() == 0,
625 "is_acquire path requires address to be base-only");
626 __ ldar(dst, src.base());
627 } else {
628 __ ldr(dst, src);
629 }
630 }
631
632 ShenandoahBarrierStubC2::load_post(masm, node, dst, src, tmp1, tmp2, is_narrow);
633 }
634
635 void ShenandoahBarrierSetAssembler::store_c2(const MachNode* node, MacroAssembler* masm, Address dst, bool dst_narrow,
636 Register src, bool src_narrow, Register tmp1, Register tmp2, Register tmp3, bool is_volatile) {
637
638 ShenandoahBarrierStubC2::store_pre(masm, node, dst, tmp1, tmp2, tmp3, dst_narrow);
639
640 // Do the actual store
641 if (dst_narrow) {
642 if (!src_narrow) {
643 // Need to encode into rscratch, because we cannot clobber src.
644 if ((node->barrier_data() & ShenandoahBitNotNull) == 0) {
645 __ encode_heap_oop(tmp2, src);
646 } else {
647 __ encode_heap_oop_not_null(tmp2, src);
648 }
649 src = tmp2;
650 }
651
652 if (is_volatile) {
653 assert(dst.getMode() == Address::base_plus_offset && dst.offset() == 0,
654 "is_acquire path requires address to be base-only");
655 __ stlrw(src, dst.base());
656 } else {
657 __ strw(src, dst);
658 }
659 } else {
660 if (is_volatile) {
661 assert(dst.getMode() == Address::base_plus_offset && dst.offset() == 0,
662 "is_acquire path requires address to be base-only");
663 __ stlr(src, dst.base());
664 } else {
665 __ str(src, dst);
666 }
667 }
668
669 ShenandoahBarrierStubC2::store_post(masm, node, dst, tmp2, tmp3);
670 }
671
672 void ShenandoahBarrierSetAssembler::compare_and_set_c2(const MachNode* node, MacroAssembler* masm, Register res, Register addr,
673 Register oldval, Register newval, Register tmp1, Register tmp2, Register tmp3, bool exchange, bool narrow, bool weak, bool acquire) {
674 Assembler::operand_size op_size = narrow ? Assembler::word : Assembler::xword;
675
676 ShenandoahBarrierStubC2::load_store_pre(masm, node, addr, tmp1, tmp2, tmp3, narrow);
677
678 atomic_memory_order order = acquire ? memory_order_seq_cst : memory_order_release;
679
680 // CAS!
681 if (weak) {
682 __ cmpxchg_weak(addr, oldval, newval, op_size, order, exchange ? res : noreg);
683 } else {
684 __ cmpxchg(addr, oldval, newval, op_size, order, exchange ? res : noreg);
685 }
686
687 // If we need a boolean result out of CAS, set the flag appropriately and promote the result.
688 if (!exchange) {
689 assert(res != noreg, "need result register");
690 __ cset(res, Assembler::EQ);
691 }
692
693 ShenandoahBarrierStubC2::load_store_post(masm, node, Address(addr, 0), tmp2, tmp3);
694 }
695
696 void ShenandoahBarrierSetAssembler::get_and_set_c2(const MachNode* node, MacroAssembler* masm, Register preval,
697 Register newval, Register addr, Register tmp1, Register tmp2, Register tmp3, bool is_acquire) {
698 bool is_narrow = node->bottom_type()->isa_narrowoop();
699
700 ShenandoahBarrierStubC2::load_store_pre(masm, node, addr, tmp1, tmp2, tmp3, is_narrow);
701
702 if (is_narrow) {
703 if (is_acquire) {
704 __ atomic_xchgalw(preval, newval, addr);
705 } else {
706 __ atomic_xchgw(preval, newval, addr);
707 }
708 } else {
709 if (is_acquire) {
710 __ atomic_xchgal(preval, newval, addr);
711 } else {
712 __ atomic_xchg(preval, newval, addr);
713 }
714 }
715
716 ShenandoahBarrierStubC2::load_store_post(masm, node, Address(addr, 0), tmp2, tmp3);
717 }
718
719 #undef __
720 #define __ masm.
721
722 void ShenandoahBarrierStubC2::cardtable(MacroAssembler& masm, Address address, Register tmp1, Register tmp2) {
723 assert(CardTable::dirty_card_val() == 0, "must be");
724 Assembler::InlineSkippedInstructionsCounter skip_counter(&masm);
725
726 // tmp1 = card table base (holder)
727 Address curr_ct_holder_addr(rthread, in_bytes(ShenandoahThreadLocalData::card_table_offset()));
728 __ ldr(tmp1, curr_ct_holder_addr);
729
730 // tmp2 = effective address
731 __ lea(tmp2, address);
732
733 // tmp2 = &card_table[ addr >> CardTable::card_shift() ] ; card index
734 __ add(tmp2, tmp1, tmp2, Assembler::LSR, CardTable::card_shift());
735
736 if (UseCondCardMark) {
737 Label L_already_dirty;
738 __ ldrb(tmp1, Address(tmp2));
739 __ cbz(tmp1, L_already_dirty);
740 __ strb(zr, Address(tmp2));
741 __ bind(L_already_dirty);
742 } else {
743 __ strb(zr, Address(tmp2));
744 }
745 }
746
747 void ShenandoahBarrierStubC2::patchable_jump(MacroAssembler& masm, const char gc_state, bool jump_when_state, Register tmp1, Register tmp2, Label* L_target) {
748 const size_t check_size = 8;
749
750 PhaseOutput* const output = Compile::current()->output();
751 if (output->in_scratch_emit_size()) {
752 // Avoid binding L_target and emitting more branches in scratch emits.
753 // We know the patched check is exactly 1 instruction long in release,
754 // and verification adds more instructions.
755 for (size_t c = 0; c < 1 DEBUG_ONLY(+ check_size); c++) {
756 __ nop();
757 }
758 return;
759 }
760
761 #ifdef ASSERT
762 Label L_fake_entry, L_skip;
763 Address gc_state_addr(rthread, in_bytes(ShenandoahThreadLocalData::gc_state_offset()));
764
765 address check_start = __ pc();
766
767 __ ldrb(tmp1, gc_state_addr);
768 __ mov(tmp2, gc_state);
769 __ andr(tmp2, tmp1, tmp2);
770
771 // Emit the secondary jump and use it to cross-check against the actual GC state.
772 // This also checks that all interesting GC state transitions are done non-racily
773 // from the perspective of the thread executing the nmethod.
774 __ relocate(patchable_barrier_Relocation::spec(ShenandoahNMethod::encode_to_reloc(gc_state, jump_when_state)));
775 __ b(L_fake_entry);
776
777 // Currently hot-patched to NOP.
778 if (jump_when_state) {
779 __ cbz(tmp2, L_skip);
780 } else {
781 __ cbnz(tmp2, L_skip);
782 }
783 __ hlt(0);
784
785 // Currently hot-patched to JUMP.
786 __ bind(L_fake_entry);
787 if (jump_when_state) {
788 __ cbnz(tmp2, L_skip);
789 } else {
790 __ cbz(tmp2, L_skip);
791 }
792 __ hlt(0);
793
794 __ bind(L_skip);
795
796 address check_end = __ pc();
797 size_t actual_check_size = pointer_delta(check_end, check_start, Assembler::instruction_size);
798 assert(check_size == actual_check_size, "Must match: %zu != %zu", check_size, actual_check_size);
799 #endif
800
801 // Emit the unconditional branch in the first version of the method.
802 // Let the rest of runtime figure out how to manage it.
803 __ relocate(patchable_barrier_Relocation::spec(ShenandoahNMethod::encode_to_reloc(gc_state, jump_when_state)));
804 __ b(*L_target);
805 }
806
807 void ShenandoahBarrierStubC2::enter_if_gc_state(MacroAssembler& masm, const char test_state, Register tmp1, Register tmp2) {
808 Assembler::InlineSkippedInstructionsCounter skip_counter(&masm);
809 patchable_jump_if_gc_state(masm, test_state, tmp1, tmp2, entry());
810 __ bind(*continuation());
811 }
812
813 void ShenandoahBarrierStubC2::emit_code(MacroAssembler& masm) {
814 Assembler::InlineSkippedInstructionsCounter skip_counter(&masm);
815 assert(_needs_keep_alive_barrier || _needs_load_ref_barrier, "Why are you here?");
816 PhaseOutput* const output = Compile::current()->output();
817
818 // We piggyback on scratch_emit_size mode to compute the slowpath stub size.
819 // We'll use that information to decide whether we need a far jump to the
820 // stub entry point or not. In scratch_emit_size mode we don't bind entry()
821 // because otherwise it will be rebound when we later emit the instructions
822 // for real.
823 if (!output->in_scratch_emit_size()) {
824 __ bind(*entry());
825 }
826
827 // If we need to load ourselves, do it here.
828 if (_do_load) {
829 if (_narrow) {
830 __ ldrw(_obj, _addr);
831 } else {
832 __ ldr(_obj, _addr);
833 }
834 }
835
836 // If the object is null, there is no point in applying barriers.
837 maybe_far_jump_if_zero(masm, _obj);
838
839 // We need to make sure that loads done by callers survive across slow-path calls.
840 // For self-loads, we need to care about the case when both KA and LRB are enabled (rare).
841 bool needs_both_barriers = _needs_keep_alive_barrier && _needs_load_ref_barrier;
842 if (!_do_load || needs_both_barriers) {
843 preserve(_obj);
844 }
845
846 // Go for barriers. Barriers can return straight to continuation, as long
847 // as another barrier is not needed and we can reach the fastpath.
848 if (needs_both_barriers) {
849 // The Load match rule in the .ad file may have legitimized the load
850 // address using a TEMP register and in that case we need to explicitly
851 // preserve them here, because the RA does not consider TEMP as live-in,
852 // and the KA runtime call may clobber them and cause a crash on the
853 // subsequent LRB stub.
854 if (_addr.base() != noreg) {
855 preserve(_addr.base());
856 }
857 if (_addr.index() != noreg) {
858 preserve(_addr.index());
859 }
860 keepalive(masm, nullptr);
861 lrb(masm);
862 } else if (_needs_keep_alive_barrier) {
863 keepalive(masm, continuation());
864 } else if (_needs_load_ref_barrier) {
865 lrb(masm);
866 } else {
867 ShouldNotReachHere();
868 }
869 }
870
871 void ShenandoahBarrierStubC2::maybe_far_jump_if_zero(MacroAssembler& masm, Register reg) {
872 if (_needs_far_jump) {
873 Label L_short_jump;
874 __ cbnz(reg, L_short_jump);
875 __ b(*continuation());
876 __ bind(L_short_jump);
877 } else {
878 __ cbz(reg, *continuation());
879 }
880 }
881
882 void ShenandoahBarrierStubC2::keepalive(MacroAssembler& masm, Label* L_done) {
883 Address index(rthread, in_bytes(ShenandoahThreadLocalData::satb_mark_queue_index_offset()));
884 Address buffer(rthread, in_bytes(ShenandoahThreadLocalData::satb_mark_queue_buffer_offset()));
885 Label L_through, L_slowpath;
886
887 // If another barrier is enabled as well, do a check for a specific barrier.
888 if (_needs_load_ref_barrier) {
889 assert(L_done == nullptr, "Should be");
890 char state_to_check = ShenandoahHeap::MARKING;
891 patchable_jump_if_not_gc_state(masm, state_to_check, _tmp1, _tmp2, &L_through);
892 }
893
894 // Fast-path: put object into buffer.
895 // If buffer is already full, go slow.
896 __ ldr(_tmp1, index);
897 __ cbz(_tmp1, L_slowpath);
898 __ sub(_tmp1, _tmp1, wordSize);
899 __ str(_tmp1, index);
900 __ ldr(_tmp2, buffer);
901
902 // Store the object in queue.
903 // If object is narrow, we need to decode it before inserting.
904 if (_narrow) {
905 __ add(_tmp2, _tmp2, _tmp1);
906 __ decode_heap_oop_not_null(_tmp1, _obj);
907 __ str(_tmp1, Address(_tmp2));
908 } else {
909 // Buffer is 64-bit address, must be in base register.
910 __ str(_obj, Address(_tmp2, _tmp1));
911 }
912
913 // Fast-path exits here.
914 if (L_done != nullptr) {
915 __ b(*L_done);
916 } else {
917 __ b(L_through);
918 }
919
920 // Slow-path: call runtime to handle.
921 __ bind(L_slowpath);
922
923 {
924 SaveLiveRegisters slr(&masm, this);
925
926 // Go to runtime and handle the rest there.
927 __ mov(c_rarg0, _obj);
928 __ lea(lr, RuntimeAddress(keepalive_runtime_entry_addr()));
929 __ blr(lr);
930 }
931 if (L_done != nullptr) {
932 __ b(*L_done);
933 } else {
934 __ bind(L_through);
935 }
936 }
937
938 void ShenandoahBarrierStubC2::lrb(MacroAssembler& masm) {
939 Label L_slow;
940
941 // If another barrier is enabled as well, do a check for a specific barrier.
942 if (_needs_keep_alive_barrier) {
943 char state_to_check = ShenandoahHeap::HAS_FORWARDED | (_needs_load_ref_weak_barrier ? ShenandoahHeap::WEAK_ROOTS : 0);
944 patchable_jump_if_not_gc_state(masm, state_to_check, _tmp1, _tmp2, continuation());
945 }
946
947 // If weak references are being processed, weak/phantom loads need to go slow,
948 // regardless of their cset status.
949 if (_needs_load_ref_weak_barrier) {
950 char state_to_check = ShenandoahHeap::WEAK_ROOTS;
951 patchable_jump_if_gc_state(masm, state_to_check, _tmp1, _tmp2, &L_slow);
952 }
953
954 // Cset-check. Fall-through to slow if in collection set.
955 bool is_aot = AOTCodeCache::is_on_for_dump();
956 if (!is_aot) {
957 __ mov(_tmp1, ShenandoahHeap::in_cset_fast_test_addr());
958 if (_narrow) {
959 __ decode_heap_oop_not_null(_tmp2, _obj);
960 __ add(_tmp1, _tmp1, _tmp2, Assembler::LSR, ShenandoahHeapRegion::region_size_bytes_shift_jint());
961 } else {
962 __ add(_tmp1, _tmp1, _obj, Assembler::LSR, ShenandoahHeapRegion::region_size_bytes_shift_jint());
963 }
964 } else {
965 // Generating AOT code, pull the cset bitmap and region shift from AOT table.
966 if (_narrow) {
967 __ decode_heap_oop_not_null(_tmp1, _obj);
968 } else {
969 __ mov(_tmp1, _obj);
970 }
971 __ lea(_tmp2, ExternalAddress(AOTRuntimeConstants::grain_shift_address()));
972 __ ldrw(_tmp2, Address(_tmp2));
973 __ lsrv(_tmp2, _tmp1, _tmp2);
974 __ lea(_tmp1, ExternalAddress(AOTRuntimeConstants::cset_base_address()));
975 __ ldr(_tmp1, Address(_tmp1));
976 __ add(_tmp1, _tmp1, _tmp2);
977 }
978 __ ldrb(_tmp1, Address(_tmp1, 0));
979 maybe_far_jump_if_zero(masm, _tmp1);
980
981 // Slow path
982 __ bind(L_slow);
983
984 // Obj is the result, need to temporarily stop preserving it.
985 bool is_obj_preserved = is_preserved(_obj);
986 if (is_obj_preserved) {
987 dont_preserve(_obj);
988 }
989 {
990 SaveLiveRegisters slr(&masm, this);
991
992 // Shuffle in the arguments. The end result should be:
993 // c_rarg0 <-- obj
994 // c_rarg1 <-- lea(addr)
995 if (c_rarg0 == _obj) {
996 __ lea(c_rarg1, _addr);
997 } else if (c_rarg1 == _obj) {
998 __ mov(_tmp1, c_rarg1);
999 __ lea(c_rarg1, _addr);
1000 __ mov(c_rarg0, _tmp1);
1001 } else {
1002 assert_different_registers(c_rarg1, _obj);
1003 __ lea(c_rarg1, _addr);
1004 __ mov(c_rarg0, _obj);
1005 }
1006
1007 // Go to runtime and handle the rest there.
1008 __ lea(lr, RuntimeAddress(lrb_runtime_entry_addr()));
1009 __ blr(lr);
1010
1011 // Save the result where needed. Narrow entries return narrowOop (32 bits)
1012 // and AAPCS does not guarantee the upper 32 bits of x0 are zero.
1013 if (_narrow) {
1014 __ movw(_obj, r0);
1015 } else if (_obj != r0) {
1016 __ mov(_obj, r0);
1017 }
1018 }
1019 if (is_obj_preserved) {
1020 preserve(_obj);
1021 }
1022
1023 __ b(*continuation());
1024 }
1025
1026 int ShenandoahBarrierStubC2::available_gp_registers() {
1027 Unimplemented(); // Not used
1028 return 0;
1029 }
1030
1031 bool ShenandoahBarrierStubC2::is_special_register(Register r) {
1032 Unimplemented(); // Not used
1033 return true;
1034 }
1035
1036 static ShenandoahBarrierSetC2State* barrier_set_state() {
1037 return reinterpret_cast<ShenandoahBarrierSetC2State*>(Compile::current()->barrier_set_state());
1038 }
1039
1040 static int get_stub_size(ShenandoahBarrierStubC2* stub) {
1041 PhaseOutput* const output = Compile::current()->output();
1042 assert(output->in_scratch_emit_size(), "only used when in scratch_emit_size.");
1043 BufferBlob* const blob = output->scratch_buffer_blob();
1044 CodeBuffer cb(blob->content_begin(), (address)output->scratch_locs_memory() - blob->content_begin());
1045 MacroAssembler masm(&cb);
1046 stub->emit_code(masm);
1047 return cb.insts_size();
1048 }
1049
1050 void ShenandoahBarrierStubC2::post_init() {
1051 // If we are in scratch emit mode we assume worst case, and force the use of
1052 // far branches.
1053 PhaseOutput* const output = Compile::current()->output();
1054 ShenandoahBarrierSetC2State* state = barrier_set_state();
1055 if (output->in_scratch_emit_size()) {
1056 state->inc_stubs_current_total_size(get_stub_size(this));
1057 _needs_far_jump = true;
1058 return;
1059 }
1060
1061 // The logic implemented in this stub only uses short jumps (cbz, cbnz) if
1062 // the aggregation of all relevant code sections of a method is less than 1MB
1063 // - 2KB. We could be more aggressive and try and compute the distance
1064 // between the fastpath branch and the stub entry but in practice not many
1065 // methods reach the 1MB size.
1066 const BufferSizingData* sizing = output->buffer_sizing_data();
1067 const int code_size = sizing->_code + state->stubs_current_total_size();
1068
1069 // Maximum backward range is 1M. Maximum forward reach is 1M - 4bytes.
1070 // Subtract 2K to be ultra conservative.
1071 const int cond_branch_max_reach = (int)(1*M - 2*K);
1072 _needs_far_jump = code_size >= cond_branch_max_reach;
1073 }
1074
1075 #endif // COMPILER2