1 /*
2 * Copyright (c) 2026, Oracle and/or its affiliates. All rights reserved.
3 * Copyright (c) 2018, 2022, Red Hat, Inc. All rights reserved.
4 * Copyright Amazon.com Inc. or its affiliates. All Rights Reserved.
5 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
6 *
7 * This code is free software; you can redistribute it and/or modify it
8 * under the terms of the GNU General Public License version 2 only, as
9 * published by the Free Software Foundation.
10 *
11 * This code is distributed in the hope that it will be useful, but WITHOUT
12 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
13 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
14 * version 2 for more details (a copy is included in the LICENSE file that
15 * accompanied this code).
16 *
17 * You should have received a copy of the GNU General Public License version
18 * 2 along with this work; if not, write to the Free Software Foundation,
19 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
20 *
21 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
22 * or visit www.oracle.com if you need additional information or have any
23 * questions.
24 *
25 */
26
27 #include "code/aotCodeCache.hpp"
28 #include "gc/shenandoah/heuristics/shenandoahHeuristics.hpp"
29 #include "gc/shenandoah/mode/shenandoahMode.hpp"
30 #include "gc/shenandoah/shenandoahBarrierSet.hpp"
31 #include "gc/shenandoah/shenandoahBarrierSetAssembler.hpp"
32 #include "gc/shenandoah/shenandoahHeap.inline.hpp"
33 #include "gc/shenandoah/shenandoahHeapRegion.hpp"
34 #include "gc/shenandoah/shenandoahNMethod.inline.hpp"
35 #include "gc/shenandoah/shenandoahRuntime.hpp"
36 #include "gc/shenandoah/shenandoahThreadLocalData.hpp"
37 #include "interpreter/interp_masm.hpp"
38 #include "interpreter/interpreter.hpp"
39 #include "nativeInst_aarch64.hpp"
40 #include "runtime/icache.hpp"
41 #include "runtime/javaThread.hpp"
42 #include "runtime/sharedRuntime.hpp"
43 #ifdef COMPILER1
44 #include "c1/c1_LIRAssembler.hpp"
45 #include "c1/c1_MacroAssembler.hpp"
46 #include "gc/shenandoah/c1/shenandoahBarrierSetC1.hpp"
47 #endif
48 #ifdef COMPILER2
49 #include "gc/shenandoah/c2/shenandoahBarrierSetC2.hpp"
50 #include "opto/output.hpp"
51 #endif
52
53 #define __ masm->
54
55 void ShenandoahBarrierSetAssembler::arraycopy_prologue(MacroAssembler* masm, DecoratorSet decorators, bool is_oop,
56 Register src, Register dst, Register count, RegSet saved_regs) {
57 if (is_oop) {
58 bool dest_uninitialized = (decorators & IS_DEST_UNINITIALIZED) != 0;
59 if ((ShenandoahSATBBarrier && !dest_uninitialized) || ShenandoahLoadRefBarrier) {
60
61 Label done;
62
63 // Avoid calling runtime if count == 0
64 __ cbz(count, done);
65
66 // Is GC active?
67 assert(!saved_regs.contains(rscratch1), "Sanity: about to clobber rscratch1");
68 assert(!saved_regs.contains(rscratch2), "Sanity: about to clobber rscratch2");
69 Address gc_state(rthread, in_bytes(ShenandoahThreadLocalData::gc_state_offset()));
70 __ ldrb(rscratch1, gc_state);
71 if (ShenandoahSATBBarrier && dest_uninitialized) {
72 __ tbz(rscratch1, ShenandoahHeap::HAS_FORWARDED_BITPOS, done);
73 } else {
74 __ mov(rscratch2, ShenandoahHeap::HAS_FORWARDED | ShenandoahHeap::MARKING);
75 __ tst(rscratch1, rscratch2);
76 __ br(Assembler::EQ, done);
77 }
78
79 __ push_call_clobbered_registers();
80 // If arguments are not in proper places, shuffle them.
81 // Doing this via the stack is the most straight-forward way to avoid
82 // accidentally smashing any register.
83 if (c_rarg0 != src || c_rarg1 != dst || c_rarg2 != count) {
84 __ push(RegSet::of(src), sp);
85 __ push(RegSet::of(dst), sp);
86 __ push(RegSet::of(count), sp);
87 __ pop(RegSet::of(c_rarg2), sp);
88 __ pop(RegSet::of(c_rarg1), sp);
89 __ pop(RegSet::of(c_rarg0), sp);
90 }
91 address target = nullptr;
92 if (UseCompressedOops) {
93 target = CAST_FROM_FN_PTR(address, ShenandoahRuntime::arraycopy_barrier_narrow_oop);
94 } else {
95 target = CAST_FROM_FN_PTR(address, ShenandoahRuntime::arraycopy_barrier_oop);
96 }
97 __ call_VM_leaf(target, 3);
98 __ pop_call_clobbered_registers();
99 __ bind(done);
100 }
101 }
102 }
103
104 void ShenandoahBarrierSetAssembler::arraycopy_epilogue(MacroAssembler* masm, DecoratorSet decorators, bool is_oop,
105 Register start, Register count, Register tmp) {
106 if (ShenandoahCardBarrier && is_oop) {
107 gen_write_ref_array_post_barrier(masm, decorators, start, count, tmp);
108 }
109 }
110
111 void ShenandoahBarrierSetAssembler::satb_barrier(MacroAssembler* masm,
112 Register obj,
113 Register pre_val,
114 Register thread,
115 Register tmp1,
116 Register tmp2) {
117 assert(ShenandoahSATBBarrier, "Should be checked by caller");
118 assert(thread == rthread, "must be");
119
120 Label done;
121 Label runtime;
122
123 assert_different_registers(obj, pre_val, tmp1, tmp2);
124 assert(pre_val != noreg && tmp1 != noreg && tmp2 != noreg, "expecting a register");
125
126 Address index(thread, in_bytes(ShenandoahThreadLocalData::satb_mark_queue_index_offset()));
127 Address buffer(thread, in_bytes(ShenandoahThreadLocalData::satb_mark_queue_buffer_offset()));
128
129 // Is marking active?
130 Address gc_state(thread, in_bytes(ShenandoahThreadLocalData::gc_state_offset()));
131 __ ldrb(tmp1, gc_state);
132 __ tbz(tmp1, ShenandoahHeap::MARKING_BITPOS, done);
133
134 // Do we need to load the previous value?
135 if (obj != noreg) {
136 if (UseCompressedOops) {
137 __ ldrw(pre_val, Address(obj, 0));
138 __ decode_heap_oop(pre_val);
139 } else {
140 __ ldr(pre_val, Address(obj, 0));
141 }
142 }
143
144 // Is the previous value null?
145 __ cbz(pre_val, done);
146
147 // Can we store original value in the thread's buffer?
148 // Is index == 0?
149 // (The index field is typed as size_t.)
150
151 __ ldr(tmp1, index); // tmp := *index_adr
152 __ cbz(tmp1, runtime); // tmp == 0?
153 // If yes, goto runtime
154
155 __ sub(tmp1, tmp1, wordSize); // tmp := tmp - wordSize
156 __ str(tmp1, index); // *index_adr := tmp
157 __ ldr(tmp2, buffer);
158 __ add(tmp1, tmp1, tmp2); // tmp := tmp + *buffer_adr
159
160 // Record the previous value
161 __ str(pre_val, Address(tmp1, 0));
162 __ b(done);
163
164 __ bind(runtime);
165
166 // Slow-path call
167 __ enter(/* strip_ret_addr = */ true);
168 __ push_call_clobbered_registers();
169 if (c_rarg0 != pre_val) {
170 __ mov(c_rarg0, pre_val);
171 }
172 // Calling with super_call_VM_leaf with c_rarg0 bypasses interpreter checks and avoids any moves.
173 __ super_call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::write_barrier_pre), c_rarg0);
174 __ pop_call_clobbered_registers();
175 __ leave();
176
177 __ bind(done);
178 }
179
180 void ShenandoahBarrierSetAssembler::load_reference_barrier(MacroAssembler* masm, Register dst, Address load_addr, DecoratorSet decorators) {
181 assert(ShenandoahLoadRefBarrier, "Should be enabled");
182 assert(dst != rscratch2, "need rscratch2");
183 assert_different_registers(load_addr.base(), load_addr.index(), rscratch1, rscratch2);
184
185 bool is_strong = ShenandoahBarrierSet::is_strong_access(decorators);
186 bool is_weak = ShenandoahBarrierSet::is_weak_access(decorators);
187 bool is_phantom = ShenandoahBarrierSet::is_phantom_access(decorators);
188 bool is_native = ShenandoahBarrierSet::is_native_access(decorators);
189 bool is_narrow = UseCompressedOops && !is_native;
190
191 Label heap_stable, not_cset;
192 Address gc_state(rthread, in_bytes(ShenandoahThreadLocalData::gc_state_offset()));
193 __ ldrb(rscratch2, gc_state);
194
195 // Check for heap stability
196 if (is_strong) {
197 __ tbz(rscratch2, ShenandoahHeap::HAS_FORWARDED_BITPOS, heap_stable);
198 } else {
199 Label lrb;
200 __ tbnz(rscratch2, ShenandoahHeap::WEAK_ROOTS_BITPOS, lrb);
201 __ tbz(rscratch2, ShenandoahHeap::HAS_FORWARDED_BITPOS, heap_stable);
202 __ bind(lrb);
203 }
204
205 // use r1 for load address
206 Register result_dst = dst;
207 if (dst == r1) {
208 __ mov(rscratch1, dst);
209 dst = rscratch1;
210 }
211
212 // Save r0 and r1, unless it is an output register
213 RegSet to_save = RegSet::of(r0, r1) - result_dst;
214 __ push(to_save, sp);
215 __ lea(r1, load_addr);
216 __ mov(r0, dst);
217
218 // Test for in-cset
219 if (is_strong) {
220 #if INCLUDE_CDS
221 if (AOTCodeCache::is_on_for_dump()) {
222 __ lea(rscratch2, ExternalAddress(AOTRuntimeConstants::cset_base_address()));
223 __ ldr(rscratch2, Address(rscratch2));
224 __ lea(rscratch1, ExternalAddress(AOTRuntimeConstants::grain_shift_address()));
225 __ ldrw(rscratch1, Address(rscratch1));
226 __ lsrv(rscratch1, r0, rscratch1);
227 } else
228 #endif
229 {
230 __ mov(rscratch2, ShenandoahHeap::in_cset_fast_test_addr());
231 __ lsr(rscratch1, r0, ShenandoahHeapRegion::region_size_bytes_shift_jint());
232 }
233 __ ldrb(rscratch2, Address(rscratch2, rscratch1));
234 __ tbz(rscratch2, 0, not_cset);
235 }
236
237 // Slow-path call
238 __ enter(/* strip_ret_addr = */ true);
239 __ push_call_clobbered_registers();
240 address target = nullptr;
241 if (is_strong) {
242 if (is_narrow) {
243 target = CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_strong_narrow);
244 } else {
245 target = CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_strong);
246 }
247 } else if (is_weak) {
248 if (is_narrow) {
249 target = CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_weak_narrow);
250 } else {
251 target = CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_weak);
252 }
253 } else {
254 assert(is_phantom, "only remaining strength");
255 assert(!is_narrow, "phantom access cannot be narrow");
256 target = CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_phantom);
257 }
258 // Calling with super_call_VM_leaf with c_rarg0/1 bypasses interpreter checks and avoids any moves.
259 __ super_call_VM_leaf(target, c_rarg0, c_rarg1);
260 __ mov(rscratch1, r0);
261 __ pop_call_clobbered_registers();
262 __ mov(r0, rscratch1);
263 __ leave();
264
265 __ bind(not_cset);
266
267 __ mov(result_dst, r0);
268 __ pop(to_save, sp);
269
270 __ bind(heap_stable);
271 }
272
273 //
274 // Arguments:
275 //
276 // Inputs:
277 // src: oop location to load from, might be clobbered
278 //
279 // Output:
280 // dst: oop loaded from src location
281 //
282 // Kill:
283 // rscratch1 (scratch reg)
284 //
285 // Alias:
286 // dst: rscratch1 (might use rscratch1 as temporary output register to avoid clobbering src)
287 //
288 void ShenandoahBarrierSetAssembler::load_at(MacroAssembler* masm, DecoratorSet decorators, BasicType type,
289 Register dst, Address src, Register tmp1, Register tmp2) {
290 // 1: non-reference load, no additional barrier is needed
291 if (!is_reference_type(type)) {
292 BarrierSetAssembler::load_at(masm, decorators, type, dst, src, tmp1, tmp2);
293 return;
294 }
295
296 // 2: load a reference from src location and apply LRB if needed
297 if (ShenandoahBarrierSet::need_load_reference_barrier(decorators, type)) {
298 Register result_dst = dst;
299
300 // Preserve src location for LRB
301 if (dst == src.base() || dst == src.index()) {
302 dst = rscratch1;
303 }
304 assert_different_registers(dst, src.base(), src.index());
305
306 BarrierSetAssembler::load_at(masm, decorators, type, dst, src, tmp1, tmp2);
307
308 load_reference_barrier(masm, dst, src, decorators);
309
310 if (dst != result_dst) {
311 __ mov(result_dst, dst);
312 dst = result_dst;
313 }
314 } else {
315 BarrierSetAssembler::load_at(masm, decorators, type, dst, src, tmp1, tmp2);
316 }
317
318 // 3: apply keep-alive barrier if needed
319 if (ShenandoahBarrierSet::need_keep_alive_barrier(decorators, type)) {
320 satb_barrier(masm /* masm */,
321 noreg /* obj */,
322 dst /* pre_val */,
323 rthread /* thread */,
324 tmp1 /* tmp1 */,
325 tmp2 /* tmp2 */);
326 }
327 }
328
329 void ShenandoahBarrierSetAssembler::card_barrier(MacroAssembler* masm, Register obj, Register tmp1, Register tmp2) {
330 assert(ShenandoahCardBarrier, "Should have been checked by caller");
331 assert_different_registers(obj, tmp1, tmp2);
332 assert(CardTable::dirty_card_val() == 0, "must be");
333
334 __ lsr(obj, obj, CardTable::card_shift());
335
336 Address curr_ct_holder_addr(rthread, in_bytes(ShenandoahThreadLocalData::card_table_offset()));
337 __ ldr(tmp1, curr_ct_holder_addr);
338
339 if (UseCondCardMark) {
340 Label L_already_dirty;
341 __ ldrb(tmp2, Address(obj, tmp1));
342 __ cbz(tmp2, L_already_dirty);
343 __ strb(zr, Address(obj, tmp1));
344 __ bind(L_already_dirty);
345 } else {
346 __ strb(zr, Address(obj, tmp1));
347 }
348 }
349
350 void ShenandoahBarrierSetAssembler::store_at(MacroAssembler* masm, DecoratorSet decorators, BasicType type,
351 Address dst, Register val, Register tmp1, Register tmp2, Register tmp3) {
352 // 1: non-reference types require no barriers
353 if (!is_reference_type(type)) {
354 BarrierSetAssembler::store_at(masm, decorators, type, dst, val, tmp1, tmp2, tmp3);
355 return;
356 }
357
358 // Flatten object address right away for simplicity: likely needed by barriers
359 if (dst.index() == noreg && dst.offset() == 0) {
360 if (dst.base() != tmp3) {
361 __ mov(tmp3, dst.base());
362 }
363 } else {
364 __ lea(tmp3, dst);
365 }
366
367 // 2: pre-barrier: SATB needs the previous value
368 if (ShenandoahBarrierSet::need_satb_barrier(decorators, type)) {
369 satb_barrier(masm,
370 tmp3 /* obj */,
371 tmp2 /* pre_val */,
372 rthread /* thread */,
373 tmp1 /* tmp */,
374 rscratch1 /* tmp2 */);
375 }
376
377 // Store!
378 BarrierSetAssembler::store_at(masm, decorators, type, Address(tmp3, 0), val, noreg, noreg, noreg);
379
380 // 3: post-barrier: card barrier needs store address
381 bool storing_non_null = (val != noreg);
382 if (ShenandoahBarrierSet::need_card_barrier(decorators, type) && storing_non_null) {
383 card_barrier(masm, tmp3, tmp1, tmp2);
384 }
385 }
386
387 void ShenandoahBarrierSetAssembler::try_resolve_jobject_in_native(MacroAssembler* masm, Register jni_env,
388 Register obj, Register tmp, Label& slowpath) {
389 Label done;
390 // Resolve jobject
391 BarrierSetAssembler::try_resolve_jobject_in_native(masm, jni_env, obj, tmp, slowpath);
392
393 // Check for null.
394 __ cbz(obj, done);
395
396 assert(obj != rscratch2, "need rscratch2");
397 Address gc_state(jni_env, ShenandoahThreadLocalData::gc_state_offset() - JavaThread::jni_environment_offset());
398 __ lea(rscratch2, gc_state);
399 __ ldrb(rscratch2, Address(rscratch2));
400
401 // Check for heap in evacuation phase
402 __ tbnz(rscratch2, ShenandoahHeap::EVACUATION_BITPOS, slowpath);
403
404 __ bind(done);
405 }
406
407 void ShenandoahBarrierSetAssembler::try_peek_weak_handle_in_nmethod(MacroAssembler* masm, Register weak_handle, Register obj,
408 Register tmp, Label& slow_path) {
409 assert_different_registers(weak_handle, tmp, noreg);
410 assert_different_registers(obj, tmp, noreg);
411
412 Label done;
413
414 // Peek weak handle using the standard implementation.
415 BarrierSetAssembler::try_peek_weak_handle_in_nmethod(masm, weak_handle, obj, tmp, slow_path);
416
417 // Check if the reference is null, and if it is, take the fast path.
418 __ cbz(obj, done);
419
420 Address gc_state(rthread, ShenandoahThreadLocalData::gc_state_offset());
421 __ lea(tmp, gc_state);
422 __ ldrb(tmp, __ legitimize_address(gc_state, 1, tmp));
423
424 // Check if the heap is under weak-reference/roots processing, in
425 // which case we need to take the slow path.
426 __ tbnz(tmp, ShenandoahHeap::WEAK_ROOTS_BITPOS, slow_path);
427 __ bind(done);
428 }
429
430 void ShenandoahBarrierSetAssembler::check_oop(MacroAssembler* masm, Register obj, Register tmp1, Register tmp2, Label& L_error) {
431 assert_different_registers(obj, tmp1, tmp2);
432 // Check if the oop is in the right area of memory
433 #if INCLUDE_CDS
434 if (AOTCodeCache::is_on_for_dump()) {
435 __ lea(tmp2, ExternalAddress(AOTRuntimeConstants::verify_oop_mask_address()));
436 __ ldr(tmp2, Address(tmp2));
437 __ andr(tmp1, obj, tmp2);
438 __ lea(tmp2, ExternalAddress(AOTRuntimeConstants::verify_oop_bits_address()));
439 __ ldr(tmp2, Address(tmp2));
440 } else
441 #endif
442 {
443 __ mov(tmp2, (intptr_t) Universe::verify_oop_mask());
444 __ andr(tmp1, obj, tmp2);
445 __ mov(tmp2, (intptr_t) Universe::verify_oop_bits());
446 }
447 // Compare tmp1 and tmp2. We don't use a compare
448 // instruction here because the flags register is live.
449 __ eor(tmp1, tmp1, tmp2);
450 __ cbnz(tmp1, L_error);
451
452 // This routine is sometimes called before applying GC barriers.
453 // With +COH, loading the klass may end up loading forwarding pointer instead.
454 Label L_skip;
455 if (UseCompactObjectHeaders) {
456 Address gc_state(rthread, in_bytes(ShenandoahThreadLocalData::gc_state_offset()));
457 __ ldrb(tmp1, gc_state);
458 __ tbnz(tmp1, ShenandoahHeap::HAS_FORWARDED_BITPOS, L_skip);
459 }
460
461 // Make sure klass is 'reasonable', which is not zero.
462 __ load_narrow_klass(tmp1, obj);
463 __ cbz(tmp1, L_error);
464 __ bind(L_skip);
465 }
466
467 void ShenandoahBarrierSetAssembler::gen_write_ref_array_post_barrier(MacroAssembler* masm, DecoratorSet decorators,
468 Register start, Register count, Register scratch) {
469 assert(ShenandoahCardBarrier, "Should have been checked by caller");
470
471 Label L_loop, L_done;
472 const Register end = count;
473
474 // Zero count? Nothing to do.
475 __ cbz(count, L_done);
476
477 // end = start + count << LogBytesPerHeapOop
478 // last element address to make inclusive
479 __ lea(end, Address(start, count, Address::lsl(LogBytesPerHeapOop)));
480 __ sub(end, end, BytesPerHeapOop);
481 __ lsr(start, start, CardTable::card_shift());
482 __ lsr(end, end, CardTable::card_shift());
483
484 // number of bytes to copy
485 __ sub(count, end, start);
486
487 Address curr_ct_holder_addr(rthread, in_bytes(ShenandoahThreadLocalData::card_table_offset()));
488 __ ldr(scratch, curr_ct_holder_addr);
489 __ add(start, start, scratch);
490 __ bind(L_loop);
491 __ strb(zr, Address(start, count));
492 __ subs(count, count, 1);
493 __ br(Assembler::GE, L_loop);
494 __ bind(L_done);
495 }
496
497 #undef __
498
499 address ShenandoahBarrierSetAssembler::parse_jump_address(address pc) {
500 NativeInstruction* ni = nativeInstruction_at(pc);
501 assert(ni->is_jump(), "Initial code version: GC barrier fastpath must be a jump");
502 NativeJump* jmp = nativeJump_at(pc);
503 return jmp->jump_destination();
504 }
505
506 static uint32_t encode_patchable_nop() {
507 return 0xD503201F;
508 }
509
510 static uint32_t encode_patchable_jump(address pc, address target_pc) {
511 int32_t disp = checked_cast<int32_t>((intptr_t)target_pc - (intptr_t)pc);
512 int64_t imm26 = disp >> 2;
513 guarantee(Assembler::is_simm(imm26, 26), "Maximum offset is 128MiB");
514 return 0x14000000 | (imm26 & 0x03FFFFFF);
515 }
516
517 void ShenandoahBarrierSetAssembler::insert_patchable_nop(address pc) {
518 *((uint32_t*)pc) = encode_patchable_nop();
519 assert(nativeInstruction_at(pc)->is_nop(), "Sanity");
520 if (!UseSingleICacheInvalidation) {
521 ICache::invalidate_word(pc);
522 }
523 }
524
525 void ShenandoahBarrierSetAssembler::insert_patchable_jump(address pc, address target_pc) {
526 *((uint32_t*)pc) = encode_patchable_jump(pc, target_pc);
527 if (!UseSingleICacheInvalidation) {
528 ICache::invalidate_word(pc);
529 }
530 }
531
532 bool ShenandoahBarrierSetAssembler::is_patchable_nop(address pc) {
533 return *((uint32_t*)pc) == encode_patchable_nop();
534 }
535
536 bool ShenandoahBarrierSetAssembler::is_patchable_jump(address pc, address target_pc) {
537 return *((uint32_t*)pc) == encode_patchable_jump(pc, target_pc);
538 }
539
540 #ifdef COMPILER1
541
542 #define __ ce->masm()->
543
544 void ShenandoahBarrierSetAssembler::keepalive_barrier_c1_stub(LIR_Assembler* ce, ShenandoahKeepaliveBarrierStub* stub) {
545 __ bind(*stub->entry());
546
547 ShenandoahBarrierSetC1* bs = (ShenandoahBarrierSetC1*)BarrierSet::barrier_set()->barrier_set_c1();
548
549 Register obj = stub->obj()->as_register();
550
551 if (stub->do_load()) {
552 ce->mem2reg(stub->addr(), stub->obj(), T_OBJECT, lir_patch_none, nullptr, /* wide = */ false);
553 }
554 __ cbz(obj, *stub->continuation());
555 ce->store_parameter(obj, 0);
556 __ far_call(RuntimeAddress(bs->keepalive_barrier_stub()));
557 __ b(*stub->continuation());
558 }
559
560 void ShenandoahBarrierSetAssembler::load_reference_barrier_c1_stub(LIR_Assembler* ce, ShenandoahLoadReferenceBarrierStub* stub) {
561 __ bind(*stub->entry());
562
563 ShenandoahBarrierSetC1* bs = (ShenandoahBarrierSetC1*)BarrierSet::barrier_set()->barrier_set_c1();
564
565 Register obj = stub->obj()->as_register();
566 Register addr = stub->addr()->as_pointer_register();
567 Register slow_result = stub->slow_result()->as_register();
568 assert_different_registers(obj, addr, slow_result);
569 assert(slow_result == r0, "C1 must know about our slow call result register");
570
571 ce->store_parameter(obj, 0);
572 ce->store_parameter(addr, 1);
573 __ far_call(RuntimeAddress(bs->load_reference_barrier_stub(stub->decorators())));
574 if (obj != slow_result) {
575 __ mov(obj, slow_result);
576 }
577
578 __ b(*stub->continuation());
579 }
580
581 #undef __
582
583 #define __ sasm->
584
585 void ShenandoahBarrierSetAssembler::keepalive_barrier_c1_runtime_stub(StubAssembler* sasm) {
586 __ prologue("shenandoah_keepalive_barrier", false);
587 const Register tmp_obj = r0;
588 const Register tmp1 = r1;
589 const Register tmp2 = r2;
590 __ push(RegSet::of(tmp1, tmp2, tmp_obj), sp);
591 __ load_parameter(0, tmp_obj);
592 satb_barrier(sasm, noreg, tmp_obj, rthread, tmp1, tmp2);
593 __ pop(RegSet::of(tmp1, tmp2, tmp_obj), sp);
594 __ epilogue();
595 }
596
597 void ShenandoahBarrierSetAssembler::load_reference_barrier_c1_runtime_stub(StubAssembler* sasm, DecoratorSet decorators) {
598 __ prologue("shenandoah_load_reference_barrier", false);
599 const Register tmp_obj = r0;
600 const Register tmp_addr = r1;
601 __ push(RegSet::of(tmp_addr), sp);
602 __ load_parameter(0, tmp_obj);
603 __ load_parameter(1, tmp_addr);
604 load_reference_barrier(sasm, tmp_obj, Address(tmp_addr, 0), decorators);
605 __ pop(RegSet::of(tmp_addr), sp);
606 __ epilogue();
607 }
608
609 #undef __
610
611 #endif // COMPILER1
612
613 #ifdef COMPILER2
614
615 #undef __
616 #define __ masm->
617
618
619 void ShenandoahBarrierSetAssembler::load_c2(const MachNode* node, MacroAssembler* masm, Register dst, Address src, Register tmp1, Register tmp2, bool is_narrow, bool is_acquire) {
620 // Do the actual load. This load is the candidate for implicit null check, and MUST come first.
621 if (is_narrow) {
622 if (is_acquire) {
623 assert(src.getMode() == Address::base_plus_offset && src.offset() == 0,
624 "is_acquire path requires address to be base-only");
625 __ ldarw(dst, src.base());
626 } else {
627 __ ldrw(dst, src);
628 }
629 } else {
630 if (is_acquire) {
631 assert(src.getMode() == Address::base_plus_offset && src.offset() == 0,
632 "is_acquire path requires address to be base-only");
633 __ ldar(dst, src.base());
634 } else {
635 __ ldr(dst, src);
636 }
637 }
638
639 ShenandoahBarrierStubC2::load_post(masm, node, dst, src, tmp1, tmp2, is_narrow);
640 }
641
642 void ShenandoahBarrierSetAssembler::store_c2(const MachNode* node, MacroAssembler* masm, Address dst, bool dst_narrow,
643 Register src, bool src_narrow, Register tmp1, Register tmp2, Register tmp3, bool is_volatile) {
644
645 ShenandoahBarrierStubC2::store_pre(masm, node, dst, tmp1, tmp2, tmp3, dst_narrow);
646
647 // Do the actual store
648 if (dst_narrow) {
649 if (!src_narrow) {
650 // Need to encode into rscratch, because we cannot clobber src.
651 if ((node->barrier_data() & ShenandoahBitNotNull) == 0) {
652 __ encode_heap_oop(tmp2, src);
653 } else {
654 __ encode_heap_oop_not_null(tmp2, src);
655 }
656 src = tmp2;
657 }
658
659 if (is_volatile) {
660 assert(dst.getMode() == Address::base_plus_offset && dst.offset() == 0,
661 "is_acquire path requires address to be base-only");
662 __ stlrw(src, dst.base());
663 } else {
664 __ strw(src, dst);
665 }
666 } else {
667 if (is_volatile) {
668 assert(dst.getMode() == Address::base_plus_offset && dst.offset() == 0,
669 "is_acquire path requires address to be base-only");
670 __ stlr(src, dst.base());
671 } else {
672 __ str(src, dst);
673 }
674 }
675
676 ShenandoahBarrierStubC2::store_post(masm, node, dst, tmp2, tmp3);
677 }
678
679 void ShenandoahBarrierSetAssembler::compare_and_set_c2(const MachNode* node, MacroAssembler* masm, Register res, Register addr,
680 Register oldval, Register newval, Register tmp1, Register tmp2, Register tmp3, bool exchange, bool narrow, bool weak, bool acquire) {
681 Assembler::operand_size op_size = narrow ? Assembler::word : Assembler::xword;
682
683 ShenandoahBarrierStubC2::load_store_pre(masm, node, addr, tmp1, tmp2, tmp3, narrow);
684
685 atomic_memory_order order = acquire ? memory_order_seq_cst : memory_order_release;
686
687 // CAS!
688 if (weak) {
689 __ cmpxchg_weak(addr, oldval, newval, op_size, order, exchange ? res : noreg);
690 } else {
691 __ cmpxchg(addr, oldval, newval, op_size, order, exchange ? res : noreg);
692 }
693
694 // If we need a boolean result out of CAS, set the flag appropriately and promote the result.
695 if (!exchange) {
696 assert(res != noreg, "need result register");
697 __ cset(res, Assembler::EQ);
698 }
699
700 ShenandoahBarrierStubC2::load_store_post(masm, node, Address(addr, 0), tmp2, tmp3);
701 }
702
703 void ShenandoahBarrierSetAssembler::get_and_set_c2(const MachNode* node, MacroAssembler* masm, Register preval,
704 Register newval, Register addr, Register tmp1, Register tmp2, Register tmp3, bool is_acquire) {
705 bool is_narrow = node->bottom_type()->isa_narrowoop();
706
707 ShenandoahBarrierStubC2::load_store_pre(masm, node, addr, tmp1, tmp2, tmp3, is_narrow);
708
709 if (is_narrow) {
710 if (is_acquire) {
711 __ atomic_xchgalw(preval, newval, addr);
712 } else {
713 __ atomic_xchgw(preval, newval, addr);
714 }
715 } else {
716 if (is_acquire) {
717 __ atomic_xchgal(preval, newval, addr);
718 } else {
719 __ atomic_xchg(preval, newval, addr);
720 }
721 }
722
723 ShenandoahBarrierStubC2::load_store_post(masm, node, Address(addr, 0), tmp2, tmp3);
724 }
725
726 #undef __
727 #define __ masm.
728
729 void ShenandoahBarrierStubC2::cardtable(MacroAssembler& masm, Address address, Register tmp1, Register tmp2) {
730 assert(CardTable::dirty_card_val() == 0, "must be");
731 Assembler::InlineSkippedInstructionsCounter skip_counter(&masm);
732
733 // tmp1 = card table base (holder)
734 Address curr_ct_holder_addr(rthread, in_bytes(ShenandoahThreadLocalData::card_table_offset()));
735 __ ldr(tmp1, curr_ct_holder_addr);
736
737 // tmp2 = effective address
738 __ lea(tmp2, address);
739
740 // tmp2 = &card_table[ addr >> CardTable::card_shift() ] ; card index
741 __ add(tmp2, tmp1, tmp2, Assembler::LSR, CardTable::card_shift());
742
743 if (UseCondCardMark) {
744 Label L_already_dirty;
745 __ ldrb(tmp1, Address(tmp2));
746 __ cbz(tmp1, L_already_dirty);
747 __ strb(zr, Address(tmp2));
748 __ bind(L_already_dirty);
749 } else {
750 __ strb(zr, Address(tmp2));
751 }
752 }
753
754 void ShenandoahBarrierStubC2::patchable_jump(MacroAssembler& masm, const char gc_state, bool jump_when_state, Register tmp1, Register tmp2, Label* L_target) {
755 const size_t check_size = 8;
756
757 PhaseOutput* const output = Compile::current()->output();
758 if (output->in_scratch_emit_size()) {
759 // Avoid binding L_target and emitting more branches in scratch emits.
760 // We know the patched check is exactly 1 instruction long in release,
761 // and verification adds more instructions.
762 for (size_t c = 0; c < 1 DEBUG_ONLY(+ check_size); c++) {
763 __ nop();
764 }
765 return;
766 }
767
768 #ifdef ASSERT
769 Label L_fake_entry, L_skip;
770 Address gc_state_addr(rthread, in_bytes(ShenandoahThreadLocalData::gc_state_offset()));
771
772 address check_start = __ pc();
773
774 __ ldrb(tmp1, gc_state_addr);
775 __ mov(tmp2, gc_state);
776 __ andr(tmp2, tmp1, tmp2);
777
778 // Emit the secondary jump and use it to cross-check against the actual GC state.
779 // This also checks that all interesting GC state transitions are done non-racily
780 // from the perspective of the thread executing the nmethod.
781 __ relocate(patchable_barrier_Relocation::spec(ShenandoahNMethod::encode_to_reloc(gc_state, jump_when_state)));
782 __ b(L_fake_entry);
783
784 // Currently hot-patched to NOP.
785 if (jump_when_state) {
786 __ cbz(tmp2, L_skip);
787 } else {
788 __ cbnz(tmp2, L_skip);
789 }
790 __ hlt(0);
791
792 // Currently hot-patched to JUMP.
793 __ bind(L_fake_entry);
794 if (jump_when_state) {
795 __ cbnz(tmp2, L_skip);
796 } else {
797 __ cbz(tmp2, L_skip);
798 }
799 __ hlt(0);
800
801 __ bind(L_skip);
802
803 address check_end = __ pc();
804 size_t actual_check_size = pointer_delta(check_end, check_start, Assembler::instruction_size);
805 assert(check_size == actual_check_size, "Must match: %zu != %zu", check_size, actual_check_size);
806 #endif
807
808 // Emit the unconditional branch in the first version of the method.
809 // Let the rest of runtime figure out how to manage it.
810 __ relocate(patchable_barrier_Relocation::spec(ShenandoahNMethod::encode_to_reloc(gc_state, jump_when_state)));
811 __ b(*L_target);
812 }
813
814 void ShenandoahBarrierStubC2::enter_if_gc_state(MacroAssembler& masm, const char test_state, Register tmp1, Register tmp2) {
815 Assembler::InlineSkippedInstructionsCounter skip_counter(&masm);
816 patchable_jump_if_gc_state(masm, test_state, tmp1, tmp2, entry());
817 __ bind(*continuation());
818 }
819
820 void ShenandoahBarrierStubC2::emit_code(MacroAssembler& masm) {
821 Assembler::InlineSkippedInstructionsCounter skip_counter(&masm);
822 assert(_needs_keep_alive_barrier || _needs_load_ref_barrier, "Why are you here?");
823 PhaseOutput* const output = Compile::current()->output();
824
825 // We piggyback on scratch_emit_size mode to compute the slowpath stub size.
826 // We'll use that information to decide whether we need a far jump to the
827 // stub entry point or not. In scratch_emit_size mode we don't bind entry()
828 // because otherwise it will be rebound when we later emit the instructions
829 // for real.
830 if (!output->in_scratch_emit_size()) {
831 __ bind(*entry());
832 }
833
834 // If we need to load ourselves, do it here.
835 if (_do_load) {
836 if (_narrow) {
837 __ ldrw(_obj, _addr);
838 } else {
839 __ ldr(_obj, _addr);
840 }
841 }
842
843 // If the object is null, there is no point in applying barriers.
844 maybe_far_jump_if_zero(masm, _obj);
845
846 // We need to make sure that loads done by callers survive across slow-path calls.
847 // For self-loads, we need to care about the case when both KA and LRB are enabled (rare).
848 bool needs_both_barriers = _needs_keep_alive_barrier && _needs_load_ref_barrier;
849 if (!_do_load || needs_both_barriers) {
850 preserve(_obj);
851 }
852
853 // Go for barriers. Barriers can return straight to continuation, as long
854 // as another barrier is not needed and we can reach the fastpath.
855 if (needs_both_barriers) {
856 // The Load match rule in the .ad file may have legitimized the load
857 // address using a TEMP register and in that case we need to explicitly
858 // preserve them here, because the RA does not consider TEMP as live-in,
859 // and the KA runtime call may clobber them and cause a crash on the
860 // subsequent LRB stub.
861 if (_addr.base() != noreg) {
862 preserve(_addr.base());
863 }
864 if (_addr.index() != noreg) {
865 preserve(_addr.index());
866 }
867 keepalive(masm, nullptr);
868 lrb(masm);
869 } else if (_needs_keep_alive_barrier) {
870 keepalive(masm, continuation());
871 } else if (_needs_load_ref_barrier) {
872 lrb(masm);
873 } else {
874 ShouldNotReachHere();
875 }
876 }
877
878 void ShenandoahBarrierStubC2::maybe_far_jump_if_zero(MacroAssembler& masm, Register reg) {
879 if (_needs_far_jump) {
880 Label L_short_jump;
881 __ cbnz(reg, L_short_jump);
882 __ b(*continuation());
883 __ bind(L_short_jump);
884 } else {
885 __ cbz(reg, *continuation());
886 }
887 }
888
889 void ShenandoahBarrierStubC2::keepalive(MacroAssembler& masm, Label* L_done) {
890 Address index(rthread, in_bytes(ShenandoahThreadLocalData::satb_mark_queue_index_offset()));
891 Address buffer(rthread, in_bytes(ShenandoahThreadLocalData::satb_mark_queue_buffer_offset()));
892 Label L_through, L_slowpath;
893
894 // If another barrier is enabled as well, do a check for a specific barrier.
895 if (_needs_load_ref_barrier) {
896 assert(L_done == nullptr, "Should be");
897 char state_to_check = ShenandoahHeap::MARKING;
898 patchable_jump_if_not_gc_state(masm, state_to_check, _tmp1, _tmp2, &L_through);
899 }
900
901 // Fast-path: put object into buffer.
902 // If buffer is already full, go slow.
903 __ ldr(_tmp1, index);
904 __ cbz(_tmp1, L_slowpath);
905 __ sub(_tmp1, _tmp1, wordSize);
906 __ str(_tmp1, index);
907 __ ldr(_tmp2, buffer);
908
909 // Store the object in queue.
910 // If object is narrow, we need to decode it before inserting.
911 if (_narrow) {
912 __ add(_tmp2, _tmp2, _tmp1);
913 __ decode_heap_oop_not_null(_tmp1, _obj);
914 __ str(_tmp1, Address(_tmp2));
915 } else {
916 // Buffer is 64-bit address, must be in base register.
917 __ str(_obj, Address(_tmp2, _tmp1));
918 }
919
920 // Fast-path exits here.
921 if (L_done != nullptr) {
922 __ b(*L_done);
923 } else {
924 __ b(L_through);
925 }
926
927 // Slow-path: call runtime to handle.
928 __ bind(L_slowpath);
929
930 {
931 SaveLiveRegisters slr(&masm, this);
932
933 // Go to runtime and handle the rest there.
934 __ mov(c_rarg0, _obj);
935 __ lea(lr, RuntimeAddress(keepalive_runtime_entry_addr()));
936 __ blr(lr);
937 }
938 if (L_done != nullptr) {
939 __ b(*L_done);
940 } else {
941 __ bind(L_through);
942 }
943 }
944
945 void ShenandoahBarrierStubC2::lrb(MacroAssembler& masm) {
946 Label L_slow;
947
948 // If another barrier is enabled as well, do a check for a specific barrier.
949 if (_needs_keep_alive_barrier) {
950 char state_to_check = ShenandoahHeap::HAS_FORWARDED | (_needs_load_ref_weak_barrier ? ShenandoahHeap::WEAK_ROOTS : 0);
951 patchable_jump_if_not_gc_state(masm, state_to_check, _tmp1, _tmp2, continuation());
952 }
953
954 // If weak references are being processed, weak/phantom loads need to go slow,
955 // regardless of their cset status.
956 if (_needs_load_ref_weak_barrier) {
957 char state_to_check = ShenandoahHeap::WEAK_ROOTS;
958 patchable_jump_if_gc_state(masm, state_to_check, _tmp1, _tmp2, &L_slow);
959 }
960
961 // Cset-check. Fall-through to slow if in collection set.
962 bool is_aot = AOTCodeCache::is_on_for_dump();
963 if (!is_aot) {
964 __ mov(_tmp1, ShenandoahHeap::in_cset_fast_test_addr());
965 if (_narrow) {
966 __ decode_heap_oop_not_null(_tmp2, _obj);
967 __ add(_tmp1, _tmp1, _tmp2, Assembler::LSR, ShenandoahHeapRegion::region_size_bytes_shift_jint());
968 } else {
969 __ add(_tmp1, _tmp1, _obj, Assembler::LSR, ShenandoahHeapRegion::region_size_bytes_shift_jint());
970 }
971 } else {
972 // Generating AOT code, pull the cset bitmap and region shift from AOT table.
973 if (_narrow) {
974 __ decode_heap_oop_not_null(_tmp1, _obj);
975 } else {
976 __ mov(_tmp1, _obj);
977 }
978 __ lea(_tmp2, ExternalAddress(AOTRuntimeConstants::grain_shift_address()));
979 __ ldrw(_tmp2, Address(_tmp2));
980 __ lsrv(_tmp2, _tmp1, _tmp2);
981 __ lea(_tmp1, ExternalAddress(AOTRuntimeConstants::cset_base_address()));
982 __ ldr(_tmp1, Address(_tmp1));
983 __ add(_tmp1, _tmp1, _tmp2);
984 }
985 __ ldrb(_tmp1, Address(_tmp1, 0));
986 maybe_far_jump_if_zero(masm, _tmp1);
987
988 // Slow path
989 __ bind(L_slow);
990
991 // Obj is the result, need to temporarily stop preserving it.
992 bool is_obj_preserved = is_preserved(_obj);
993 if (is_obj_preserved) {
994 dont_preserve(_obj);
995 }
996 {
997 SaveLiveRegisters slr(&masm, this);
998
999 // Shuffle in the arguments. The end result should be:
1000 // c_rarg0 <-- obj
1001 // c_rarg1 <-- lea(addr)
1002 if (c_rarg0 == _obj) {
1003 __ lea(c_rarg1, _addr);
1004 } else if (c_rarg1 == _obj) {
1005 __ mov(_tmp1, c_rarg1);
1006 __ lea(c_rarg1, _addr);
1007 __ mov(c_rarg0, _tmp1);
1008 } else {
1009 assert_different_registers(c_rarg1, _obj);
1010 __ lea(c_rarg1, _addr);
1011 __ mov(c_rarg0, _obj);
1012 }
1013
1014 // Go to runtime and handle the rest there.
1015 __ lea(lr, RuntimeAddress(lrb_runtime_entry_addr()));
1016 __ blr(lr);
1017
1018 // Save the result where needed. Narrow entries return narrowOop (32 bits)
1019 // and AAPCS does not guarantee the upper 32 bits of x0 are zero.
1020 if (_narrow) {
1021 __ movw(_obj, r0);
1022 } else if (_obj != r0) {
1023 __ mov(_obj, r0);
1024 }
1025 }
1026 if (is_obj_preserved) {
1027 preserve(_obj);
1028 }
1029
1030 __ b(*continuation());
1031 }
1032
1033 int ShenandoahBarrierStubC2::available_gp_registers() {
1034 Unimplemented(); // Not used
1035 return 0;
1036 }
1037
1038 bool ShenandoahBarrierStubC2::is_special_register(Register r) {
1039 Unimplemented(); // Not used
1040 return true;
1041 }
1042
1043 static ShenandoahBarrierSetC2State* barrier_set_state() {
1044 return reinterpret_cast<ShenandoahBarrierSetC2State*>(Compile::current()->barrier_set_state());
1045 }
1046
1047 static int get_stub_size(ShenandoahBarrierStubC2* stub) {
1048 PhaseOutput* const output = Compile::current()->output();
1049 assert(output->in_scratch_emit_size(), "only used when in scratch_emit_size.");
1050 BufferBlob* const blob = output->scratch_buffer_blob();
1051 CodeBuffer cb(blob->content_begin(), (address)output->scratch_locs_memory() - blob->content_begin());
1052 MacroAssembler masm(&cb);
1053 stub->emit_code(masm);
1054 return cb.insts_size();
1055 }
1056
1057 void ShenandoahBarrierStubC2::post_init() {
1058 // If we are in scratch emit mode we assume worst case, and force the use of
1059 // far branches.
1060 PhaseOutput* const output = Compile::current()->output();
1061 ShenandoahBarrierSetC2State* state = barrier_set_state();
1062 if (output->in_scratch_emit_size()) {
1063 state->inc_stubs_current_total_size(get_stub_size(this));
1064 _needs_far_jump = true;
1065 return;
1066 }
1067
1068 // The logic implemented in this stub only uses short jumps (cbz, cbnz) if
1069 // the aggregation of all relevant code sections of a method is less than 1MB
1070 // - 2KB. We could be more aggressive and try and compute the distance
1071 // between the fastpath branch and the stub entry but in practice not many
1072 // methods reach the 1MB size.
1073 const BufferSizingData* sizing = output->buffer_sizing_data();
1074 const int code_size = sizing->_code + state->stubs_current_total_size();
1075
1076 // Maximum backward range is 1M. Maximum forward reach is 1M - 4bytes.
1077 // Subtract 2K to be ultra conservative.
1078 const int cond_branch_max_reach = (int)(1*M - 2*K);
1079 _needs_far_jump = code_size >= cond_branch_max_reach;
1080 }
1081
1082 #endif // COMPILER2