1 /*
2 * Copyright (c) 2026, Oracle and/or its affiliates. All rights reserved.
3 * Copyright (c) 2018, 2022, Red Hat, Inc. All rights reserved.
4 * Copyright Amazon.com Inc. or its affiliates. All Rights Reserved.
5 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
6 *
7 * This code is free software; you can redistribute it and/or modify it
8 * under the terms of the GNU General Public License version 2 only, as
9 * published by the Free Software Foundation.
10 *
11 * This code is distributed in the hope that it will be useful, but WITHOUT
12 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
13 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
14 * version 2 for more details (a copy is included in the LICENSE file that
15 * accompanied this code).
16 *
17 * You should have received a copy of the GNU General Public License version
18 * 2 along with this work; if not, write to the Free Software Foundation,
19 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
20 *
21 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
22 * or visit www.oracle.com if you need additional information or have any
23 * questions.
24 *
25 */
26
27 #include "code/aotCodeCache.hpp"
28 #include "gc/shenandoah/heuristics/shenandoahHeuristics.hpp"
29 #include "gc/shenandoah/mode/shenandoahMode.hpp"
30 #include "gc/shenandoah/shenandoahBarrierSet.hpp"
31 #include "gc/shenandoah/shenandoahBarrierSetAssembler.hpp"
32 #include "gc/shenandoah/shenandoahHeap.inline.hpp"
33 #include "gc/shenandoah/shenandoahHeapRegion.hpp"
34 #include "gc/shenandoah/shenandoahRuntime.hpp"
35 #include "gc/shenandoah/shenandoahThreadLocalData.hpp"
36 #include "interpreter/interp_masm.hpp"
37 #include "interpreter/interpreter.hpp"
38 #include "runtime/javaThread.hpp"
39 #include "runtime/sharedRuntime.hpp"
40 #ifdef COMPILER1
41 #include "c1/c1_LIRAssembler.hpp"
42 #include "c1/c1_MacroAssembler.hpp"
43 #include "gc/shenandoah/c1/shenandoahBarrierSetC1.hpp"
44 #endif
45 #ifdef COMPILER2
46 #include "gc/shenandoah/c2/shenandoahBarrierSetC2.hpp"
47 #include "opto/output.hpp"
48 #endif
49
50 #define __ masm->
51
52 void ShenandoahBarrierSetAssembler::arraycopy_prologue(MacroAssembler* masm, DecoratorSet decorators, bool is_oop,
53 Register src, Register dst, Register count, RegSet saved_regs) {
54 if (is_oop) {
55 bool dest_uninitialized = (decorators & IS_DEST_UNINITIALIZED) != 0;
56 if ((ShenandoahSATBBarrier && !dest_uninitialized) || ShenandoahLoadRefBarrier) {
57
58 Label done;
59
60 // Avoid calling runtime if count == 0
61 __ cbz(count, done);
62
63 // Is GC active?
64 assert(!saved_regs.contains(rscratch1), "Sanity: about to clobber rscratch1");
65 assert(!saved_regs.contains(rscratch2), "Sanity: about to clobber rscratch2");
66 Address gc_state(rthread, in_bytes(ShenandoahThreadLocalData::gc_state_offset()));
67 __ ldrb(rscratch1, gc_state);
68 if (ShenandoahSATBBarrier && dest_uninitialized) {
69 __ tbz(rscratch1, ShenandoahHeap::HAS_FORWARDED_BITPOS, done);
70 } else {
71 __ mov(rscratch2, ShenandoahHeap::HAS_FORWARDED | ShenandoahHeap::MARKING);
72 __ tst(rscratch1, rscratch2);
73 __ br(Assembler::EQ, done);
74 }
75
76 __ push_call_clobbered_registers();
77 // If arguments are not in proper places, shuffle them.
78 // Doing this via the stack is the most straight-forward way to avoid
79 // accidentally smashing any register.
80 if (c_rarg0 != src || c_rarg1 != dst || c_rarg2 != count) {
81 __ push(RegSet::of(src), sp);
82 __ push(RegSet::of(dst), sp);
83 __ push(RegSet::of(count), sp);
84 __ pop(RegSet::of(c_rarg2), sp);
85 __ pop(RegSet::of(c_rarg1), sp);
86 __ pop(RegSet::of(c_rarg0), sp);
87 }
88 address target = nullptr;
89 if (UseCompressedOops) {
90 target = CAST_FROM_FN_PTR(address, ShenandoahRuntime::arraycopy_barrier_narrow_oop);
91 } else {
92 target = CAST_FROM_FN_PTR(address, ShenandoahRuntime::arraycopy_barrier_oop);
93 }
94 __ call_VM_leaf(target, 3);
95 __ pop_call_clobbered_registers();
96 __ bind(done);
97 }
98 }
99 }
100
101 void ShenandoahBarrierSetAssembler::arraycopy_epilogue(MacroAssembler* masm, DecoratorSet decorators, bool is_oop,
102 Register start, Register count, Register tmp) {
103 if (ShenandoahCardBarrier && is_oop) {
104 gen_write_ref_array_post_barrier(masm, decorators, start, count, tmp);
105 }
106 }
107
108 void ShenandoahBarrierSetAssembler::satb_barrier(MacroAssembler* masm,
109 Register obj,
110 Register pre_val,
111 Register thread,
112 Register tmp1,
113 Register tmp2) {
114 assert(ShenandoahSATBBarrier, "Should be checked by caller");
115 assert(thread == rthread, "must be");
116
117 Label done;
118 Label runtime;
119
120 assert_different_registers(obj, pre_val, tmp1, tmp2);
121 assert(pre_val != noreg && tmp1 != noreg && tmp2 != noreg, "expecting a register");
122
123 Address index(thread, in_bytes(ShenandoahThreadLocalData::satb_mark_queue_index_offset()));
124 Address buffer(thread, in_bytes(ShenandoahThreadLocalData::satb_mark_queue_buffer_offset()));
125
126 // Is marking active?
127 Address gc_state(thread, in_bytes(ShenandoahThreadLocalData::gc_state_offset()));
128 __ ldrb(tmp1, gc_state);
129 __ tbz(tmp1, ShenandoahHeap::MARKING_BITPOS, done);
130
131 // Do we need to load the previous value?
132 if (obj != noreg) {
133 if (UseCompressedOops) {
134 __ ldrw(pre_val, Address(obj, 0));
135 __ decode_heap_oop(pre_val);
136 } else {
137 __ ldr(pre_val, Address(obj, 0));
138 }
139 }
140
141 // Is the previous value null?
142 __ cbz(pre_val, done);
143
144 // Can we store original value in the thread's buffer?
145 // Is index == 0?
146 // (The index field is typed as size_t.)
147
148 __ ldr(tmp1, index); // tmp := *index_adr
149 __ cbz(tmp1, runtime); // tmp == 0?
150 // If yes, goto runtime
151
152 __ sub(tmp1, tmp1, wordSize); // tmp := tmp - wordSize
153 __ str(tmp1, index); // *index_adr := tmp
154 __ ldr(tmp2, buffer);
155 __ add(tmp1, tmp1, tmp2); // tmp := tmp + *buffer_adr
156
157 // Record the previous value
158 __ str(pre_val, Address(tmp1, 0));
159 __ b(done);
160
161 __ bind(runtime);
162
163 // Slow-path call
164 __ enter(/* strip_ret_addr = */ true);
165 __ push_call_clobbered_registers();
166 if (c_rarg0 != pre_val) {
167 __ mov(c_rarg0, pre_val);
168 }
169 // Calling with super_call_VM_leaf with c_rarg0 bypasses interpreter checks and avoids any moves.
170 __ super_call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::write_barrier_pre), c_rarg0);
171 __ pop_call_clobbered_registers();
172 __ leave();
173
174 __ bind(done);
175 }
176
177 void ShenandoahBarrierSetAssembler::load_reference_barrier(MacroAssembler* masm, Register dst, Address load_addr, DecoratorSet decorators) {
178 assert(ShenandoahLoadRefBarrier, "Should be enabled");
179 assert(dst != rscratch2, "need rscratch2");
180 assert_different_registers(load_addr.base(), load_addr.index(), rscratch1, rscratch2);
181
182 bool is_strong = ShenandoahBarrierSet::is_strong_access(decorators);
183 bool is_weak = ShenandoahBarrierSet::is_weak_access(decorators);
184 bool is_phantom = ShenandoahBarrierSet::is_phantom_access(decorators);
185 bool is_native = ShenandoahBarrierSet::is_native_access(decorators);
186 bool is_narrow = UseCompressedOops && !is_native;
187
188 Label heap_stable, not_cset;
189 Address gc_state(rthread, in_bytes(ShenandoahThreadLocalData::gc_state_offset()));
190 __ ldrb(rscratch2, gc_state);
191
192 // Check for heap stability
193 if (is_strong) {
194 __ tbz(rscratch2, ShenandoahHeap::HAS_FORWARDED_BITPOS, heap_stable);
195 } else {
196 Label lrb;
197 __ tbnz(rscratch2, ShenandoahHeap::WEAK_ROOTS_BITPOS, lrb);
198 __ tbz(rscratch2, ShenandoahHeap::HAS_FORWARDED_BITPOS, heap_stable);
199 __ bind(lrb);
200 }
201
202 // use r1 for load address
203 Register result_dst = dst;
204 if (dst == r1) {
205 __ mov(rscratch1, dst);
206 dst = rscratch1;
207 }
208
209 // Save r0 and r1, unless it is an output register
210 RegSet to_save = RegSet::of(r0, r1) - result_dst;
211 __ push(to_save, sp);
212 __ lea(r1, load_addr);
213 __ mov(r0, dst);
214
215 // Test for in-cset
216 if (is_strong) {
217 #if INCLUDE_CDS
218 if (AOTCodeCache::is_on_for_dump()) {
219 __ lea(rscratch2, ExternalAddress(AOTRuntimeConstants::cset_base_address()));
220 __ ldr(rscratch2, Address(rscratch2));
221 __ lea(rscratch1, ExternalAddress(AOTRuntimeConstants::grain_shift_address()));
222 __ ldrw(rscratch1, Address(rscratch1));
223 __ lsrv(rscratch1, r0, rscratch1);
224 } else
225 #endif
226 {
227 __ mov(rscratch2, ShenandoahHeap::in_cset_fast_test_addr());
228 __ lsr(rscratch1, r0, ShenandoahHeapRegion::region_size_bytes_shift_jint());
229 }
230 __ ldrb(rscratch2, Address(rscratch2, rscratch1));
231 __ tbz(rscratch2, 0, not_cset);
232 }
233
234 // Slow-path call
235 __ enter(/* strip_ret_addr = */ true);
236 __ push_call_clobbered_registers();
237 address target = nullptr;
238 if (is_strong) {
239 if (is_narrow) {
240 target = CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_strong_narrow);
241 } else {
242 target = CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_strong);
243 }
244 } else if (is_weak) {
245 if (is_narrow) {
246 target = CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_weak_narrow);
247 } else {
248 target = CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_weak);
249 }
250 } else {
251 assert(is_phantom, "only remaining strength");
252 assert(!is_narrow, "phantom access cannot be narrow");
253 target = CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_phantom);
254 }
255 // Calling with super_call_VM_leaf with c_rarg0/1 bypasses interpreter checks and avoids any moves.
256 __ super_call_VM_leaf(target, c_rarg0, c_rarg1);
257 __ mov(rscratch1, r0);
258 __ pop_call_clobbered_registers();
259 __ mov(r0, rscratch1);
260 __ leave();
261
262 __ bind(not_cset);
263
264 __ mov(result_dst, r0);
265 __ pop(to_save, sp);
266
267 __ bind(heap_stable);
268 }
269
270 //
271 // Arguments:
272 //
273 // Inputs:
274 // src: oop location to load from, might be clobbered
275 //
276 // Output:
277 // dst: oop loaded from src location
278 //
279 // Kill:
280 // rscratch1 (scratch reg)
281 //
282 // Alias:
283 // dst: rscratch1 (might use rscratch1 as temporary output register to avoid clobbering src)
284 //
285 void ShenandoahBarrierSetAssembler::load_at(MacroAssembler* masm, DecoratorSet decorators, BasicType type,
286 Register dst, Address src, Register tmp1, Register tmp2) {
287 // 1: non-reference load, no additional barrier is needed
288 if (!is_reference_type(type)) {
289 BarrierSetAssembler::load_at(masm, decorators, type, dst, src, tmp1, tmp2);
290 return;
291 }
292
293 // 2: load a reference from src location and apply LRB if needed
294 if (ShenandoahBarrierSet::need_load_reference_barrier(decorators, type)) {
295 Register result_dst = dst;
296
297 // Preserve src location for LRB
298 if (dst == src.base() || dst == src.index()) {
299 dst = rscratch1;
300 }
301 assert_different_registers(dst, src.base(), src.index());
302
303 BarrierSetAssembler::load_at(masm, decorators, type, dst, src, tmp1, tmp2);
304
305 load_reference_barrier(masm, dst, src, decorators);
306
307 if (dst != result_dst) {
308 __ mov(result_dst, dst);
309 dst = result_dst;
310 }
311 } else {
312 BarrierSetAssembler::load_at(masm, decorators, type, dst, src, tmp1, tmp2);
313 }
314
315 // 3: apply keep-alive barrier if needed
316 if (ShenandoahBarrierSet::need_keep_alive_barrier(decorators, type)) {
317 satb_barrier(masm /* masm */,
318 noreg /* obj */,
319 dst /* pre_val */,
320 rthread /* thread */,
321 tmp1 /* tmp1 */,
322 tmp2 /* tmp2 */);
323 }
324 }
325
326 void ShenandoahBarrierSetAssembler::card_barrier(MacroAssembler* masm, Register obj, Register tmp1, Register tmp2) {
327 assert(ShenandoahCardBarrier, "Should have been checked by caller");
328 assert_different_registers(obj, tmp1, tmp2);
329 assert(CardTable::dirty_card_val() == 0, "must be");
330
331 __ lsr(obj, obj, CardTable::card_shift());
332
333 Address curr_ct_holder_addr(rthread, in_bytes(ShenandoahThreadLocalData::card_table_offset()));
334 __ ldr(tmp1, curr_ct_holder_addr);
335
336 if (UseCondCardMark) {
337 Label L_already_dirty;
338 __ ldrb(tmp2, Address(obj, tmp1));
339 __ cbz(tmp2, L_already_dirty);
340 __ strb(zr, Address(obj, tmp1));
341 __ bind(L_already_dirty);
342 } else {
343 __ strb(zr, Address(obj, tmp1));
344 }
345 }
346
347 void ShenandoahBarrierSetAssembler::store_at(MacroAssembler* masm, DecoratorSet decorators, BasicType type,
348 Address dst, Register val, Register tmp1, Register tmp2, Register tmp3) {
349 // 1: non-reference types require no barriers
350 if (!is_reference_type(type)) {
351 BarrierSetAssembler::store_at(masm, decorators, type, dst, val, tmp1, tmp2, tmp3);
352 return;
353 }
354
355 // Flatten object address right away for simplicity: likely needed by barriers
356 if (dst.index() == noreg && dst.offset() == 0) {
357 if (dst.base() != tmp3) {
358 __ mov(tmp3, dst.base());
359 }
360 } else {
361 __ lea(tmp3, dst);
362 }
363
364 // 2: pre-barrier: SATB needs the previous value
365 if (ShenandoahBarrierSet::need_satb_barrier(decorators, type)) {
366 satb_barrier(masm,
367 tmp3 /* obj */,
368 tmp2 /* pre_val */,
369 rthread /* thread */,
370 tmp1 /* tmp */,
371 rscratch1 /* tmp2 */);
372 }
373
374 // Store!
375 BarrierSetAssembler::store_at(masm, decorators, type, Address(tmp3, 0), val, noreg, noreg, noreg);
376
377 // 3: post-barrier: card barrier needs store address
378 bool storing_non_null = (val != noreg);
379 if (ShenandoahBarrierSet::need_card_barrier(decorators, type) && storing_non_null) {
380 card_barrier(masm, tmp3, tmp1, tmp2);
381 }
382 }
383
384 void ShenandoahBarrierSetAssembler::try_resolve_jobject_in_native(MacroAssembler* masm, Register jni_env,
385 Register obj, Register tmp, Label& slowpath) {
386 Label done;
387 // Resolve jobject
388 BarrierSetAssembler::try_resolve_jobject_in_native(masm, jni_env, obj, tmp, slowpath);
389
390 // Check for null.
391 __ cbz(obj, done);
392
393 assert(obj != rscratch2, "need rscratch2");
394 Address gc_state(jni_env, ShenandoahThreadLocalData::gc_state_offset() - JavaThread::jni_environment_offset());
395 __ lea(rscratch2, gc_state);
396 __ ldrb(rscratch2, Address(rscratch2));
397
398 // Check for heap in evacuation phase
399 __ tbnz(rscratch2, ShenandoahHeap::EVACUATION_BITPOS, slowpath);
400
401 __ bind(done);
402 }
403
404 void ShenandoahBarrierSetAssembler::try_peek_weak_handle_in_nmethod(MacroAssembler* masm, Register weak_handle, Register obj,
405 Register tmp, Label& slow_path) {
406 assert_different_registers(weak_handle, tmp, noreg);
407 assert_different_registers(obj, tmp, noreg);
408
409 Label done;
410
411 // Peek weak handle using the standard implementation.
412 BarrierSetAssembler::try_peek_weak_handle_in_nmethod(masm, weak_handle, obj, tmp, slow_path);
413
414 // Check if the reference is null, and if it is, take the fast path.
415 __ cbz(obj, done);
416
417 Address gc_state(rthread, ShenandoahThreadLocalData::gc_state_offset());
418 __ lea(tmp, gc_state);
419 __ ldrb(tmp, __ legitimize_address(gc_state, 1, tmp));
420
421 // Check if the heap is under weak-reference/roots processing, in
422 // which case we need to take the slow path.
423 __ tbnz(tmp, ShenandoahHeap::WEAK_ROOTS_BITPOS, slow_path);
424 __ bind(done);
425 }
426
427 void ShenandoahBarrierSetAssembler::check_oop(MacroAssembler* masm, Register obj, Register tmp1, Register tmp2, Label& L_error) {
428 assert_different_registers(obj, tmp1, tmp2);
429 // Check if the oop is in the right area of memory
430 #if INCLUDE_CDS
431 if (AOTCodeCache::is_on_for_dump()) {
432 __ lea(tmp2, ExternalAddress(AOTRuntimeConstants::verify_oop_mask_address()));
433 __ ldr(tmp2, Address(tmp2));
434 __ andr(tmp1, obj, tmp2);
435 __ lea(tmp2, ExternalAddress(AOTRuntimeConstants::verify_oop_bits_address()));
436 __ ldr(tmp2, Address(tmp2));
437 } else
438 #endif
439 {
440 __ mov(tmp2, (intptr_t) Universe::verify_oop_mask());
441 __ andr(tmp1, obj, tmp2);
442 __ mov(tmp2, (intptr_t) Universe::verify_oop_bits());
443 }
444 // Compare tmp1 and tmp2. We don't use a compare
445 // instruction here because the flags register is live.
446 __ eor(tmp1, tmp1, tmp2);
447 __ cbnz(tmp1, L_error);
448
449 // This routine is sometimes called before applying GC barriers.
450 // With +COH, loading the klass may end up loading forwarding pointer instead.
451 Label L_skip;
452 if (UseCompactObjectHeaders) {
453 Address gc_state(rthread, in_bytes(ShenandoahThreadLocalData::gc_state_offset()));
454 __ ldrb(tmp1, gc_state);
455 __ tbnz(tmp1, ShenandoahHeap::HAS_FORWARDED_BITPOS, L_skip);
456 }
457
458 // Make sure klass is 'reasonable', which is not zero.
459 __ load_narrow_klass(tmp1, obj);
460 __ cbz(tmp1, L_error);
461 __ bind(L_skip);
462 }
463
464 void ShenandoahBarrierSetAssembler::gen_write_ref_array_post_barrier(MacroAssembler* masm, DecoratorSet decorators,
465 Register start, Register count, Register scratch) {
466 assert(ShenandoahCardBarrier, "Should have been checked by caller");
467
468 Label L_loop, L_done;
469 const Register end = count;
470
471 // Zero count? Nothing to do.
472 __ cbz(count, L_done);
473
474 // end = start + count << LogBytesPerHeapOop
475 // last element address to make inclusive
476 __ lea(end, Address(start, count, Address::lsl(LogBytesPerHeapOop)));
477 __ sub(end, end, BytesPerHeapOop);
478 __ lsr(start, start, CardTable::card_shift());
479 __ lsr(end, end, CardTable::card_shift());
480
481 // number of bytes to copy
482 __ sub(count, end, start);
483
484 Address curr_ct_holder_addr(rthread, in_bytes(ShenandoahThreadLocalData::card_table_offset()));
485 __ ldr(scratch, curr_ct_holder_addr);
486 __ add(start, start, scratch);
487 __ bind(L_loop);
488 __ strb(zr, Address(start, count));
489 __ subs(count, count, 1);
490 __ br(Assembler::GE, L_loop);
491 __ bind(L_done);
492 }
493
494 #undef __
495
496 #ifdef COMPILER1
497
498 #define __ ce->masm()->
499
500 void ShenandoahBarrierSetAssembler::keepalive_barrier_c1_stub(LIR_Assembler* ce, ShenandoahKeepaliveBarrierStub* stub) {
501 __ bind(*stub->entry());
502
503 ShenandoahBarrierSetC1* bs = (ShenandoahBarrierSetC1*)BarrierSet::barrier_set()->barrier_set_c1();
504
505 Register obj = stub->obj()->as_register();
506
507 if (stub->do_load()) {
508 ce->mem2reg(stub->addr(), stub->obj(), T_OBJECT, lir_patch_none, nullptr, /* wide = */ false);
509 }
510 __ cbz(obj, *stub->continuation());
511 ce->store_parameter(obj, 0);
512 __ far_call(RuntimeAddress(bs->keepalive_barrier_stub()));
513 __ b(*stub->continuation());
514 }
515
516 void ShenandoahBarrierSetAssembler::load_reference_barrier_c1_stub(LIR_Assembler* ce, ShenandoahLoadReferenceBarrierStub* stub) {
517 __ bind(*stub->entry());
518
519 ShenandoahBarrierSetC1* bs = (ShenandoahBarrierSetC1*)BarrierSet::barrier_set()->barrier_set_c1();
520
521 Register obj = stub->obj()->as_register();
522 Register addr = stub->addr()->as_pointer_register();
523 Register slow_result = stub->slow_result()->as_register();
524 assert_different_registers(obj, addr, slow_result);
525 assert(slow_result == r0, "C1 must know about our slow call result register");
526
527 ce->store_parameter(obj, 0);
528 ce->store_parameter(addr, 1);
529 __ far_call(RuntimeAddress(bs->load_reference_barrier_stub(stub->decorators())));
530 if (obj != slow_result) {
531 __ mov(obj, slow_result);
532 }
533
534 __ b(*stub->continuation());
535 }
536
537 #undef __
538
539 #define __ sasm->
540
541 void ShenandoahBarrierSetAssembler::keepalive_barrier_c1_runtime_stub(StubAssembler* sasm) {
542 __ prologue("shenandoah_keepalive_barrier", false);
543 const Register tmp_obj = r0;
544 const Register tmp1 = r1;
545 const Register tmp2 = r2;
546 __ push(RegSet::of(tmp1, tmp2, tmp_obj), sp);
547 __ load_parameter(0, tmp_obj);
548 satb_barrier(sasm, noreg, tmp_obj, rthread, tmp1, tmp2);
549 __ pop(RegSet::of(tmp1, tmp2, tmp_obj), sp);
550 __ epilogue();
551 }
552
553 void ShenandoahBarrierSetAssembler::load_reference_barrier_c1_runtime_stub(StubAssembler* sasm, DecoratorSet decorators) {
554 __ prologue("shenandoah_load_reference_barrier", false);
555 const Register tmp_obj = r0;
556 const Register tmp_addr = r1;
557 __ push(RegSet::of(tmp_addr), sp);
558 __ load_parameter(0, tmp_obj);
559 __ load_parameter(1, tmp_addr);
560 load_reference_barrier(sasm, tmp_obj, Address(tmp_addr, 0), decorators);
561 __ pop(RegSet::of(tmp_addr), sp);
562 __ epilogue();
563 }
564
565 #undef __
566
567 #endif // COMPILER1
568
569 #ifdef COMPILER2
570
571 #undef __
572 #define __ masm->
573
574
575 void ShenandoahBarrierSetAssembler::load_c2(const MachNode* node, MacroAssembler* masm, Register dst, Address src, Register tmp1, Register tmp2, bool is_narrow, bool is_acquire) {
576 // Do the actual load. This load is the candidate for implicit null check, and MUST come first.
577 if (is_narrow) {
578 if (is_acquire) {
579 assert(src.getMode() == Address::base_plus_offset && src.offset() == 0,
580 "is_acquire path requires address to be base-only");
581 __ ldarw(dst, src.base());
582 } else {
583 __ ldrw(dst, src);
584 }
585 } else {
586 if (is_acquire) {
587 assert(src.getMode() == Address::base_plus_offset && src.offset() == 0,
588 "is_acquire path requires address to be base-only");
589 __ ldar(dst, src.base());
590 } else {
591 __ ldr(dst, src);
592 }
593 }
594
595 ShenandoahBarrierStubC2::load_post(masm, node, dst, src, tmp1, tmp2, is_narrow);
596 }
597
598 void ShenandoahBarrierSetAssembler::store_c2(const MachNode* node, MacroAssembler* masm, Address dst, bool dst_narrow,
599 Register src, bool src_narrow, Register tmp1, Register tmp2, Register tmp3, bool is_volatile) {
600
601 ShenandoahBarrierStubC2::store_pre(masm, node, dst, tmp1, tmp2, tmp3, dst_narrow);
602
603 // Do the actual store
604 if (dst_narrow) {
605 if (!src_narrow) {
606 // Need to encode into rscratch, because we cannot clobber src.
607 if ((node->barrier_data() & ShenandoahBitNotNull) == 0) {
608 __ encode_heap_oop(tmp2, src);
609 } else {
610 __ encode_heap_oop_not_null(tmp2, src);
611 }
612 src = tmp2;
613 }
614
615 if (is_volatile) {
616 assert(dst.getMode() == Address::base_plus_offset && dst.offset() == 0,
617 "is_acquire path requires address to be base-only");
618 __ stlrw(src, dst.base());
619 } else {
620 __ strw(src, dst);
621 }
622 } else {
623 if (is_volatile) {
624 assert(dst.getMode() == Address::base_plus_offset && dst.offset() == 0,
625 "is_acquire path requires address to be base-only");
626 __ stlr(src, dst.base());
627 } else {
628 __ str(src, dst);
629 }
630 }
631
632 ShenandoahBarrierStubC2::store_post(masm, node, dst, tmp2, tmp3);
633 }
634
635 void ShenandoahBarrierSetAssembler::compare_and_set_c2(const MachNode* node, MacroAssembler* masm, Register res, Register addr,
636 Register oldval, Register newval, Register tmp1, Register tmp2, Register tmp3, bool exchange, bool narrow, bool weak, bool acquire) {
637 Assembler::operand_size op_size = narrow ? Assembler::word : Assembler::xword;
638
639 ShenandoahBarrierStubC2::load_store_pre(masm, node, addr, tmp1, tmp2, tmp3, narrow);
640
641 atomic_memory_order order = acquire ? memory_order_seq_cst : memory_order_release;
642
643 // CAS!
644 if (weak) {
645 __ cmpxchg_weak(addr, oldval, newval, op_size, order, exchange ? res : noreg);
646 } else {
647 __ cmpxchg(addr, oldval, newval, op_size, order, exchange ? res : noreg);
648 }
649
650 // If we need a boolean result out of CAS, set the flag appropriately and promote the result.
651 if (!exchange) {
652 assert(res != noreg, "need result register");
653 __ cset(res, Assembler::EQ);
654 }
655
656 ShenandoahBarrierStubC2::load_store_post(masm, node, Address(addr, 0), tmp2, tmp3);
657 }
658
659 void ShenandoahBarrierSetAssembler::get_and_set_c2(const MachNode* node, MacroAssembler* masm, Register preval,
660 Register newval, Register addr, Register tmp1, Register tmp2, Register tmp3, bool is_acquire) {
661 bool is_narrow = node->bottom_type()->isa_narrowoop();
662
663 ShenandoahBarrierStubC2::load_store_pre(masm, node, addr, tmp1, tmp2, tmp3, is_narrow);
664
665 if (is_narrow) {
666 if (is_acquire) {
667 __ atomic_xchgalw(preval, newval, addr);
668 } else {
669 __ atomic_xchgw(preval, newval, addr);
670 }
671 } else {
672 if (is_acquire) {
673 __ atomic_xchgal(preval, newval, addr);
674 } else {
675 __ atomic_xchg(preval, newval, addr);
676 }
677 }
678
679 ShenandoahBarrierStubC2::load_store_post(masm, node, Address(addr, 0), tmp2, tmp3);
680 }
681
682 #undef __
683 #define __ masm.
684
685 void ShenandoahBarrierStubC2::cardtable(MacroAssembler& masm, Address address, Register tmp1, Register tmp2) {
686 assert(CardTable::dirty_card_val() == 0, "must be");
687 Assembler::InlineSkippedInstructionsCounter skip_counter(&masm);
688
689 // tmp1 = card table base (holder)
690 Address curr_ct_holder_addr(rthread, in_bytes(ShenandoahThreadLocalData::card_table_offset()));
691 __ ldr(tmp1, curr_ct_holder_addr);
692
693 // tmp2 = effective address
694 __ lea(tmp2, address);
695
696 // tmp2 = &card_table[ addr >> CardTable::card_shift() ] ; card index
697 __ add(tmp2, tmp1, tmp2, Assembler::LSR, CardTable::card_shift());
698
699 if (UseCondCardMark) {
700 Label L_already_dirty;
701 __ ldrb(tmp1, Address(tmp2));
702 __ cbz(tmp1, L_already_dirty);
703 __ strb(zr, Address(tmp2));
704 __ bind(L_already_dirty);
705 } else {
706 __ strb(zr, Address(tmp2));
707 }
708 }
709
710 void ShenandoahBarrierStubC2::enter_if_gc_state(MacroAssembler& masm, const char test_state, Register tmp) {
711 Assembler::InlineSkippedInstructionsCounter skip_counter(&masm);
712 PhaseOutput* const output = Compile::current()->output();
713 Address gc_state_fast(rthread, in_bytes(ShenandoahThreadLocalData::gc_state_fast_array_offset(test_state)));
714
715 // We piggyback on scratch_emit_size mode to compute the slowpath stub size.
716 // We'll use that information to decide whether we need a far jump to the
717 // stub entry point or not. In scratch_emit_size mode we don't bind entry()
718 // because otherwise it will be rebound when we later emit the instructions
719 // for real.
720 if (_needs_far_jump) {
721 __ ldrb(tmp, gc_state_fast);
722 __ cbz(tmp, *continuation());
723 __ b(output->in_scratch_emit_size() ? *continuation() : *entry());
724 } else {
725 __ ldrb(tmp, gc_state_fast);
726 __ cbnz(tmp, output->in_scratch_emit_size() ? *continuation() : *entry());
727 }
728
729 // This is were the slowpath stub will return to or the code above will
730 // jump to if the checks are false
731 __ bind(*continuation());
732 }
733
734 void ShenandoahBarrierStubC2::emit_code(MacroAssembler& masm) {
735 Assembler::InlineSkippedInstructionsCounter skip_counter(&masm);
736 assert(_needs_keep_alive_barrier || _needs_load_ref_barrier, "Why are you here?");
737 PhaseOutput* const output = Compile::current()->output();
738
739 // We piggyback on scratch_emit_size mode to compute the slowpath stub size.
740 // We'll use that information to decide whether we need a far jump to the
741 // stub entry point or not. In scratch_emit_size mode we don't bind entry()
742 // because otherwise it will be rebound when we later emit the instructions
743 // for real.
744 if (!output->in_scratch_emit_size()) {
745 __ bind(*entry());
746 }
747
748 // If we need to load ourselves, do it here.
749 if (_do_load) {
750 if (_narrow) {
751 __ ldrw(_obj, _addr);
752 } else {
753 __ ldr(_obj, _addr);
754 }
755 }
756
757 // If the object is null, there is no point in applying barriers.
758 maybe_far_jump_if_zero(masm, _obj);
759
760 // We need to make sure that loads done by callers survive across slow-path calls.
761 // For self-loads, we need to care about the case when both KA and LRB are enabled (rare).
762 bool needs_both_barriers = _needs_keep_alive_barrier && _needs_load_ref_barrier;
763 if (!_do_load || needs_both_barriers) {
764 preserve(_obj);
765 }
766
767 // Go for barriers. Barriers can return straight to continuation, as long
768 // as another barrier is not needed and we can reach the fastpath.
769 if (needs_both_barriers) {
770 // The Load match rule in the .ad file may have legitimized the load
771 // address using a TEMP register and in that case we need to explicitly
772 // preserve them here, because the RA does not consider TEMP as live-in,
773 // and the KA runtime call may clobber them and cause a crash on the
774 // subsequent LRB stub.
775 if (_addr.base() != noreg) {
776 preserve(_addr.base());
777 }
778 if (_addr.index() != noreg) {
779 preserve(_addr.index());
780 }
781 keepalive(masm, nullptr);
782 lrb(masm);
783 } else if (_needs_keep_alive_barrier) {
784 keepalive(masm, continuation());
785 } else if (_needs_load_ref_barrier) {
786 lrb(masm);
787 } else {
788 ShouldNotReachHere();
789 }
790 }
791
792 void ShenandoahBarrierStubC2::maybe_far_jump_if_zero(MacroAssembler& masm, Register reg) {
793 if (_needs_far_jump) {
794 Label L_short_jump;
795 __ cbnz(reg, L_short_jump);
796 __ b(*continuation());
797 __ bind(L_short_jump);
798 } else {
799 __ cbz(reg, *continuation());
800 }
801 }
802
803 void ShenandoahBarrierStubC2::keepalive(MacroAssembler& masm, Label* L_done) {
804 Address gcstate(rthread, in_bytes(ShenandoahThreadLocalData::gc_state_fast_array_offset(ShenandoahHeap::MARKING)));
805 Address index(rthread, in_bytes(ShenandoahThreadLocalData::satb_mark_queue_index_offset()));
806 Address buffer(rthread, in_bytes(ShenandoahThreadLocalData::satb_mark_queue_buffer_offset()));
807 Label L_through, L_slowpath;
808
809 // If another barrier is enabled as well, do a runtime check for a specific barrier.
810 if (_needs_load_ref_barrier) {
811 assert(L_done == nullptr, "L_done is always null when _needs_load_ref_barrier is true");
812 __ ldrb(_tmp1, gcstate);
813 __ cbz(_tmp1, L_through);
814 }
815
816 // Fast-path: put object into buffer.
817 // If buffer is already full, go slow.
818 __ ldr(_tmp1, index);
819 __ cbz(_tmp1, L_slowpath);
820 __ sub(_tmp1, _tmp1, wordSize);
821 __ str(_tmp1, index);
822 __ ldr(_tmp2, buffer);
823
824 // Store the object in queue.
825 // If object is narrow, we need to decode it before inserting.
826 if (_narrow) {
827 __ add(_tmp2, _tmp2, _tmp1);
828 __ decode_heap_oop_not_null(_tmp1, _obj);
829 __ str(_tmp1, Address(_tmp2));
830 } else {
831 // Buffer is 64-bit address, must be in base register.
832 __ str(_obj, Address(_tmp2, _tmp1));
833 }
834
835 // Fast-path exits here.
836 if (L_done != nullptr) {
837 __ b(*L_done);
838 } else {
839 __ b(L_through);
840 }
841
842 // Slow-path: call runtime to handle.
843 __ bind(L_slowpath);
844
845 {
846 SaveLiveRegisters slr(&masm, this);
847
848 // Go to runtime and handle the rest there.
849 __ mov(c_rarg0, _obj);
850 __ lea(lr, RuntimeAddress(keepalive_runtime_entry_addr()));
851 __ blr(lr);
852 }
853 if (L_done != nullptr) {
854 __ b(*L_done);
855 } else {
856 __ bind(L_through);
857 }
858 }
859
860 void ShenandoahBarrierStubC2::lrb(MacroAssembler& masm) {
861 Label L_slow;
862
863 // If another barrier is enabled as well, do a runtime check for a specific barrier.
864 if (_needs_keep_alive_barrier) {
865 char state_to_check = ShenandoahHeap::HAS_FORWARDED | (_needs_load_ref_weak_barrier ? ShenandoahHeap::WEAK_ROOTS : 0);
866 Address gc_state_fast(rthread, in_bytes(ShenandoahThreadLocalData::gc_state_fast_array_offset(state_to_check)));
867 __ ldrb(_tmp1, gc_state_fast);
868 maybe_far_jump_if_zero(masm, _tmp1);
869 }
870
871 // If weak references are being processed, weak/phantom loads need to go slow,
872 // regardless of their cset status.
873 if (_needs_load_ref_weak_barrier) {
874 Address gc_state_fast(rthread, in_bytes(ShenandoahThreadLocalData::gc_state_fast_array_offset(ShenandoahHeap::WEAK_ROOTS)));
875 __ ldrb(_tmp1, gc_state_fast);
876 __ cbnz(_tmp1, L_slow);
877 }
878
879 // Cset-check. Fall-through to slow if in collection set.
880 bool is_aot = AOTCodeCache::is_on_for_dump();
881 if (!is_aot) {
882 __ mov(_tmp1, ShenandoahHeap::in_cset_fast_test_addr());
883 if (_narrow) {
884 __ decode_heap_oop_not_null(_tmp2, _obj);
885 __ add(_tmp1, _tmp1, _tmp2, Assembler::LSR, ShenandoahHeapRegion::region_size_bytes_shift_jint());
886 } else {
887 __ add(_tmp1, _tmp1, _obj, Assembler::LSR, ShenandoahHeapRegion::region_size_bytes_shift_jint());
888 }
889 } else {
890 // Generating AOT code, pull the cset bitmap and region shift from AOT table.
891 if (_narrow) {
892 __ decode_heap_oop_not_null(_tmp1, _obj);
893 } else {
894 __ mov(_tmp1, _obj);
895 }
896 __ lea(_tmp2, ExternalAddress(AOTRuntimeConstants::grain_shift_address()));
897 __ ldrw(_tmp2, Address(_tmp2));
898 __ lsrv(_tmp2, _tmp1, _tmp2);
899 __ lea(_tmp1, ExternalAddress(AOTRuntimeConstants::cset_base_address()));
900 __ ldr(_tmp1, Address(_tmp1));
901 __ add(_tmp1, _tmp1, _tmp2);
902 }
903 __ ldrb(_tmp1, Address(_tmp1, 0));
904 maybe_far_jump_if_zero(masm, _tmp1);
905
906 // Slow path
907 __ bind(L_slow);
908
909 // Obj is the result, need to temporarily stop preserving it.
910 bool is_obj_preserved = is_preserved(_obj);
911 if (is_obj_preserved) {
912 dont_preserve(_obj);
913 }
914 {
915 SaveLiveRegisters slr(&masm, this);
916
917 // Shuffle in the arguments. The end result should be:
918 // c_rarg0 <-- obj
919 // c_rarg1 <-- lea(addr)
920 if (c_rarg0 == _obj) {
921 __ lea(c_rarg1, _addr);
922 } else if (c_rarg1 == _obj) {
923 __ mov(_tmp1, c_rarg1);
924 __ lea(c_rarg1, _addr);
925 __ mov(c_rarg0, _tmp1);
926 } else {
927 assert_different_registers(c_rarg1, _obj);
928 __ lea(c_rarg1, _addr);
929 __ mov(c_rarg0, _obj);
930 }
931
932 // Go to runtime and handle the rest there.
933 __ lea(lr, RuntimeAddress(lrb_runtime_entry_addr()));
934 __ blr(lr);
935
936 // Save the result where needed. Narrow entries return narrowOop (32 bits)
937 // and AAPCS does not guarantee the upper 32 bits of x0 are zero.
938 if (_narrow) {
939 __ movw(_obj, r0);
940 } else if (_obj != r0) {
941 __ mov(_obj, r0);
942 }
943 }
944 if (is_obj_preserved) {
945 preserve(_obj);
946 }
947
948 __ b(*continuation());
949 }
950
951 int ShenandoahBarrierStubC2::available_gp_registers() {
952 Unimplemented(); // Not used
953 return 0;
954 }
955
956 bool ShenandoahBarrierStubC2::is_special_register(Register r) {
957 Unimplemented(); // Not used
958 return true;
959 }
960
961 static ShenandoahBarrierSetC2State* barrier_set_state() {
962 return reinterpret_cast<ShenandoahBarrierSetC2State*>(Compile::current()->barrier_set_state());
963 }
964
965 static int get_stub_size(ShenandoahBarrierStubC2* stub) {
966 PhaseOutput* const output = Compile::current()->output();
967 assert(output->in_scratch_emit_size(), "only used when in scratch_emit_size.");
968 BufferBlob* const blob = output->scratch_buffer_blob();
969 CodeBuffer cb(blob->content_begin(), (address)output->scratch_locs_memory() - blob->content_begin());
970 MacroAssembler masm(&cb);
971 stub->emit_code(masm);
972 return cb.insts_size();
973 }
974
975 void ShenandoahBarrierStubC2::post_init() {
976 // If we are in scratch emit mode we assume worst case, and force the use of
977 // far branches.
978 PhaseOutput* const output = Compile::current()->output();
979 ShenandoahBarrierSetC2State* state = barrier_set_state();
980 if (output->in_scratch_emit_size()) {
981 state->inc_stubs_current_total_size(get_stub_size(this));
982 _needs_far_jump = true;
983 return;
984 }
985
986 // The logic implemented in this stub only uses short jumps (cbz, cbnz) if
987 // the aggregation of all relevant code sections of a method is less than 1MB
988 // - 2KB. We could be more aggressive and try and compute the distance
989 // between the fastpath branch and the stub entry but in practice not many
990 // methods reach the 1MB size.
991 const BufferSizingData* sizing = output->buffer_sizing_data();
992 const int code_size = sizing->_code + state->stubs_current_total_size();
993
994 // Maximum backward range is 1M. Maximum forward reach is 1M - 4bytes.
995 // Subtract 2K to be ultra conservative.
996 const int cond_branch_max_reach = (int)(1*M - 2*K);
997 _needs_far_jump = code_size >= cond_branch_max_reach;
998 }
999
1000 #endif // COMPILER2