1 /*
2 * Copyright (c) 2026, Oracle and/or its affiliates. All rights reserved.
3 * Copyright (c) 2018, 2022, Red Hat, Inc. All rights reserved.
4 * Copyright Amazon.com Inc. or its affiliates. All Rights Reserved.
5 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
6 *
7 * This code is free software; you can redistribute it and/or modify it
8 * under the terms of the GNU General Public License version 2 only, as
9 * published by the Free Software Foundation.
10 *
11 * This code is distributed in the hope that it will be useful, but WITHOUT
12 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
13 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
14 * version 2 for more details (a copy is included in the LICENSE file that
15 * accompanied this code).
16 *
17 * You should have received a copy of the GNU General Public License version
18 * 2 along with this work; if not, write to the Free Software Foundation,
19 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
20 *
21 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
22 * or visit www.oracle.com if you need additional information or have any
23 * questions.
24 *
25 */
26
27 #include "gc/shenandoah/heuristics/shenandoahHeuristics.hpp"
28 #include "gc/shenandoah/mode/shenandoahMode.hpp"
29 #include "gc/shenandoah/shenandoahBarrierSet.hpp"
30 #include "gc/shenandoah/shenandoahBarrierSetAssembler.hpp"
31 #include "gc/shenandoah/shenandoahForwarding.hpp"
32 #include "gc/shenandoah/shenandoahHeap.inline.hpp"
33 #include "gc/shenandoah/shenandoahHeapRegion.hpp"
34 #include "gc/shenandoah/shenandoahRuntime.hpp"
35 #include "gc/shenandoah/shenandoahThreadLocalData.hpp"
36 #include "interpreter/interp_masm.hpp"
37 #include "interpreter/interpreter.hpp"
38 #include "runtime/javaThread.hpp"
39 #include "runtime/sharedRuntime.hpp"
40 #ifdef COMPILER1
41 #include "c1/c1_LIRAssembler.hpp"
42 #include "c1/c1_MacroAssembler.hpp"
43 #include "gc/shenandoah/c1/shenandoahBarrierSetC1.hpp"
44 #endif
45 #ifdef COMPILER2
46 #include "gc/shenandoah/c2/shenandoahBarrierSetC2.hpp"
47 #include "opto/output.hpp"
48 #endif
49
50 #define __ masm->
51
52 void ShenandoahBarrierSetAssembler::arraycopy_prologue(MacroAssembler* masm, DecoratorSet decorators, bool is_oop,
53 Register src, Register dst, Register count, RegSet saved_regs) {
54 if (is_oop) {
55 bool dest_uninitialized = (decorators & IS_DEST_UNINITIALIZED) != 0;
56 if ((ShenandoahSATBBarrier && !dest_uninitialized) || ShenandoahLoadRefBarrier) {
57
58 Label done;
59
60 // Avoid calling runtime if count == 0
61 __ cbz(count, done);
62
63 // Is GC active?
64 Address gc_state(rthread, in_bytes(ShenandoahThreadLocalData::gc_state_offset()));
65 __ ldrb(rscratch1, gc_state);
66 if (ShenandoahSATBBarrier && dest_uninitialized) {
67 __ tbz(rscratch1, ShenandoahHeap::HAS_FORWARDED_BITPOS, done);
68 } else {
69 __ mov(rscratch2, ShenandoahHeap::HAS_FORWARDED | ShenandoahHeap::MARKING);
70 __ tst(rscratch1, rscratch2);
71 __ br(Assembler::EQ, done);
72 }
73
74 __ push(saved_regs, sp);
75 if (UseCompressedOops) {
76 __ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::arraycopy_barrier_narrow_oop), src, dst, count);
77 } else {
78 __ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::arraycopy_barrier_oop), src, dst, count);
79 }
80 __ pop(saved_regs, sp);
81 __ bind(done);
82 }
83 }
84 }
85
86 void ShenandoahBarrierSetAssembler::arraycopy_epilogue(MacroAssembler* masm, DecoratorSet decorators, bool is_oop,
87 Register start, Register count, Register tmp) {
88 if (ShenandoahCardBarrier && is_oop) {
89 gen_write_ref_array_post_barrier(masm, decorators, start, count, tmp);
90 }
91 }
92
93 void ShenandoahBarrierSetAssembler::satb_barrier(MacroAssembler* masm,
94 Register obj,
95 Register pre_val,
96 Register thread,
97 Register tmp1,
98 Register tmp2,
99 bool tosca_live,
100 bool expand_call) {
101 assert(ShenandoahSATBBarrier, "Should be checked by caller");
102
103 // If expand_call is true then we expand the call_VM_leaf macro
104 // directly to skip generating the check by
105 // InterpreterMacroAssembler::call_VM_leaf_base that checks _last_sp.
106
107 assert(thread == rthread, "must be");
108
109 Label done;
110 Label runtime;
111
112 assert_different_registers(obj, pre_val, tmp1, tmp2);
113 assert(pre_val != noreg && tmp1 != noreg && tmp2 != noreg, "expecting a register");
114
115 Address index(thread, in_bytes(ShenandoahThreadLocalData::satb_mark_queue_index_offset()));
116 Address buffer(thread, in_bytes(ShenandoahThreadLocalData::satb_mark_queue_buffer_offset()));
117
118 // Is marking active?
119 Address gc_state(thread, in_bytes(ShenandoahThreadLocalData::gc_state_offset()));
120 __ ldrb(tmp1, gc_state);
121 __ tbz(tmp1, ShenandoahHeap::MARKING_BITPOS, done);
122
123 // Do we need to load the previous value?
124 if (obj != noreg) {
125 __ load_heap_oop(pre_val, Address(obj, 0), noreg, noreg, AS_RAW);
126 }
127
128 // Is the previous value null?
129 __ cbz(pre_val, done);
130
131 // Can we store original value in the thread's buffer?
132 // Is index == 0?
133 // (The index field is typed as size_t.)
134
135 __ ldr(tmp1, index); // tmp := *index_adr
136 __ cbz(tmp1, runtime); // tmp == 0?
137 // If yes, goto runtime
138
139 __ sub(tmp1, tmp1, wordSize); // tmp := tmp - wordSize
140 __ str(tmp1, index); // *index_adr := tmp
141 __ ldr(tmp2, buffer);
142 __ add(tmp1, tmp1, tmp2); // tmp := tmp + *buffer_adr
143
144 // Record the previous value
145 __ str(pre_val, Address(tmp1, 0));
146 __ b(done);
147
148 __ bind(runtime);
149 // save the live input values
150 RegSet saved = RegSet::of(pre_val);
151 if (tosca_live) saved += RegSet::of(r0);
152 if (obj != noreg) saved += RegSet::of(obj);
153
154 __ push(saved, sp);
155
156 // Calling the runtime using the regular call_VM_leaf mechanism generates
157 // code (generated by InterpreterMacroAssember::call_VM_leaf_base)
158 // that checks that the *(rfp+frame::interpreter_frame_last_sp) == nullptr.
159 //
160 // If we care generating the pre-barrier without a frame (e.g. in the
161 // intrinsified Reference.get() routine) then rfp might be pointing to
162 // the caller frame and so this check will most likely fail at runtime.
163 //
164 // Expanding the call directly bypasses the generation of the check.
165 // So when we do not have have a full interpreter frame on the stack
166 // expand_call should be passed true.
167
168 if (expand_call) {
169 assert(pre_val != c_rarg1, "smashed arg");
170 __ super_call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::write_barrier_pre), pre_val);
171 } else {
172 __ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::write_barrier_pre), pre_val);
173 }
174
175 __ pop(saved, sp);
176
177 __ bind(done);
178 }
179
180 void ShenandoahBarrierSetAssembler::resolve_forward_pointer(MacroAssembler* masm, Register dst, Register tmp) {
181 assert(ShenandoahLoadRefBarrier || ShenandoahCASBarrier, "Should be enabled");
182 Label is_null;
183 __ cbz(dst, is_null);
184 resolve_forward_pointer_not_null(masm, dst, tmp);
185 __ bind(is_null);
186 }
187
188 // IMPORTANT: This must preserve all registers, even rscratch1 and rscratch2, except those explicitly
189 // passed in.
190 void ShenandoahBarrierSetAssembler::resolve_forward_pointer_not_null(MacroAssembler* masm, Register dst, Register tmp) {
191 assert(ShenandoahLoadRefBarrier || ShenandoahCASBarrier, "Should be enabled");
192 // The below loads the mark word, checks if the lowest two bits are
193 // set, and if so, clear the lowest two bits and copy the result
194 // to dst. Otherwise it leaves dst alone.
195 // Implementing this is surprisingly awkward. I do it here by:
196 // - Inverting the mark word
197 // - Test lowest two bits == 0
198 // - If so, set the lowest two bits
199 // - Invert the result back, and copy to dst
200
201 bool borrow_reg = (tmp == noreg);
202 if (borrow_reg) {
203 // No free registers available. Make one useful.
204 tmp = rscratch1;
205 if (tmp == dst) {
206 tmp = rscratch2;
207 }
208 __ push(RegSet::of(tmp), sp);
209 }
210
211 assert_different_registers(tmp, dst);
212
213 Label done;
214 __ ldr(tmp, Address(dst, oopDesc::mark_offset_in_bytes()));
215 __ eon(tmp, tmp, zr);
216 __ ands(zr, tmp, markWord::lock_mask_in_place);
217 __ br(Assembler::NE, done);
218 __ orr(tmp, tmp, markWord::marked_value);
219 __ eon(dst, tmp, zr);
220 __ bind(done);
221
222 if (borrow_reg) {
223 __ pop(RegSet::of(tmp), sp);
224 }
225 }
226
227 void ShenandoahBarrierSetAssembler::load_reference_barrier(MacroAssembler* masm, Register dst, Address load_addr, DecoratorSet decorators) {
228 assert(ShenandoahLoadRefBarrier, "Should be enabled");
229 assert(dst != rscratch2, "need rscratch2");
230 assert_different_registers(load_addr.base(), load_addr.index(), rscratch1, rscratch2);
231
232 bool is_strong = ShenandoahBarrierSet::is_strong_access(decorators);
233 bool is_weak = ShenandoahBarrierSet::is_weak_access(decorators);
234 bool is_phantom = ShenandoahBarrierSet::is_phantom_access(decorators);
235 bool is_native = ShenandoahBarrierSet::is_native_access(decorators);
236 bool is_narrow = UseCompressedOops && !is_native;
237
238 Label heap_stable, not_cset;
239 __ enter(/*strip_ret_addr*/true);
240 Address gc_state(rthread, in_bytes(ShenandoahThreadLocalData::gc_state_offset()));
241 __ ldrb(rscratch2, gc_state);
242
243 // Check for heap stability
244 if (is_strong) {
245 __ tbz(rscratch2, ShenandoahHeap::HAS_FORWARDED_BITPOS, heap_stable);
246 } else {
247 Label lrb;
248 __ tbnz(rscratch2, ShenandoahHeap::WEAK_ROOTS_BITPOS, lrb);
249 __ tbz(rscratch2, ShenandoahHeap::HAS_FORWARDED_BITPOS, heap_stable);
250 __ bind(lrb);
251 }
252
253 // use r1 for load address
254 Register result_dst = dst;
255 if (dst == r1) {
256 __ mov(rscratch1, dst);
257 dst = rscratch1;
258 }
259
260 // Save r0 and r1, unless it is an output register
261 RegSet to_save = RegSet::of(r0, r1) - result_dst;
262 __ push(to_save, sp);
263 __ lea(r1, load_addr);
264 __ mov(r0, dst);
265
266 // Test for in-cset
267 if (is_strong) {
268 __ mov(rscratch2, ShenandoahHeap::in_cset_fast_test_addr());
269 __ lsr(rscratch1, r0, ShenandoahHeapRegion::region_size_bytes_shift_jint());
270 __ ldrb(rscratch2, Address(rscratch2, rscratch1));
271 __ tbz(rscratch2, 0, not_cset);
272 }
273
274 __ push_call_clobbered_registers();
275 if (is_strong) {
276 if (is_narrow) {
277 __ mov(lr, CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_strong_narrow));
278 } else {
279 __ mov(lr, CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_strong));
280 }
281 } else if (is_weak) {
282 if (is_narrow) {
283 __ mov(lr, CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_weak_narrow));
284 } else {
285 __ mov(lr, CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_weak));
286 }
287 } else {
288 assert(is_phantom, "only remaining strength");
289 assert(!is_narrow, "phantom access cannot be narrow");
290 // AOT saved adapters need relocation for this call.
291 __ lea(lr, RuntimeAddress(CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_phantom)));
292 }
293 __ blr(lr);
294 __ mov(rscratch1, r0);
295 __ pop_call_clobbered_registers();
296 __ mov(r0, rscratch1);
297
298 __ bind(not_cset);
299
300 __ mov(result_dst, r0);
301 __ pop(to_save, sp);
302
303 __ bind(heap_stable);
304 __ leave();
305 }
306
307 //
308 // Arguments:
309 //
310 // Inputs:
311 // src: oop location to load from, might be clobbered
312 //
313 // Output:
314 // dst: oop loaded from src location
315 //
316 // Kill:
317 // rscratch1 (scratch reg)
318 //
319 // Alias:
320 // dst: rscratch1 (might use rscratch1 as temporary output register to avoid clobbering src)
321 //
322 void ShenandoahBarrierSetAssembler::load_at(MacroAssembler* masm, DecoratorSet decorators, BasicType type,
323 Register dst, Address src, Register tmp1, Register tmp2) {
324 // 1: non-reference load, no additional barrier is needed
325 if (!is_reference_type(type)) {
326 BarrierSetAssembler::load_at(masm, decorators, type, dst, src, tmp1, tmp2);
327 return;
328 }
329
330 // 2: load a reference from src location and apply LRB if needed
331 if (ShenandoahBarrierSet::need_load_reference_barrier(decorators, type)) {
332 Register result_dst = dst;
333
334 // Preserve src location for LRB
335 if (dst == src.base() || dst == src.index()) {
336 dst = rscratch1;
337 }
338 assert_different_registers(dst, src.base(), src.index());
339
340 BarrierSetAssembler::load_at(masm, decorators, type, dst, src, tmp1, tmp2);
341
342 load_reference_barrier(masm, dst, src, decorators);
343
344 if (dst != result_dst) {
345 __ mov(result_dst, dst);
346 dst = result_dst;
347 }
348 } else {
349 BarrierSetAssembler::load_at(masm, decorators, type, dst, src, tmp1, tmp2);
350 }
351
352 // 3: apply keep-alive barrier if needed
353 if (ShenandoahBarrierSet::need_keep_alive_barrier(decorators, type)) {
354 __ enter(/*strip_ret_addr*/true);
355 __ push_call_clobbered_registers();
356 satb_barrier(masm /* masm */,
357 noreg /* obj */,
358 dst /* pre_val */,
359 rthread /* thread */,
360 tmp1 /* tmp1 */,
361 tmp2 /* tmp2 */,
362 true /* tosca_live */,
363 true /* expand_call */);
364 __ pop_call_clobbered_registers();
365 __ leave();
366 }
367 }
368
369 void ShenandoahBarrierSetAssembler::card_barrier(MacroAssembler* masm, Register obj) {
370 assert(ShenandoahCardBarrier, "Should have been checked by caller");
371
372 __ lsr(obj, obj, CardTable::card_shift());
373
374 assert(CardTable::dirty_card_val() == 0, "must be");
375
376 Address curr_ct_holder_addr(rthread, in_bytes(ShenandoahThreadLocalData::card_table_offset()));
377 __ ldr(rscratch1, curr_ct_holder_addr);
378
379 if (UseCondCardMark) {
380 Label L_already_dirty;
381 __ ldrb(rscratch2, Address(obj, rscratch1));
382 __ cbz(rscratch2, L_already_dirty);
383 __ strb(zr, Address(obj, rscratch1));
384 __ bind(L_already_dirty);
385 } else {
386 __ strb(zr, Address(obj, rscratch1));
387 }
388 }
389
390 void ShenandoahBarrierSetAssembler::store_at(MacroAssembler* masm, DecoratorSet decorators, BasicType type,
391 Address dst, Register val, Register tmp1, Register tmp2, Register tmp3) {
392 // 1: non-reference types require no barriers
393 if (!is_reference_type(type)) {
394 BarrierSetAssembler::store_at(masm, decorators, type, dst, val, tmp1, tmp2, tmp3);
395 return;
396 }
397
398 // Flatten object address right away for simplicity: likely needed by barriers
399 if (dst.index() == noreg && dst.offset() == 0) {
400 if (dst.base() != tmp3) {
401 __ mov(tmp3, dst.base());
402 }
403 } else {
404 __ lea(tmp3, dst);
405 }
406
407 bool storing_non_null = (val != noreg);
408
409 // 2: pre-barrier: SATB needs the previous value
410 if (ShenandoahBarrierSet::need_satb_barrier(decorators, type)) {
411 satb_barrier(masm,
412 tmp3 /* obj */,
413 tmp2 /* pre_val */,
414 rthread /* thread */,
415 tmp1 /* tmp */,
416 rscratch1 /* tmp2 */,
417 storing_non_null /* tosca_live */,
418 false /* expand_call */);
419 }
420
421 // Store!
422 BarrierSetAssembler::store_at(masm, decorators, type, Address(tmp3, 0), val, noreg, noreg, noreg);
423
424 // 3: post-barrier: card barrier needs store address
425 if (ShenandoahBarrierSet::need_card_barrier(decorators, type) && storing_non_null) {
426 card_barrier(masm, tmp3);
427 }
428 }
429
430 void ShenandoahBarrierSetAssembler::try_resolve_jobject_in_native(MacroAssembler* masm, Register jni_env,
431 Register obj, Register tmp, Label& slowpath) {
432 Label done;
433 // Resolve jobject
434 BarrierSetAssembler::try_resolve_jobject_in_native(masm, jni_env, obj, tmp, slowpath);
435
436 // Check for null.
437 __ cbz(obj, done);
438
439 assert(obj != rscratch2, "need rscratch2");
440 Address gc_state(jni_env, ShenandoahThreadLocalData::gc_state_offset() - JavaThread::jni_environment_offset());
441 __ lea(rscratch2, gc_state);
442 __ ldrb(rscratch2, Address(rscratch2));
443
444 // Check for heap in evacuation phase
445 __ tbnz(rscratch2, ShenandoahHeap::EVACUATION_BITPOS, slowpath);
446
447 __ bind(done);
448 }
449
450 #ifdef COMPILER2
451 void ShenandoahBarrierSetAssembler::try_resolve_weak_handle_in_c2(MacroAssembler* masm, Register obj,
452 Register tmp, Label& slow_path) {
453 assert_different_registers(obj, tmp);
454
455 Label done;
456
457 // Resolve weak handle using the standard implementation.
458 BarrierSetAssembler::try_resolve_weak_handle_in_c2(masm, obj, tmp, slow_path);
459
460 // Check if the reference is null, and if it is, take the fast path.
461 __ cbz(obj, done);
462
463 Address gc_state(rthread, ShenandoahThreadLocalData::gc_state_offset());
464 __ lea(tmp, gc_state);
465 __ ldrb(tmp, __ legitimize_address(gc_state, 1, tmp));
466
467 // Check if the heap is under weak-reference/roots processing, in
468 // which case we need to take the slow path.
469 __ tbnz(tmp, ShenandoahHeap::WEAK_ROOTS_BITPOS, slow_path);
470 __ bind(done);
471 }
472 #endif
473
474 // Special Shenandoah CAS implementation that handles false negatives due
475 // to concurrent evacuation. The service is more complex than a
476 // traditional CAS operation because the CAS operation is intended to
477 // succeed if the reference at addr exactly matches expected or if the
478 // reference at addr holds a pointer to a from-space object that has
479 // been relocated to the location named by expected. There are two
480 // races that must be addressed:
481 // a) A parallel thread may mutate the contents of addr so that it points
482 // to a different object. In this case, the CAS operation should fail.
483 // b) A parallel thread may heal the contents of addr, replacing a
484 // from-space pointer held in addr with the to-space pointer
485 // representing the new location of the object.
486 // Upon entry to cmpxchg_oop, it is assured that new_val equals null
487 // or it refers to an object that is not being evacuated out of
488 // from-space, or it refers to the to-space version of an object that
489 // is being evacuated out of from-space.
490 //
491 // By default the value held in the result register following execution
492 // of the generated code sequence is 0 to indicate failure of CAS,
493 // non-zero to indicate success. If is_cae, the result is the value most
494 // recently fetched from addr rather than a boolean success indicator.
495 //
496 // Clobbers rscratch1, rscratch2
497 void ShenandoahBarrierSetAssembler::cmpxchg_oop(MacroAssembler* masm,
498 Register addr,
499 Register expected,
500 Register new_val,
501 bool acquire, bool release,
502 bool is_cae,
503 Register result) {
504 Register tmp1 = rscratch1;
505 Register tmp2 = rscratch2;
506 bool is_narrow = UseCompressedOops;
507 Assembler::operand_size size = is_narrow ? Assembler::word : Assembler::xword;
508
509 assert_different_registers(addr, expected, tmp1, tmp2);
510 assert_different_registers(addr, new_val, tmp1, tmp2);
511
512 Label step4, done;
513
514 // There are two ways to reach this label. Initial entry into the
515 // cmpxchg_oop code expansion starts at step1 (which is equivalent
516 // to label step4). Additionally, in the rare case that four steps
517 // are required to perform the requested operation, the fourth step
518 // is the same as the first. On a second pass through step 1,
519 // control may flow through step 2 on its way to failure. It will
520 // not flow from step 2 to step 3 since we are assured that the
521 // memory at addr no longer holds a from-space pointer.
522 //
523 // The comments that immediately follow the step4 label apply only
524 // to the case in which control reaches this label by branch from
525 // step 3.
526
527 __ bind (step4);
528
529 // Step 4. CAS has failed because the value most recently fetched
530 // from addr is no longer the from-space pointer held in tmp2. If a
531 // different thread replaced the in-memory value with its equivalent
532 // to-space pointer, then CAS may still be able to succeed. The
533 // value held in the expected register has not changed.
534 //
535 // It is extremely rare we reach this point. For this reason, the
536 // implementation opts for smaller rather than potentially faster
537 // code. Ultimately, smaller code for this rare case most likely
538 // delivers higher overall throughput by enabling improved icache
539 // performance.
540
541 // Step 1. Fast-path.
542 //
543 // Try to CAS with given arguments. If successful, then we are done.
544 //
545 // No label required for step 1.
546
547 __ cmpxchg(addr, expected, new_val, size, acquire, release, false, tmp2);
548 // EQ flag set iff success. tmp2 holds value fetched.
549
550 // If expected equals null but tmp2 does not equal null, the
551 // following branches to done to report failure of CAS. If both
552 // expected and tmp2 equal null, the following branches to done to
553 // report success of CAS. There's no need for a special test of
554 // expected equal to null.
555
556 __ br(Assembler::EQ, done);
557 // if CAS failed, fall through to step 2
558
559 // Step 2. CAS has failed because the value held at addr does not
560 // match expected. This may be a false negative because the value fetched
561 // from addr (now held in tmp2) may be a from-space pointer to the
562 // original copy of same object referenced by to-space pointer expected.
563 //
564 // To resolve this, it suffices to find the forward pointer associated
565 // with fetched value. If this matches expected, retry CAS with new
566 // parameters. If this mismatches, then we have a legitimate
567 // failure, and we're done.
568 //
569 // No need for step2 label.
570
571 // overwrite tmp1 with from-space pointer fetched from memory
572 __ mov(tmp1, tmp2);
573
574 if (is_narrow) {
575 // Decode tmp1 in order to resolve its forward pointer
576 __ decode_heap_oop(tmp1, tmp1);
577 }
578 resolve_forward_pointer(masm, tmp1);
579 // Encode tmp1 to compare against expected.
580 __ encode_heap_oop(tmp1, tmp1);
581
582 // Does forwarded value of fetched from-space pointer match original
583 // value of expected? If tmp1 holds null, this comparison will fail
584 // because we know from step1 that expected is not null. There is
585 // no need for a separate test for tmp1 (the value originally held
586 // in memory) equal to null.
587 __ cmp(tmp1, expected);
588
589 // If not, then the failure was legitimate and we're done.
590 // Branching to done with NE condition denotes failure.
591 __ br(Assembler::NE, done);
592
593 // Fall through to step 3. No need for step3 label.
594
595 // Step 3. We've confirmed that the value originally held in memory
596 // (now held in tmp2) pointed to from-space version of original
597 // expected value. Try the CAS again with the from-space expected
598 // value. If it now succeeds, we're good.
599 //
600 // Note: tmp2 holds encoded from-space pointer that matches to-space
601 // object residing at expected. tmp2 is the new "expected".
602
603 // Note that macro implementation of __cmpxchg cannot use same register
604 // tmp2 for result and expected since it overwrites result before it
605 // compares result with expected.
606 __ cmpxchg(addr, tmp2, new_val, size, acquire, release, false, noreg);
607 // EQ flag set iff success. tmp2 holds value fetched, tmp1 (rscratch1) clobbered.
608
609 // If fetched value did not equal the new expected, this could
610 // still be a false negative because some other thread may have
611 // newly overwritten the memory value with its to-space equivalent.
612 __ br(Assembler::NE, step4);
613
614 if (is_cae) {
615 // We're falling through to done to indicate success. Success
616 // with is_cae is denoted by returning the value of expected as
617 // result.
618 __ mov(tmp2, expected);
619 }
620
621 __ bind(done);
622 // At entry to done, the Z (EQ) flag is on iff if the CAS
623 // operation was successful. Additionally, if is_cae, tmp2 holds
624 // the value most recently fetched from addr. In this case, success
625 // is denoted by tmp2 matching expected.
626
627 if (is_cae) {
628 __ mov(result, tmp2);
629 } else {
630 __ cset(result, Assembler::EQ);
631 }
632 }
633
634 #ifdef COMPILER2
635 #undef __
636 #define __ masm.
637
638 bool ShenandoahBarrierStubC2::push_save_register_if_live(MacroAssembler& masm, Register reg) {
639 if (is_live(reg)) {
640 push_save_register(masm, reg);
641 return true;
642 } else {
643 return false;
644 }
645 }
646
647 void ShenandoahBarrierStubC2::push_save_register(MacroAssembler& masm, Register reg) {
648 __ str(reg, Address(sp, push_save_slot()));
649 }
650
651 void ShenandoahBarrierStubC2::pop_save_register(MacroAssembler& masm, Register reg) {
652 __ ldr(reg, Address(sp, pop_save_slot()));
653 }
654
655 bool ShenandoahBarrierStubC2::is_live(Register reg) {
656 // TODO: Precompute the generic register map for faster lookups.
657 RegMaskIterator rmi(preserve_set());
658 while (rmi.has_next()) {
659 const OptoReg::Name opto_reg = rmi.next();
660 const VMReg vm_reg = OptoReg::as_VMReg(opto_reg);
661 if (vm_reg->is_Register() && reg == vm_reg->as_Register()) {
662 return true;
663 }
664 }
665 return false;
666 }
667
668 Register ShenandoahBarrierStubC2::select_temp_register(bool& selected_live, Address addr, Register reg1) {
669 Register tmp = noreg;
670 Register fallback_live = noreg;
671
672 // Try to select non-live first:
673 for (int i = 0; i < Register::available_gp_registers(); i++) {
674 Register r = as_Register(i);
675 if (r != rfp && r != sp && r != lr &&
676 r != rheapbase && r != rthread &&
677 r != rscratch1 && r != rscratch2 &&
678 r != reg1 && r != addr.base() && r != addr.index()) {
679 if (!is_live(r)) {
680 tmp = r;
681 break;
682 } else if (fallback_live == noreg) {
683 fallback_live = r;
684 }
685 }
686 }
687
688 // If we could not find a non-live register, select the live fallback:
689 if (tmp == noreg) {
690 tmp = fallback_live;
691 selected_live = true;
692 } else {
693 selected_live = false;
694 }
695
696 assert(tmp != noreg, "successfully selected");
697 assert_different_registers(tmp, reg1);
698 assert_different_registers(tmp, addr.base());
699 assert_different_registers(tmp, addr.index());
700 return tmp;
701 }
702
703 void ShenandoahBarrierStubC2::enter_if_gc_state(MacroAssembler& masm, const char test_state) {
704 Assembler::InlineSkippedInstructionsCounter skip_counter(&masm);
705
706 if (ShenandoahGCStateCheckRemove) {
707 // Unrealistic: remove all barrier fastpath checks.
708 } else if (ShenandoahGCStateCheckHotpatch) {
709 // Emit the unconditional branch in the first version of the method.
710 // Let the rest of runtime figure out how to manage it.
711 __ relocate(barrier_Relocation::spec());
712 __ b(*entry());
713
714 #ifdef ASSERT
715 Address gc_state_fast(rthread, in_bytes(ShenandoahThreadLocalData::gc_state_fast_offset()));
716 __ ldrb(rscratch1, gc_state_fast);
717 __ cbz(rscratch1, *continuation());
718 __ hlt(0); // Correctness bug: barrier is NOP-ed, but heap is NOT IDLE
719 #endif
720 __ bind(*continuation());
721 } else {
722 int bit_to_check = ShenandoahThreadLocalData::gc_state_to_fast_bit(test_state);
723 Address gc_state_fast(rthread, in_bytes(ShenandoahThreadLocalData::gc_state_fast_offset()));
724 __ ldrb(rscratch1, gc_state_fast);
725 if (_test_and_branch_reachable) {
726 __ tbnz(rscratch1, bit_to_check, _test_and_branch_reachable_entry);
727 } else {
728 __ tbz(rscratch1, bit_to_check, *continuation());
729 __ b(*entry());
730 }
731
732 // This is were the slowpath stub will return to or the code above will
733 // jump to if the checks are false
734 __ bind(*continuation());
735 }
736 }
737
738 address ShenandoahBarrierSetAssembler::parse_stub_address(address pc) {
739 NativeInstruction* ni = nativeInstruction_at(pc);
740 assert(ni->is_jump(), "Initial code version: GC barrier fastpath must be a jump");
741 NativeJump* jmp = nativeJump_at(pc);
742 return jmp->jump_destination();
743 }
744
745 void insert_nop(address pc) {
746 *(pc + 0) = 0x1F;
747 *(pc + 1) = 0x20;
748 *(pc + 2) = 0x03;
749 *(pc + 3) = 0xD5;
750 ICache::invalidate_range(pc, 4);
751 }
752
753 bool is_nop(address pc) {
754 if (*(pc + 0) != 0x1F) return false;
755 if (*(pc + 1) != 0x20) return false;
756 if (*(pc + 2) != 0x03) return false;
757 if (*(pc + 3) != 0xD5) return false;
758 return true;
759 }
760
761 void check_at(bool cond, address pc, const char* msg) {
762 assert(cond, "%s: at PC " PTR_FORMAT ": %02x%02x%02x%02x%02x",
763 msg, p2i(pc), *(pc + 0), *(pc + 1), *(pc + 2), *(pc + 3), *(pc + 4));
764 }
765
766 void ShenandoahBarrierSetAssembler::patch_branch_to_nop(address pc) {
767 NativeInstruction* ni = nativeInstruction_at(pc);
768 if (ni->is_jump()) {
769 insert_nop(pc);
770 } else {
771 check_at(is_nop(pc), pc, "Should already be nop");
772 }
773 }
774
775 void ShenandoahBarrierSetAssembler::patch_nop_to_branch(address pc, address stub_addr) {
776 NativeInstruction* ni = nativeInstruction_at(pc);
777 if (is_nop(pc)) {
778 NativeJump::insert(pc, stub_addr);
779 } else {
780 check_at(ni->is_jump(), pc, "Should already be jump");
781 check_at(nativeJump_at(pc)->jump_destination() == stub_addr, pc, "Jump should be to the same address");
782 }
783 }
784
785 bool needs_acquiring_load_exclusive(const MachNode *n) {
786 assert(n->is_CAS(true), "expecting a compare and swap");
787 if (n->is_CAS(false)) {
788 assert(n->has_trailing_membar(), "expected trailing membar");
789 } else {
790 return n->has_trailing_membar();
791 }
792
793 // so we can just return true here
794 return true;
795 }
796
797 #undef __
798 #define __ masm->
799
800 void ShenandoahBarrierSetAssembler::compare_and_set_c2(const MachNode* node, MacroAssembler* masm, Register res, Register addr,
801 Register oldval, Register newval, bool exchange, bool narrow, bool weak) {
802 bool acquire = needs_acquiring_load_exclusive(node);
803 Assembler::operand_size op_size = narrow ? Assembler::word : Assembler::xword;
804
805 // Pre-barrier covers several things:
806 // a. Avoids false positives from CAS encountering to-space memory values.
807 // b. Satisfies the need for LRB for the CAE result.
808 // c. Records old value for the sake of SATB.
809 //
810 // (a) and (b) are covered because load barrier does memory location fixup.
811 // (c) is covered by KA on the current memory value.
812 if (ShenandoahBarrierStubC2::needs_slow_barrier(node)) {
813 ShenandoahBarrierStubC2* const stub = ShenandoahBarrierStubC2::create(node, noreg, addr, narrow, /* do_load: */ true, __ offset());
814 char check = 0;
815 check |= ShenandoahBarrierStubC2::needs_keep_alive_barrier(node) ? ShenandoahHeap::MARKING : 0;
816 check |= ShenandoahBarrierStubC2::needs_load_ref_barrier(node) ? ShenandoahHeap::HAS_FORWARDED : 0;
817 assert(!ShenandoahBarrierStubC2::needs_load_ref_barrier_weak(node), "Not supported for CAS");
818 stub->enter_if_gc_state(*masm, check);
819 }
820
821 // CAS!
822 __ cmpxchg(addr, oldval, newval, op_size, acquire, /* release */ true, weak, exchange ? res : noreg);
823
824 // If we need a boolean result out of CAS, set the flag appropriately and promote the result.
825 if (!exchange) {
826 assert(res != noreg, "need result register");
827 __ cset(res, Assembler::EQ);
828 }
829
830 // Post-barrier deals with card updates.
831 card_barrier_c2(node, masm, Address(addr, 0));
832 }
833
834 void ShenandoahBarrierSetAssembler::get_and_set_c2(const MachNode* node, MacroAssembler* masm, Register preval,
835 Register newval, Register addr) {
836 bool acquire = needs_acquiring_load_exclusive(node);
837 bool narrow = node->bottom_type()->isa_narrowoop();
838
839 // Pre-barrier covers several things:
840 // a. Satisfies the need for LRB for the GAS result.
841 // b. Records old value for the sake of SATB.
842 //
843 // (a) is covered because load barrier does memory location fixup.
844 // (b) is covered by KA on the current memory value.
845 if (ShenandoahBarrierStubC2::needs_slow_barrier(node)) {
846 ShenandoahBarrierStubC2* const stub = ShenandoahBarrierStubC2::create(node, noreg, addr, narrow, /* do_load: */ true, __ offset());
847 char check = 0;
848 check |= ShenandoahBarrierStubC2::needs_keep_alive_barrier(node) ? ShenandoahHeap::MARKING : 0;
849 check |= ShenandoahBarrierStubC2::needs_load_ref_barrier(node) ? ShenandoahHeap::HAS_FORWARDED : 0;
850 assert(!ShenandoahBarrierStubC2::needs_load_ref_barrier_weak(node), "Not supported for GAS");
851 stub->enter_if_gc_state(*masm, check);
852 }
853
854 if (narrow) {
855 if (acquire) {
856 __ atomic_xchgalw(preval, newval, addr);
857 } else {
858 __ atomic_xchgw(preval, newval, addr);
859 }
860 } else {
861 if (acquire) {
862 __ atomic_xchgal(preval, newval, addr);
863 } else {
864 __ atomic_xchg(preval, newval, addr);
865 }
866 }
867
868 // Post-barrier deals with card updates.
869 card_barrier_c2(node, masm, Address(addr, 0));
870 }
871
872 void ShenandoahBarrierSetAssembler::store_c2(const MachNode* node, MacroAssembler* masm, Address dst, bool dst_narrow,
873 Register src, bool src_narrow) {
874
875 // Pre-barrier: SATB, keep-alive the current memory value.
876 if (ShenandoahBarrierStubC2::needs_slow_barrier(node)) {
877 assert(!ShenandoahBarrierStubC2::needs_load_ref_barrier(node), "Should not be required for stores");
878 ShenandoahBarrierStubC2* const stub = ShenandoahBarrierStubC2::create(node, noreg, dst, dst_narrow, /* do_load: */ true, __ offset());
879 stub->enter_if_gc_state(*masm, ShenandoahHeap::MARKING);
880 }
881
882 // Do the actual store
883 bool is_volatile = node->has_trailing_membar();
884 if (dst_narrow) {
885 if (!src_narrow) {
886 // Need to encode into rscratch, because we cannot clobber src.
887 // TODO: Maybe there is a matcher way to test that src is unused after this?
888 __ mov(rscratch1, src);
889 if (ShenandoahBarrierStubC2::maybe_null(node)) {
890 __ encode_heap_oop(rscratch1);
891 } else {
892 __ encode_heap_oop_not_null(rscratch1);
893 }
894 src = rscratch1;
895 }
896
897 if (is_volatile) {
898 __ stlrw(src, dst.base());
899 } else {
900 __ strw(src, dst);
901 }
902 } else {
903 if (is_volatile) {
904 __ stlr(src, dst.base());
905 } else {
906 __ str(src, dst);
907 }
908 }
909
910 // Post-barrier: card updates.
911 card_barrier_c2(node, masm, dst);
912 }
913
914 void ShenandoahBarrierSetAssembler::load_c2(const MachNode* node, MacroAssembler* masm, Register dst, Address src) {
915 bool acquire = node->memory_order() == MemNode::MemOrd::acquire;
916 bool narrow = node->bottom_type()->isa_narrowoop();
917
918 // Do the actual load. This load is the candidate for implicit null check, and MUST come first.
919 if (narrow) {
920 if (acquire) {
921 __ ldarw(dst, src.base());
922 } else {
923 __ ldrw(dst, src);
924 }
925 } else {
926 if (acquire) {
927 __ ldar(dst, src.base());
928 } else {
929 __ ldr(dst, src);
930 }
931 }
932
933 // Post-barrier: LRB / KA / weak-root processing.
934 if (ShenandoahBarrierStubC2::needs_slow_barrier(node)) {
935 ShenandoahBarrierStubC2* const stub = ShenandoahBarrierStubC2::create(node, dst, src, narrow, /* do_load: */ false, __ offset());
936 char check = 0;
937 check |= ShenandoahBarrierStubC2::needs_keep_alive_barrier(node) ? ShenandoahHeap::MARKING : 0;
938 check |= ShenandoahBarrierStubC2::needs_load_ref_barrier(node) ? ShenandoahHeap::HAS_FORWARDED : 0;
939 check |= ShenandoahBarrierStubC2::needs_load_ref_barrier_weak(node) ? ShenandoahHeap::WEAK_ROOTS : 0;
940 stub->enter_if_gc_state(*masm, check);
941 }
942 }
943
944 void ShenandoahBarrierSetAssembler::card_barrier_c2(const MachNode* node, MacroAssembler* masm, Address address) {
945 if (ShenandoahSkipBarriers || (node->barrier_data() & ShenandoahBitCardMark) == 0) {
946 return;
947 }
948
949 assert(CardTable::dirty_card_val() == 0, "must be");
950 Assembler::InlineSkippedInstructionsCounter skip_counter(masm);
951
952 // rscratch1 = card table base (holder)
953 Address curr_ct_holder_addr(rthread, in_bytes(ShenandoahThreadLocalData::card_table_offset()));
954 __ ldr(rscratch1, curr_ct_holder_addr);
955
956 // rscratch2 = addr
957 __ lea(rscratch2, address);
958
959 // rscratch2 = &card_table[ addr >> CardTable::card_shift() ]
960 __ add(rscratch2, rscratch1, rscratch2, Assembler::LSR, CardTable::card_shift());
961
962 if (UseCondCardMark) {
963 Label L_already_dirty;
964 __ ldrb(rscratch1, Address(rscratch2));
965 __ cbz(rscratch1, L_already_dirty);
966 __ strb(zr, Address(rscratch2));
967 __ bind(L_already_dirty);
968 } else {
969 __ strb(zr, Address(rscratch2));
970 }
971 }
972 #undef __
973 #define __ masm.
974
975 // Only handles forward branch jumps, target_offset >= branch_offset
976 // FIXME: copied verbatim from ZGC, duplicated code.
977 static bool aarch64_test_and_branch_reachable(int branch_offset, int target_offset) {
978 assert(branch_offset >= 0, "branch to stub offsets must be positive");
979 assert(target_offset >= 0, "offset in stubs section must be positive");
980 assert(target_offset >= branch_offset, "forward branches only, branch_offset -> target_offset");
981
982 const int test_and_branch_delta_limit = 32 * K;
983
984 const int test_and_branch_to_trampoline_delta = target_offset - branch_offset;
985
986 return test_and_branch_to_trampoline_delta < test_and_branch_delta_limit;
987 }
988
989 void ShenandoahBarrierStubC2::post_init(int offset) {
990 // If we are in scratch emit mode we assume worse case by leaving
991 // _test_and_branch_reachable false.
992 PhaseOutput* const output = Compile::current()->output();
993 if (output->in_scratch_emit_size()) {
994 return;
995 }
996
997 // Assume that each trampoline is one single instruction and that the stubs
998 // will follow immediatelly after the _code section. Therefore, we are
999 // checking if the distance between the fastpath branch and the
1000 // trampoline/entry of the current Stub is less than 32K.
1001 const int code_size = output->buffer_sizing_data()->_code;
1002 const int trampoline_offset = trampoline_stubs_count() * NativeInstruction::instruction_size;
1003 _test_and_branch_reachable = aarch64_test_and_branch_reachable(_fastpath_branch_offset, code_size + trampoline_offset);
1004 if (_test_and_branch_reachable) {
1005 inc_trampoline_stubs_count();
1006 }
1007 }
1008
1009 void ShenandoahBarrierStubC2::emit_code(MacroAssembler& masm) {
1010 // If we reach here with _skip_trampoline set it means that earlier we
1011 // emitted a trampoline to this stub and now we need to emit the actual stub.
1012 if (ShenandoahGCStateCheckHotpatch || _skip_trampoline) {
1013 emit_code_actual(masm);
1014 } else {
1015 _skip_trampoline = true;
1016
1017 // The fastpath executes two branch instructions to reach this stub, let's
1018 // just emit the stub here and not add a third one.
1019 if (!_test_and_branch_reachable) {
1020 // By registering the stub again, after setting _skip_trampoline, we'll
1021 // effectivelly cause the stub to be emitted the next time ::emit_code is
1022 // called.
1023 ShenandoahBarrierStubC2::register_stub(this);
1024 return;
1025 }
1026
1027 // This is entry point when coming from fastpath, IFF it's able to reach here
1028 // with a test and branch instruction, otherwise the entry is
1029 // ShenandoahBarrierStubC2::entry();
1030 const int target_offset = __ offset();
1031 __ bind(_test_and_branch_reachable_entry);
1032
1033 #ifdef ASSERT
1034 // Current assumption is that the barrier stubs are the first stubs emitted
1035 // after the actual code
1036 PhaseOutput* const output = Compile::current()->output();
1037 assert(stubs_start_offset() <= output->buffer_sizing_data()->_code, "stubs are assumed to be emitted directly after code and code_size is a hard limit on where it can start");
1038 assert(aarch64_test_and_branch_reachable(_fastpath_branch_offset, target_offset), "trampoline should be reachable");
1039 #endif
1040
1041 // Next fastpath branch's offset is unknown, but it's > current _fastpath_branch_offset
1042 const int next_branch_offset = _fastpath_branch_offset + NativeInstruction::instruction_size;
1043
1044 // If emitting the current stub directly does not interfere with emission of
1045 // the next potential trampoline then do it to avoid executing additional
1046 // branch when coming from fastpath.
1047 if (aarch64_test_and_branch_reachable(next_branch_offset, target_offset + get_stub_size())) {
1048 emit_code_actual(masm);
1049 } else {
1050 __ b(*entry());
1051 // By registering the stub again, after setting _skip_trampoline to true,
1052 // we'll effectivelly cause the stub to be emitted the next time
1053 // ::emit_code is called.
1054 ShenandoahBarrierStubC2::register_stub(this);
1055 }
1056 }
1057 }
1058
1059 int ShenandoahBarrierStubC2::get_stub_size() {
1060 PhaseOutput* const output = Compile::current()->output();
1061 assert(!output->in_scratch_emit_size(), "only used when emitting stubs");
1062 BufferBlob* const blob = output->scratch_buffer_blob();
1063 CodeBuffer cb(blob->content_begin(), (address)output->scratch_locs_memory() - blob->content_begin());
1064 MacroAssembler masm(&cb);
1065 output->set_in_scratch_emit_size(true);
1066 emit_code_actual(masm);
1067 output->set_in_scratch_emit_size(false);
1068 return cb.insts_size();
1069 }
1070
1071 void ShenandoahBarrierStubC2::emit_code_actual(MacroAssembler& masm) {
1072 assert(_needs_keep_alive_barrier || _needs_load_ref_barrier, "Why are you here?");
1073
1074 Label L_done;
1075
1076 if (ShenandoahGCStateCheckHotpatch || !Compile::current()->output()->in_scratch_emit_size()) {
1077 __ bind(*entry());
1078 }
1079
1080 // If we need to load ourselves, do it here.
1081 bool selected_live = false;
1082 if (_do_load) {
1083 _obj = select_temp_register(selected_live, _addr, noreg);
1084 if (selected_live) {
1085 push_save_register(masm, _obj);
1086 }
1087
1088 // This does the load and the decode if necessary
1089 __ load_heap_oop(_obj, _addr, noreg, noreg, AS_RAW);
1090 }
1091
1092 // If object is narrow, we need to decode it first: barrier checks need full oops.
1093 if (!_do_load && _narrow) {
1094 if (_maybe_null) {
1095 __ decode_heap_oop(_obj);
1096 } else {
1097 __ decode_heap_oop_not_null(_obj);
1098 }
1099 }
1100
1101 if (_do_load || _maybe_null) {
1102 __ cbz(_obj, L_done);
1103 }
1104
1105 keepalive(masm, _obj, rscratch1, rscratch2);
1106
1107 lrb(masm, _obj, _addr, rscratch1);
1108
1109 // If object is narrow, we need to encode it before exiting.
1110 // For encoding, dst can only turn null if we are dealing with weak loads.
1111 // Otherwise, we have already null-checked. We can skip all this if we performed
1112 // the load ourselves, which means the value is not used by caller.
1113 if (_narrow && !_do_load) {
1114 if (_needs_load_ref_weak_barrier) {
1115 __ encode_heap_oop(_obj);
1116 } else {
1117 __ encode_heap_oop_not_null(_obj);
1118 }
1119 }
1120
1121 __ bind(L_done);
1122
1123 // If we picked up a live register to store the load of _addr then we restore it now
1124 if (selected_live) {
1125 pop_save_register(masm, _obj);
1126 }
1127
1128 // Go back to fast path
1129 __ b(*continuation());
1130 }
1131
1132 void ShenandoahBarrierStubC2::keepalive(MacroAssembler& masm, Register obj, Register tmp1, Register tmp2) {
1133 Address index(rthread, in_bytes(ShenandoahThreadLocalData::satb_mark_queue_index_offset()));
1134 Address buffer(rthread, in_bytes(ShenandoahThreadLocalData::satb_mark_queue_buffer_offset()));
1135 Label L_runtime;
1136 Label L_done;
1137
1138 // The node doesn't even need keepalive barrier, just don't check anything else
1139 if (!_needs_keep_alive_barrier) {
1140 return ;
1141 }
1142
1143 // If another barrier is enabled as well, do a runtime check for a specific barrier.
1144 // Hotpatched GC checks only care about idle/non-idle state, so needs a check anyhow.
1145 if (_needs_load_ref_barrier || ShenandoahGCStateCheckHotpatch) {
1146 Address gcs_addr(rthread, in_bytes(ShenandoahThreadLocalData::gc_state_offset()));
1147 __ ldrb(tmp1, gcs_addr);
1148 __ tbz(tmp1, ShenandoahHeap::MARKING_BITPOS, L_done);
1149 }
1150
1151 // If buffer is full, call into runtime.
1152 __ ldr(tmp1, index);
1153 __ cbz(tmp1, L_runtime);
1154
1155 // The buffer is not full, store value into it.
1156 __ sub(tmp1, tmp1, wordSize);
1157 __ str(tmp1, index);
1158 __ ldr(tmp2, buffer);
1159 __ str(obj, Address(tmp2, tmp1));
1160 __ b(L_done);
1161
1162 // Runtime call
1163 __ bind(L_runtime);
1164
1165 preserve(obj);
1166 {
1167 bool clobbered_c_rarg0 = false;
1168 if (c_rarg0 != obj) {
1169 clobbered_c_rarg0 = push_save_register_if_live(masm, c_rarg0);
1170 __ mov(c_rarg0, obj);
1171 }
1172
1173 // Go to runtime stub and handle the rest there.
1174 __ far_call(RuntimeAddress(keepalive_runtime_entry_addr()));
1175
1176 // Restore the clobbered registers.
1177 if (clobbered_c_rarg0) {
1178 pop_save_register(masm, c_rarg0);
1179 }
1180 }
1181
1182 __ bind(L_done);
1183 }
1184
1185 void ShenandoahBarrierStubC2::lrb(MacroAssembler& masm, Register obj, Address addr, Register tmp) {
1186 Label L_done;
1187
1188 // The node doesn't even need LRB barrier, just don't check anything else
1189 if (!_needs_load_ref_barrier) {
1190 return ;
1191 }
1192
1193 if ((_node->barrier_data() & ShenandoahBitStrong) != 0) {
1194 // If another barrier is enabled as well, do a runtime check for a specific barrier.
1195 // Hotpatched GC checks only care about idle/non-idle state, so needs a check anyhow.
1196 if (_needs_keep_alive_barrier || ShenandoahGCStateCheckHotpatch) {
1197 char state_to_check = ShenandoahHeap::HAS_FORWARDED | (_needs_load_ref_weak_barrier ? ShenandoahHeap::WEAK_ROOTS : 0);
1198 int bit_to_check = ShenandoahThreadLocalData::gc_state_to_fast_bit(state_to_check);
1199 Address gc_state_fast(rthread, in_bytes(ShenandoahThreadLocalData::gc_state_fast_offset()));
1200 __ ldrb(tmp, gc_state_fast);
1201 __ tbz(tmp, bit_to_check, L_done);
1202 }
1203
1204 // Weak/phantom loads always need to go to runtime. For strong refs we
1205 // check if the object in cset, if they are not, then we are done with LRB.
1206 assert(ShenandoahHeapRegion::region_size_bytes_shift_jint() <= 63, "Maximum shift of the add is 63");
1207 __ mov(tmp, ShenandoahHeap::in_cset_fast_test_addr());
1208 __ add(tmp, tmp, obj, Assembler::LSR, ShenandoahHeapRegion::region_size_bytes_shift_jint());
1209 __ ldrb(tmp, Address(tmp, 0));
1210 __ cbz(tmp, L_done);
1211 }
1212
1213 dont_preserve(obj);
1214 {
1215 // Shuffle in the arguments. The end result should be:
1216 // c_rarg0 <-- obj
1217 // c_rarg1 <-- lea(addr)
1218 //
1219 // Save clobbered registers before overwriting them, unless they
1220 // carry obj, which would be overwritten on return.
1221 bool clobbered_c_rarg0 = false;
1222 bool clobbered_c_rarg1 = false;
1223 bool clobbered_r0 = false;
1224
1225 if (c_rarg0 == obj) {
1226 clobbered_c_rarg1 = push_save_register_if_live(masm, c_rarg1);
1227 __ lea(c_rarg1, addr);
1228 } else if (c_rarg1 == obj) {
1229 // Set up arguments in reverse, and then flip them
1230 clobbered_c_rarg0 = push_save_register_if_live(masm, c_rarg0);
1231 __ lea(c_rarg0, addr);
1232 // flip them
1233 __ mov(rscratch1, c_rarg0);
1234 __ mov(c_rarg0, c_rarg1);
1235 __ mov(c_rarg1, rscratch1);
1236 } else {
1237 assert_different_registers(c_rarg1, obj);
1238 clobbered_c_rarg0 = push_save_register_if_live(masm, c_rarg0);
1239 clobbered_c_rarg1 = push_save_register_if_live(masm, c_rarg1);
1240 __ lea(c_rarg1, addr);
1241 __ mov(c_rarg0, obj);
1242 }
1243
1244 // The runtime call will clobber r0 at return. If obj isn't r0 then we need
1245 // to save obj.
1246 if (obj != r0) {
1247 clobbered_r0 = push_save_register_if_live(masm, r0);
1248 }
1249
1250 // Go to runtime stub and handle the rest there.
1251 __ far_call(RuntimeAddress(lrb_runtime_entry_addr()));
1252
1253 // Save the result where needed and restore the clobbered registers.
1254 if (obj != r0) {
1255 __ mov(obj, r0);
1256 }
1257 if (clobbered_r0) {
1258 pop_save_register(masm, r0);
1259 }
1260 if (clobbered_c_rarg1) {
1261 pop_save_register(masm, c_rarg1);
1262 }
1263 if (clobbered_c_rarg0) {
1264 pop_save_register(masm, c_rarg0);
1265 }
1266 }
1267
1268 __ bind(L_done);
1269 }
1270
1271 #undef __
1272 #define __ masm->
1273
1274 #endif // COMPILER2
1275
1276 void ShenandoahBarrierSetAssembler::gen_write_ref_array_post_barrier(MacroAssembler* masm, DecoratorSet decorators,
1277 Register start, Register count, Register scratch) {
1278 assert(ShenandoahCardBarrier, "Should have been checked by caller");
1279
1280 Label L_loop, L_done;
1281 const Register end = count;
1282
1283 // Zero count? Nothing to do.
1284 __ cbz(count, L_done);
1285
1286 // end = start + count << LogBytesPerHeapOop
1287 // last element address to make inclusive
1288 __ lea(end, Address(start, count, Address::lsl(LogBytesPerHeapOop)));
1289 __ sub(end, end, BytesPerHeapOop);
1290 __ lsr(start, start, CardTable::card_shift());
1291 __ lsr(end, end, CardTable::card_shift());
1292
1293 // number of bytes to copy
1294 __ sub(count, end, start);
1295
1296 Address curr_ct_holder_addr(rthread, in_bytes(ShenandoahThreadLocalData::card_table_offset()));
1297 __ ldr(scratch, curr_ct_holder_addr);
1298 __ add(start, start, scratch);
1299 __ bind(L_loop);
1300 __ strb(zr, Address(start, count));
1301 __ subs(count, count, 1);
1302 __ br(Assembler::GE, L_loop);
1303 __ bind(L_done);
1304 }
1305
1306 #undef __
1307
1308 #ifdef COMPILER1
1309
1310 #define __ ce->masm()->
1311
1312 void ShenandoahBarrierSetAssembler::gen_pre_barrier_stub(LIR_Assembler* ce, ShenandoahPreBarrierStub* stub) {
1313 ShenandoahBarrierSetC1* bs = (ShenandoahBarrierSetC1*)BarrierSet::barrier_set()->barrier_set_c1();
1314 // At this point we know that marking is in progress.
1315 // If do_load() is true then we have to emit the
1316 // load of the previous value; otherwise it has already
1317 // been loaded into _pre_val.
1318
1319 __ bind(*stub->entry());
1320
1321 assert(stub->pre_val()->is_register(), "Precondition.");
1322
1323 Register pre_val_reg = stub->pre_val()->as_register();
1324
1325 if (stub->do_load()) {
1326 ce->mem2reg(stub->addr(), stub->pre_val(), T_OBJECT, stub->patch_code(), stub->info(), false /*wide*/);
1327 }
1328 __ cbz(pre_val_reg, *stub->continuation());
1329 ce->store_parameter(stub->pre_val()->as_register(), 0);
1330 __ far_call(RuntimeAddress(bs->pre_barrier_c1_runtime_code_blob()->code_begin()));
1331 __ b(*stub->continuation());
1332 }
1333
1334 void ShenandoahBarrierSetAssembler::gen_load_reference_barrier_stub(LIR_Assembler* ce, ShenandoahLoadReferenceBarrierStub* stub) {
1335 ShenandoahBarrierSetC1* bs = (ShenandoahBarrierSetC1*)BarrierSet::barrier_set()->barrier_set_c1();
1336 __ bind(*stub->entry());
1337
1338 DecoratorSet decorators = stub->decorators();
1339 bool is_strong = ShenandoahBarrierSet::is_strong_access(decorators);
1340 bool is_weak = ShenandoahBarrierSet::is_weak_access(decorators);
1341 bool is_phantom = ShenandoahBarrierSet::is_phantom_access(decorators);
1342 bool is_native = ShenandoahBarrierSet::is_native_access(decorators);
1343
1344 Register obj = stub->obj()->as_register();
1345 Register res = stub->result()->as_register();
1346 Register addr = stub->addr()->as_pointer_register();
1347 Register tmp1 = stub->tmp1()->as_register();
1348 Register tmp2 = stub->tmp2()->as_register();
1349
1350 assert(res == r0, "result must arrive in r0");
1351
1352 if (res != obj) {
1353 __ mov(res, obj);
1354 }
1355
1356 if (is_strong) {
1357 // Check for object in cset.
1358 __ mov(tmp2, ShenandoahHeap::in_cset_fast_test_addr());
1359 __ lsr(tmp1, res, ShenandoahHeapRegion::region_size_bytes_shift_jint());
1360 __ ldrb(tmp2, Address(tmp2, tmp1));
1361 __ cbz(tmp2, *stub->continuation());
1362 }
1363
1364 ce->store_parameter(res, 0);
1365 ce->store_parameter(addr, 1);
1366 if (is_strong) {
1367 if (is_native) {
1368 __ far_call(RuntimeAddress(bs->load_reference_barrier_strong_native_rt_code_blob()->code_begin()));
1369 } else {
1370 __ far_call(RuntimeAddress(bs->load_reference_barrier_strong_rt_code_blob()->code_begin()));
1371 }
1372 } else if (is_weak) {
1373 __ far_call(RuntimeAddress(bs->load_reference_barrier_weak_rt_code_blob()->code_begin()));
1374 } else {
1375 assert(is_phantom, "only remaining strength");
1376 __ far_call(RuntimeAddress(bs->load_reference_barrier_phantom_rt_code_blob()->code_begin()));
1377 }
1378
1379 __ b(*stub->continuation());
1380 }
1381
1382 #undef __
1383
1384 #define __ sasm->
1385
1386 void ShenandoahBarrierSetAssembler::generate_c1_pre_barrier_runtime_stub(StubAssembler* sasm) {
1387 __ prologue("shenandoah_pre_barrier", false);
1388
1389 // arg0 : previous value of memory
1390
1391 BarrierSet* bs = BarrierSet::barrier_set();
1392
1393 const Register pre_val = r0;
1394 const Register thread = rthread;
1395 const Register tmp = rscratch1;
1396
1397 Address queue_index(thread, in_bytes(ShenandoahThreadLocalData::satb_mark_queue_index_offset()));
1398 Address buffer(thread, in_bytes(ShenandoahThreadLocalData::satb_mark_queue_buffer_offset()));
1399
1400 Label done;
1401 Label runtime;
1402
1403 // Is marking still active?
1404 Address gc_state(thread, in_bytes(ShenandoahThreadLocalData::gc_state_offset()));
1405 __ ldrb(tmp, gc_state);
1406 __ tbz(tmp, ShenandoahHeap::MARKING_BITPOS, done);
1407
1408 // Can we store original value in the thread's buffer?
1409 __ ldr(tmp, queue_index);
1410 __ cbz(tmp, runtime);
1411
1412 __ sub(tmp, tmp, wordSize);
1413 __ str(tmp, queue_index);
1414 __ ldr(rscratch2, buffer);
1415 __ add(tmp, tmp, rscratch2);
1416 __ load_parameter(0, rscratch2);
1417 __ str(rscratch2, Address(tmp, 0));
1418 __ b(done);
1419
1420 __ bind(runtime);
1421 __ push_call_clobbered_registers();
1422 __ load_parameter(0, pre_val);
1423 __ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::write_barrier_pre), pre_val);
1424 __ pop_call_clobbered_registers();
1425 __ bind(done);
1426
1427 __ epilogue();
1428 }
1429
1430 void ShenandoahBarrierSetAssembler::generate_c1_load_reference_barrier_runtime_stub(StubAssembler* sasm, DecoratorSet decorators) {
1431 __ prologue("shenandoah_load_reference_barrier", false);
1432 // arg0 : object to be resolved
1433
1434 __ push_call_clobbered_registers();
1435 __ load_parameter(0, r0);
1436 __ load_parameter(1, r1);
1437
1438 bool is_strong = ShenandoahBarrierSet::is_strong_access(decorators);
1439 bool is_weak = ShenandoahBarrierSet::is_weak_access(decorators);
1440 bool is_phantom = ShenandoahBarrierSet::is_phantom_access(decorators);
1441 bool is_native = ShenandoahBarrierSet::is_native_access(decorators);
1442 if (is_strong) {
1443 if (is_native) {
1444 __ mov(lr, CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_strong));
1445 } else {
1446 if (UseCompressedOops) {
1447 __ mov(lr, CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_strong_narrow));
1448 } else {
1449 __ mov(lr, CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_strong));
1450 }
1451 }
1452 } else if (is_weak) {
1453 assert(!is_native, "weak must not be called off-heap");
1454 if (UseCompressedOops) {
1455 __ mov(lr, CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_weak_narrow));
1456 } else {
1457 __ mov(lr, CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_weak));
1458 }
1459 } else {
1460 assert(is_phantom, "only remaining strength");
1461 assert(is_native, "phantom must only be called off-heap");
1462 __ mov(lr, CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_phantom));
1463 }
1464 __ blr(lr);
1465 __ mov(rscratch1, r0);
1466 __ pop_call_clobbered_registers();
1467 __ mov(r0, rscratch1);
1468
1469 __ epilogue();
1470 }
1471
1472 #undef __
1473
1474 #endif // COMPILER1