< prev index next > src/hotspot/cpu/aarch64/gc/shenandoah/shenandoahBarrierSetAssembler_aarch64.cpp
Print this page
}
void ShenandoahBarrierStubC2::enter_if_gc_state(MacroAssembler& masm, const char test_state, Register tmp) {
Assembler::InlineSkippedInstructionsCounter skip_counter(&masm);
PhaseOutput* const output = Compile::current()->output();
- Address gc_state_fast(rthread, in_bytes(ShenandoahThreadLocalData::gc_state_fast_array_offset(test_state)));
- // We piggyback on scratch_emit_size mode to compute the slowpath stub size.
- // We'll use that information to decide whether we need a far jump to the
- // stub entry point or not. In scratch_emit_size mode we don't bind entry()
- // because otherwise it will be rebound when we later emit the instructions
- // for real.
- if (_needs_far_jump) {
- __ ldrb(tmp, gc_state_fast);
- __ cbz(tmp, *continuation());
- __ b(output->in_scratch_emit_size() ? *continuation() : *entry());
+ // Emit the unconditional branch in the first version of the method.
+ // Let the rest of runtime figure out how to manage it.
+ if (output->in_scratch_emit_size()) {
+ // We piggyback on scratch_emit_size mode to compute the slowpath stub size.
+ // We'll use that information to decide whether we need a far jump to the
+ // stub entry point or not. In scratch_emit_size mode we don't bind entry()
+ // because otherwise it will be rebound when we later emit the instructions
+ // for real.
+ __ nop();
} else {
- __ ldrb(tmp, gc_state_fast);
- __ cbnz(tmp, output->in_scratch_emit_size() ? *continuation() : *entry());
+ __ relocate(barrier_Relocation::spec(), ShenandoahThreadLocalData::gc_state_to_fast_array_index(test_state));
+ __ b(*entry());
+
+ // #ifdef ASSERT
+ // Address gc_state_fast(rthread, in_bytes(ShenandoahThreadLocalData::gc_state_fast_array_offset(test_state)));
+ // __ ldrb(rscratch1, gc_state_fast);
+ // __ cbz(rscratch1, *continuation());
+ // __ hlt(0); // Correctness bug: barrier is NOP-ed, but heap is NOT IDLE
+ // #endif
}
// This is were the slowpath stub will return to or the code above will
// jump to if the checks are false
__ bind(*continuation());
}
+ address ShenandoahBarrierSetAssembler::parse_stub_address(address pc) {
+ NativeInstruction* ni = nativeInstruction_at(pc);
+ assert(ni->is_jump(), "Initial code version: GC barrier fastpath must be a jump");
+ NativeJump* jmp = nativeJump_at(pc);
+ return jmp->jump_destination();
+ }
+
+ void insert_nop(address pc) {
+ *(pc + 0) = 0x1F;
+ *(pc + 1) = 0x20;
+ *(pc + 2) = 0x03;
+ *(pc + 3) = 0xD5;
+ ICache::invalidate_range(pc, 4);
+ }
+
+ bool is_nop(address pc) {
+ if (*(pc + 0) != 0x1F) return false;
+ if (*(pc + 1) != 0x20) return false;
+ if (*(pc + 2) != 0x03) return false;
+ if (*(pc + 3) != 0xD5) return false;
+ return true;
+ }
+
+ void check_at(bool cond, address pc, const char* msg) {
+ assert(cond, "%s: at PC " PTR_FORMAT ": %02x%02x%02x%02x%02x",
+ msg, p2i(pc), *(pc + 0), *(pc + 1), *(pc + 2), *(pc + 3), *(pc + 4));
+ }
+
+ bool ShenandoahBarrierSetAssembler::is_active(address pc) {
+ NativeInstruction* ni = nativeInstruction_at(pc);
+ return ni->is_jump();
+ }
+
+ void ShenandoahBarrierSetAssembler::patch_branch_to_nop(address pc) {
+ NativeInstruction* ni = nativeInstruction_at(pc);
+ if (ni->is_jump()) {
+ insert_nop(pc);
+ } else {
+ check_at(is_nop(pc), pc, "Should already be nop");
+ }
+ }
+
+ void ShenandoahBarrierSetAssembler::patch_nop_to_branch(address pc, address stub_addr) {
+ NativeInstruction* ni = nativeInstruction_at(pc);
+ if (is_nop(pc)) {
+ NativeJump::insert(pc, stub_addr);
+ } else {
+ check_at(ni->is_jump(), pc, "Should already be jump");
+ check_at(nativeJump_at(pc)->jump_destination() == stub_addr, pc, "Jump should be to the same address");
+ }
+ }
+
void ShenandoahBarrierStubC2::emit_code(MacroAssembler& masm) {
Assembler::InlineSkippedInstructionsCounter skip_counter(&masm);
assert(_needs_keep_alive_barrier || _needs_load_ref_barrier, "Why are you here?");
PhaseOutput* const output = Compile::current()->output();
__ ldr(_obj, _addr);
}
}
// If the object is null, there is no point in applying barriers.
- maybe_far_jump_if_zero(masm, _obj);
+ maybe_far_jump_if_zero(masm, _obj, continuation());
// We need to make sure that loads done by callers survive across slow-path calls.
// For self-loads, we need to care about the case when both KA and LRB are enabled (rare).
bool needs_both_barriers = _needs_keep_alive_barrier && _needs_load_ref_barrier;
if (!_do_load || needs_both_barriers) {
} else {
ShouldNotReachHere();
}
}
- void ShenandoahBarrierStubC2::maybe_far_jump_if_zero(MacroAssembler& masm, Register reg) {
+ void ShenandoahBarrierStubC2::maybe_far_jump_if_zero(MacroAssembler& masm, Register reg, Label* L_target) {
+ assert(L_target == continuation(), "Should be");
if (_needs_far_jump) {
Label L_short_jump;
__ cbnz(reg, L_short_jump);
- __ b(*continuation());
+ __ b(*L_target);
__ bind(L_short_jump);
} else {
- __ cbz(reg, *continuation());
+ __ cbz(reg, *L_target);
}
}
void ShenandoahBarrierStubC2::keepalive(MacroAssembler& masm, Label* L_done) {
Address gcstate(rthread, in_bytes(ShenandoahThreadLocalData::gc_state_fast_array_offset(ShenandoahHeap::MARKING)));
Address index(rthread, in_bytes(ShenandoahThreadLocalData::satb_mark_queue_index_offset()));
Address buffer(rthread, in_bytes(ShenandoahThreadLocalData::satb_mark_queue_buffer_offset()));
Label L_through, L_slowpath;
- // If another barrier is enabled as well, do a runtime check for a specific barrier.
- if (_needs_load_ref_barrier) {
- assert(L_done == nullptr, "L_done is always null when _needs_load_ref_barrier is true");
- __ ldrb(_tmp1, gcstate);
+ // Hotpatched GC checks are racy: we can turn off GC state before we patch the barriers.
+ // Therefore, alas we need a separate check here. TODO: Figure this out.
+ __ ldrb(_tmp1, gcstate);
+ if (L_done != nullptr) {
+ maybe_far_jump_if_zero(masm, _tmp1, L_done);
+ } else {
__ cbz(_tmp1, L_through);
}
// Fast-path: put object into buffer.
// If buffer is already full, go slow.
}
void ShenandoahBarrierStubC2::lrb(MacroAssembler& masm) {
Label L_slow;
- // If another barrier is enabled as well, do a runtime check for a specific barrier.
- if (_needs_keep_alive_barrier) {
- char state_to_check = ShenandoahHeap::HAS_FORWARDED | (_needs_load_ref_weak_barrier ? ShenandoahHeap::WEAK_ROOTS : 0);
- Address gc_state_fast(rthread, in_bytes(ShenandoahThreadLocalData::gc_state_fast_array_offset(state_to_check)));
- __ ldrb(_tmp1, gc_state_fast);
- maybe_far_jump_if_zero(masm, _tmp1);
- }
+ // Hotpatched GC checks are racy: we can turn off GC state before we patch the barriers.
+ // Therefore, alas we need a separate check here. TODO: Figure this out.
+ char state_to_check = ShenandoahHeap::HAS_FORWARDED | (_needs_load_ref_weak_barrier ? ShenandoahHeap::WEAK_ROOTS : 0);
+ Address gc_state_fast(rthread, in_bytes(ShenandoahThreadLocalData::gc_state_fast_array_offset(state_to_check)));
+ __ ldrb(_tmp1, gc_state_fast);
+ maybe_far_jump_if_zero(masm, _tmp1, continuation());
// If weak references are being processed, weak/phantom loads need to go slow,
// regardless of their cset status.
if (_needs_load_ref_weak_barrier) {
Address gc_state_fast(rthread, in_bytes(ShenandoahThreadLocalData::gc_state_fast_array_offset(ShenandoahHeap::WEAK_ROOTS)));
__ lea(_tmp1, ExternalAddress(AOTRuntimeConstants::cset_base_address()));
__ ldr(_tmp1, Address(_tmp1));
__ add(_tmp1, _tmp1, _tmp2);
}
__ ldrb(_tmp1, Address(_tmp1, 0));
- maybe_far_jump_if_zero(masm, _tmp1);
+ maybe_far_jump_if_zero(masm, _tmp1, continuation());
// Slow path
__ bind(L_slow);
// Obj is the result, need to temporarily stop preserving it.
< prev index next >