< prev index next > src/hotspot/cpu/aarch64/gc/shenandoah/shenandoahBarrierSetAssembler_aarch64.cpp
Print this page
#include "gc/shenandoah/mode/shenandoahMode.hpp"
#include "gc/shenandoah/shenandoahBarrierSet.hpp"
#include "gc/shenandoah/shenandoahBarrierSetAssembler.hpp"
#include "gc/shenandoah/shenandoahHeap.inline.hpp"
#include "gc/shenandoah/shenandoahHeapRegion.hpp"
+ #include "gc/shenandoah/shenandoahNMethod.inline.hpp"
#include "gc/shenandoah/shenandoahRuntime.hpp"
#include "gc/shenandoah/shenandoahThreadLocalData.hpp"
#include "interpreter/interp_masm.hpp"
#include "interpreter/interpreter.hpp"
+ #include "nativeInst_aarch64.hpp"
#include "runtime/javaThread.hpp"
#include "runtime/sharedRuntime.hpp"
#ifdef COMPILER1
#include "c1/c1_LIRAssembler.hpp"
#include "c1/c1_MacroAssembler.hpp"
} else {
__ strb(zr, Address(tmp2));
}
}
! void ShenandoahBarrierStubC2::enter_if_gc_state(MacroAssembler& masm, const char test_state, Register tmp) {
- Assembler::InlineSkippedInstructionsCounter skip_counter(&masm);
PhaseOutput* const output = Compile::current()->output();
! Address gc_state_fast(rthread, in_bytes(ShenandoahThreadLocalData::gc_state_fast_array_offset(test_state)));
! // We piggyback on scratch_emit_size mode to compute the slowpath stub size.
! // We'll use that information to decide whether we need a far jump to the
! // stub entry point or not. In scratch_emit_size mode we don't bind entry()
! // because otherwise it will be rebound when we later emit the instructions
! // for real.
! if (_needs_far_jump) {
! __ ldrb(tmp, gc_state_fast);
! __ cbz(tmp, *continuation());
! __ b(output->in_scratch_emit_size() ? *continuation() : *entry());
} else {
! __ ldrb(tmp, gc_state_fast);
! __ cbnz(tmp, output->in_scratch_emit_size() ? *continuation() : *entry());
}
! // This is were the slowpath stub will return to or the code above will
! // jump to if the checks are false
__ bind(*continuation());
}
void ShenandoahBarrierStubC2::emit_code(MacroAssembler& masm) {
Assembler::InlineSkippedInstructionsCounter skip_counter(&masm);
assert(_needs_keep_alive_barrier || _needs_load_ref_barrier, "Why are you here?");
- PhaseOutput* const output = Compile::current()->output();
! // We piggyback on scratch_emit_size mode to compute the slowpath stub size.
- // We'll use that information to decide whether we need a far jump to the
- // stub entry point or not. In scratch_emit_size mode we don't bind entry()
- // because otherwise it will be rebound when we later emit the instructions
- // for real.
- if (!output->in_scratch_emit_size()) {
- __ bind(*entry());
- }
// If we need to load ourselves, do it here.
if (_do_load) {
if (_narrow) {
__ ldrw(_obj, _addr);
} else {
__ strb(zr, Address(tmp2));
}
}
! void ShenandoahBarrierStubC2::patchable_jump(MacroAssembler& masm, const char gc_state, bool jump_when_state, Label* L_target, bool needs_far_jump) {
PhaseOutput* const output = Compile::current()->output();
! if (output->in_scratch_emit_size()) {
+ // We piggyback on scratch_emit_size mode to compute the slowpath stub size.
+ // Avoid binding L_target at this time. We know the patched check is exactly
+ // two instructions long.
+ __ nop();
+ __ nop();
+ return;
+ }
! // Emit the unconditional branch in the first version of the method.
! // Let the rest of runtime figure out how to manage it. If the jump target
! // is far away, we need to flip it to make sure patchable jumps are encodeable.
! if (needs_far_jump) {
! Label L_over;
! __ relocate(patchable_barrier_Relocation::spec(ShenandoahNMethod::encode_to_reloc(gc_state, !jump_when_state)));
! __ b(L_over);
! __ b(*L_target);
! __ bind(L_over);
} else {
! __ relocate(patchable_barrier_Relocation::spec(ShenandoahNMethod::encode_to_reloc(gc_state, jump_when_state)));
! __ b(*L_target);
}
+ }
! void ShenandoahBarrierStubC2::enter_if_gc_state(MacroAssembler& masm, const char test_state) {
! Assembler::InlineSkippedInstructionsCounter skip_counter(&masm);
+ patchable_jump_if_gc_state(masm, test_state, entry());
__ bind(*continuation());
}
+ address ShenandoahBarrierSetAssembler::parse_jump_address(address pc) {
+ NativeInstruction* ni = nativeInstruction_at(pc);
+ assert(ni->is_jump(), "Initial code version: GC barrier fastpath must be a jump");
+ NativeJump* jmp = nativeJump_at(pc);
+ return jmp->jump_destination();
+ }
+
+ static uint32_t encode_patchable_nop() {
+ return 0xD503201F;
+ }
+
+ static uint32_t encode_patchable_jump(address pc, address target_pc) {
+ int32_t disp = checked_cast<int32_t>((intptr_t)target_pc - (intptr_t)pc);
+ int64_t imm26 = disp >> 2;
+ assert(Assembler::is_simm(imm26, 26), "maximum offset is 128MiB");
+ return 0x14000000 | (imm26 & 0x03FFFFFF);
+ }
+
+ void ShenandoahBarrierSetAssembler::insert_patchable_nop(address pc) {
+ *((uint32_t*)pc) = encode_patchable_nop();
+ }
+
+ void ShenandoahBarrierSetAssembler::insert_patchable_jump(address pc, address target_pc) {
+ *((uint32_t*)pc) = encode_patchable_jump(pc, target_pc);
+ }
+
+ bool ShenandoahBarrierSetAssembler::is_patchable_nop(address pc) {
+ return *((uint32_t*)pc) == encode_patchable_nop();
+ }
+
+ bool ShenandoahBarrierSetAssembler::is_patchable_jump(address pc, address target_pc) {
+ return *((uint32_t*)pc) == encode_patchable_jump(pc, target_pc);
+ }
+
void ShenandoahBarrierStubC2::emit_code(MacroAssembler& masm) {
Assembler::InlineSkippedInstructionsCounter skip_counter(&masm);
assert(_needs_keep_alive_barrier || _needs_load_ref_barrier, "Why are you here?");
! __ bind(*entry());
// If we need to load ourselves, do it here.
if (_do_load) {
if (_narrow) {
__ ldrw(_obj, _addr);
__ cbz(reg, *continuation());
}
}
void ShenandoahBarrierStubC2::keepalive(MacroAssembler& masm, Label* L_done) {
- Address gcstate(rthread, in_bytes(ShenandoahThreadLocalData::gc_state_fast_array_offset(ShenandoahHeap::MARKING)));
Address index(rthread, in_bytes(ShenandoahThreadLocalData::satb_mark_queue_index_offset()));
Address buffer(rthread, in_bytes(ShenandoahThreadLocalData::satb_mark_queue_buffer_offset()));
Label L_through, L_slowpath;
! // If another barrier is enabled as well, do a runtime check for a specific barrier.
if (_needs_load_ref_barrier) {
! assert(L_done == nullptr, "L_done is always null when _needs_load_ref_barrier is true");
! __ ldrb(_tmp1, gcstate);
! __ cbz(_tmp1, L_through);
}
// Fast-path: put object into buffer.
// If buffer is already full, go slow.
__ ldr(_tmp1, index);
__ cbz(reg, *continuation());
}
}
void ShenandoahBarrierStubC2::keepalive(MacroAssembler& masm, Label* L_done) {
Address index(rthread, in_bytes(ShenandoahThreadLocalData::satb_mark_queue_index_offset()));
Address buffer(rthread, in_bytes(ShenandoahThreadLocalData::satb_mark_queue_buffer_offset()));
Label L_through, L_slowpath;
! // If another barrier is enabled as well, do a check for a specific barrier.
if (_needs_load_ref_barrier) {
! assert(L_done == nullptr, "Should be");
! char state_to_check = ShenandoahHeap::MARKING;
! patchable_short_jump_if_not_gc_state(masm, state_to_check, &L_through);
}
// Fast-path: put object into buffer.
// If buffer is already full, go slow.
__ ldr(_tmp1, index);
}
void ShenandoahBarrierStubC2::lrb(MacroAssembler& masm) {
Label L_slow;
! // If another barrier is enabled as well, do a runtime check for a specific barrier.
if (_needs_keep_alive_barrier) {
char state_to_check = ShenandoahHeap::HAS_FORWARDED | (_needs_load_ref_weak_barrier ? ShenandoahHeap::WEAK_ROOTS : 0);
! Address gc_state_fast(rthread, in_bytes(ShenandoahThreadLocalData::gc_state_fast_array_offset(state_to_check)));
- __ ldrb(_tmp1, gc_state_fast);
- maybe_far_jump_if_zero(masm, _tmp1);
}
// If weak references are being processed, weak/phantom loads need to go slow,
// regardless of their cset status.
if (_needs_load_ref_weak_barrier) {
! Address gc_state_fast(rthread, in_bytes(ShenandoahThreadLocalData::gc_state_fast_array_offset(ShenandoahHeap::WEAK_ROOTS)));
! __ ldrb(_tmp1, gc_state_fast);
- __ cbnz(_tmp1, L_slow);
}
// Cset-check. Fall-through to slow if in collection set.
bool is_aot = AOTCodeCache::is_on_for_dump();
if (!is_aot) {
}
void ShenandoahBarrierStubC2::lrb(MacroAssembler& masm) {
Label L_slow;
! // If another barrier is enabled as well, do a check for a specific barrier.
if (_needs_keep_alive_barrier) {
char state_to_check = ShenandoahHeap::HAS_FORWARDED | (_needs_load_ref_weak_barrier ? ShenandoahHeap::WEAK_ROOTS : 0);
! patchable_jump_if_not_gc_state(masm, state_to_check, continuation());
}
// If weak references are being processed, weak/phantom loads need to go slow,
// regardless of their cset status.
if (_needs_load_ref_weak_barrier) {
! char state_to_check = ShenandoahHeap::WEAK_ROOTS;
! patchable_short_jump_if_gc_state(masm, state_to_check, &L_slow);
}
// Cset-check. Fall-through to slow if in collection set.
bool is_aot = AOTCodeCache::is_on_for_dump();
if (!is_aot) {
bool ShenandoahBarrierStubC2::is_special_register(Register r) {
Unimplemented(); // Not used
return true;
}
! static ShenandoahBarrierSetC2State* barrier_set_state() {
! return reinterpret_cast<ShenandoahBarrierSetC2State*>(Compile::current()->barrier_set_state());
! }
!
- static int get_stub_size(ShenandoahBarrierStubC2* stub) {
- PhaseOutput* const output = Compile::current()->output();
- assert(output->in_scratch_emit_size(), "only used when in scratch_emit_size.");
- BufferBlob* const blob = output->scratch_buffer_blob();
- CodeBuffer cb(blob->content_begin(), (address)output->scratch_locs_memory() - blob->content_begin());
- MacroAssembler masm(&cb);
- stub->emit_code(masm);
- return cb.insts_size();
- }
-
- void ShenandoahBarrierStubC2::post_init() {
- // If we are in scratch emit mode we assume worst case, and force the use of
- // far branches.
- PhaseOutput* const output = Compile::current()->output();
- ShenandoahBarrierSetC2State* state = barrier_set_state();
- if (output->in_scratch_emit_size()) {
- state->inc_stubs_current_total_size(get_stub_size(this));
- _needs_far_jump = true;
- return;
- }
-
- // The logic implemented in this stub only uses short jumps (cbz, cbnz) if
- // the aggregation of all relevant code sections of a method is less than 1MB
- // - 2KB. We could be more aggressive and try and compute the distance
- // between the fastpath branch and the stub entry but in practice not many
- // methods reach the 1MB size.
- const BufferSizingData* sizing = output->buffer_sizing_data();
- const int code_size = sizing->_code + state->stubs_current_total_size();
-
- // Maximum backward range is 1M. Maximum forward reach is 1M - 4bytes.
- // Subtract 2K to be ultra conservative.
- const int cond_branch_max_reach = (int)(1*M - 2*K);
- _needs_far_jump = code_size >= cond_branch_max_reach;
}
#endif // COMPILER2
bool ShenandoahBarrierStubC2::is_special_register(Register r) {
Unimplemented(); // Not used
return true;
}
! int ShenandoahBarrierStubC2::max_branch_reach() {
! // For cbz/cbnz, the target range is 1M. For b, the range is 128M.
! // Choose the lowest range and subtract 2K to be ultra conservative.
! return (int)(1*M - 2*K);
}
#endif // COMPILER2
< prev index next >