< prev index next > src/hotspot/cpu/x86/gc/shenandoah/shenandoahBarrierSetAssembler_x86.cpp
Print this page
#ifdef COMPILER1
#include "c1/c1_LIRAssembler.hpp"
#include "c1/c1_MacroAssembler.hpp"
#include "gc/shenandoah/c1/shenandoahBarrierSetC1.hpp"
#endif
+ #ifdef COMPILER2
+ #include "gc/shenandoah/c2/shenandoahBarrierSetC2.hpp"
+ #endif
#define __ masm->
static void save_machine_state(MacroAssembler* masm, bool handle_gpr, bool handle_fp) {
if (handle_gpr) {
}
#undef __
#endif // COMPILER1
+
+ #ifdef COMPILER2
+
+ #undef __
+ #define __ masm->
+
+ void ShenandoahBarrierSetAssembler::load_c2(const MachNode* node, MacroAssembler* masm, Register dst, Address src, bool narrow) {
+ // Do the actual load. This load is the candidate for implicit null check, and MUST come first.
+ if (narrow) {
+ __ movl(dst, src);
+ } else {
+ __ movq(dst, src);
+ }
+
+ // Post-barrier: LRB
+ if (ShenandoahBarrierStubC2::needs_slow_barrier(node)) {
+ ShenandoahBarrierStubC2* const stub = ShenandoahBarrierStubC2::create(node, dst, src, narrow, false);
+ char check = 0;
+ check |= ShenandoahBarrierStubC2::needs_keep_alive_barrier(node) ? ShenandoahHeap::MARKING : 0;
+ check |= ShenandoahBarrierStubC2::needs_load_ref_barrier(node) ? ShenandoahHeap::HAS_FORWARDED : 0;
+ check |= ShenandoahBarrierStubC2::needs_load_ref_barrier_weak(node) ? ShenandoahHeap::WEAK_ROOTS : 0;
+ stub->enter_if_gc_state(*masm, check);
+ }
+ }
+
+ void ShenandoahBarrierSetAssembler::store_c2(const MachNode* node, MacroAssembler* masm,
+ Address dst, bool dst_narrow,
+ Register src, bool src_narrow,
+ Register tmp) {
+
+ // Pre-barrier: SATB, keep-alive the current memory value.
+ if (ShenandoahBarrierStubC2::needs_slow_barrier(node)) {
+ assert(!ShenandoahBarrierStubC2::needs_load_ref_barrier(node), "Should not be required for stores");
+ ShenandoahBarrierStubC2* const stub = ShenandoahBarrierStubC2::create(node, tmp, dst, dst_narrow, true);
+ stub->enter_if_gc_state(*masm, ShenandoahHeap::MARKING);
+ }
+
+ // Need to encode into tmp, because we cannot clobber src.
+ // TODO: Maybe there is a matcher way to test that src is unused after this?
+ if (dst_narrow && !src_narrow) {
+ __ movq(tmp, src);
+ if (ShenandoahBarrierStubC2::maybe_null(node)) {
+ __ encode_heap_oop(tmp);
+ } else {
+ __ encode_heap_oop_not_null(tmp);
+ }
+ src = tmp;
+ }
+
+ // Do the actual store
+ if (dst_narrow) {
+ __ movl(dst, src);
+ } else {
+ __ movq(dst, src);
+ }
+
+ // Post-barrier: card updates.
+ if (ShenandoahBarrierStubC2::needs_card_barrier(node)) {
+ card_barrier_c2(masm, dst, tmp);
+ }
+ }
+
+ void ShenandoahBarrierSetAssembler::compare_and_set_c2(const MachNode* node, MacroAssembler* masm,
+ Register res, Address addr,
+ Register oldval, Register newval, Register tmp,
+ bool narrow) {
+
+ assert(oldval == rax, "must be in rax for implicit use in cmpxchg");
+
+ // Oldval and newval can be in the same register, but all other registers should be
+ // distinct for extra safety, as we shuffle register values around.
+ assert_different_registers(oldval, tmp, addr.base(), addr.index());
+ assert_different_registers(newval, tmp, addr.base(), addr.index());
+
+ // Pre-barrier covers several things:
+ // a. Avoids false positives from CAS encountering to-space memory values.
+ // b. Satisfies the need for LRB for the CAE result.
+ // c. Records old value for the sake of SATB.
+ //
+ // (a) and (b) are covered because load barrier does memory location fixup.
+ // (c) is covered by KA on the current memory value.
+ if (ShenandoahBarrierStubC2::needs_slow_barrier(node)) {
+ ShenandoahBarrierStubC2* const stub = ShenandoahBarrierStubC2::create(node, tmp, addr, narrow, true);
+ char check = 0;
+ check |= ShenandoahBarrierStubC2::needs_keep_alive_barrier(node) ? ShenandoahHeap::MARKING : 0;
+ check |= ShenandoahBarrierStubC2::needs_load_ref_barrier(node) ? ShenandoahHeap::HAS_FORWARDED : 0;
+ assert(!ShenandoahBarrierStubC2::needs_load_ref_barrier_weak(node), "Not supported for CAS");
+ stub->enter_if_gc_state(*masm, check);
+ }
+
+ // CAS!
+ __ lock();
+ if (narrow) {
+ __ cmpxchgl(newval, addr);
+ } else {
+ __ cmpxchgptr(newval, addr);
+ }
+
+ // If we need a boolean result out of CAS, set the flag appropriately and promote the result.
+ if (res != noreg) {
+ __ setcc(Assembler::equal, res);
+ }
+
+ // Post-barrier deals with card updates.
+ if (ShenandoahBarrierStubC2::needs_card_barrier(node)) {
+ card_barrier_c2(masm, addr, tmp);
+ }
+ }
+
+ void ShenandoahBarrierSetAssembler::get_and_set_c2(const MachNode* node, MacroAssembler* masm, Register newval, Address addr, Register tmp, bool narrow) {
+ assert_different_registers(newval, tmp, addr.base(), addr.index());
+
+ // Pre-barrier covers several things:
+ // a. Satisfies the need for LRB for the GAS result.
+ // b. Records old value for the sake of SATB.
+ //
+ // (a) is covered because load barrier does memory location fixup.
+ // (b) is covered by KA on the current memory value.
+ if (ShenandoahBarrierStubC2::needs_slow_barrier(node)) {
+ ShenandoahBarrierStubC2* const stub = ShenandoahBarrierStubC2::create(node, tmp, addr, narrow, true);
+ char check = 0;
+ check |= ShenandoahBarrierStubC2::needs_keep_alive_barrier(node) ? ShenandoahHeap::MARKING : 0;
+ check |= ShenandoahBarrierStubC2::needs_load_ref_barrier(node) ? ShenandoahHeap::HAS_FORWARDED : 0;
+ assert(!ShenandoahBarrierStubC2::needs_load_ref_barrier_weak(node), "Not supported for GAS");
+ stub->enter_if_gc_state(*masm, check);
+ }
+
+ if (narrow) {
+ __ xchgl(newval, addr);
+ } else {
+ __ xchgq(newval, addr);
+ }
+
+ // Post-barrier deals with card updates.
+ if (ShenandoahBarrierStubC2::needs_card_barrier(node)) {
+ card_barrier_c2(masm, addr, tmp);
+ }
+ }
+
+ void ShenandoahBarrierSetAssembler::card_barrier_c2(MacroAssembler* masm, Address dst, Register tmp) {
+ Assembler::InlineSkippedInstructionsCounter skip_counter(masm);
+
+ // TODO: Might be a good place to implement some filters here.
+ // For example, G1 only flips card marks for stores within a single region.
+
+ __ lea(tmp, dst);
+ __ shrptr(tmp, CardTable::card_shift());
+ __ addptr(tmp, Address(r15_thread, in_bytes(ShenandoahThreadLocalData::card_table_offset())));
+ Address card_address(tmp, 0);
+
+ assert(CardTable::dirty_card_val() == 0, "Encoding assumption");
+ Label L_done;
+ if (UseCondCardMark) {
+ __ cmpb(card_address, 0);
+ __ jccb(Assembler::equal, L_done);
+ }
+ if (UseCompressedOops && CompressedOops::base() == nullptr) {
+ __ movb(card_address, r12);
+ } else {
+ __ movb(card_address, 0);
+ }
+ __ bind(L_done);
+ }
+
+ #undef __
+ #define __ masm.
+
+ void ShenandoahBarrierStubC2::enter_if_gc_state(MacroAssembler& masm, const char test_state) {
+ Assembler::InlineSkippedInstructionsCounter skip_counter(&masm);
+
+ Address gc_state_fast(r15_thread, in_bytes(ShenandoahThreadLocalData::gc_state_fast_offset()));
+ __ testb(gc_state_fast, ShenandoahThreadLocalData::gc_state_to_fast(test_state));
+ __ jcc(Assembler::notZero, *entry());
+ __ bind(*continuation());
+ }
+
+ void ShenandoahBarrierStubC2::emit_code(MacroAssembler& masm) {
+ Assembler::InlineSkippedInstructionsCounter skip_counter(&masm);
+
+ assert(_needs_keep_alive_barrier || _needs_load_ref_barrier, "Why are you here?");
+
+ __ align(InteriorEntryAlignment);
+ __ bind(*entry());
+
+ // If we need to load ourselves, do it here.
+ if (_do_load) {
+ if (_narrow) {
+ __ movl(_obj, _addr);
+ } else {
+ __ movq(_obj, _addr);
+ }
+ }
+
+ // If the object is null, there is no point in applying barriers.
+ if (_narrow) {
+ __ testl(_obj, _obj);
+ } else {
+ __ testptr(_obj, _obj);
+ }
+ __ jcc(Assembler::zero, *continuation());
+
+ // Barriers need temp to work, allocate one now.
+ bool tmp_live;
+ Register tmp = select_temp_register(tmp_live, _addr, _obj);
+ if (tmp_live) {
+ push_save_register(masm, tmp);
+ }
+
+ // Go for barriers. Barriers can return straight to continuation, as long
+ // as another barrier is not needed and tmp is not live.
+ if (_needs_keep_alive_barrier) {
+ keepalive(masm, _obj, tmp, (_needs_load_ref_barrier || tmp_live) ? nullptr : continuation());
+ }
+ if (_needs_load_ref_barrier) {
+ lrb(masm, _obj, _addr, tmp, (tmp_live) ? nullptr : continuation());
+ }
+
+ if (tmp_live) {
+ pop_save_register(masm, tmp);
+ __ jmp(*continuation());
+ } else {
+ #ifdef ASSERT
+ __ hlt(); // Should not reach here
+ #endif
+ }
+ }
+
+ void ShenandoahBarrierStubC2::keepalive(MacroAssembler& masm, Register obj, Register tmp, Label* L_done) {
+ Address index(r15_thread, in_bytes(ShenandoahThreadLocalData::satb_mark_queue_index_offset()));
+ Address buffer(r15_thread, in_bytes(ShenandoahThreadLocalData::satb_mark_queue_buffer_offset()));
+
+ Label L_through, L_fast, L_pack_and_done;
+
+ // If another barrier is enabled as well, do a runtime check for a specific barrier.
+ if (_needs_load_ref_barrier) {
+ Address gc_state(r15_thread, in_bytes(ShenandoahThreadLocalData::gc_state_offset()));
+ __ testb(gc_state, ShenandoahHeap::MARKING);
+ __ jcc(Assembler::zero, (L_done != nullptr) ? *L_done : L_through);
+ }
+
+ // If object is narrow, we need to unpack it before inserting into buffer.
+ if (_narrow) {
+ __ decode_heap_oop_not_null(obj);
+ }
+
+ // Check if buffer is already full. Go slow, if so.
+ __ movptr(tmp, index);
+ __ testptr(tmp, tmp);
+ __ jcc(Assembler::notZero, L_fast);
+
+ // Slow-path: call runtime to handle.
+ {
+ // Shuffle in the arguments. The end result should be:
+ // c_rarg0 <-- obj
+ //
+ // Save clobbered registers before overwriting them.
+ bool clobbered_c_rarg0 = false;
+ if (c_rarg0 != obj) {
+ clobbered_c_rarg0 = push_save_register_if_live(masm, c_rarg0);
+ __ mov(c_rarg0, obj);
+ }
+
+ // Handle the rest there.
+ if (!ShenandoahFasterRuntimeStubs || has_live_vector_registers()) {
+ __ call(RuntimeAddress(keepalive_runtime_entry_addr(SaveMode::All)));
+ } else if (has_save_space_for_live_gp_registers(clobbered_c_rarg0, false, false)) {
+ save_live_gp_regs(masm, clobbered_c_rarg0, false, false);
+ __ call(RuntimeAddress(keepalive_runtime_entry_addr(SaveMode::Nothing)));
+ restore_live_gp_regs(masm, clobbered_c_rarg0, false, false);
+ } else {
+ __ call(RuntimeAddress(keepalive_runtime_entry_addr(SaveMode::GP)));
+ }
+
+ // Restore the clobbered registers.
+ if (clobbered_c_rarg0) {
+ pop_save_register(masm, c_rarg0);
+ }
+ __ jmpb(L_pack_and_done);
+ }
+
+ // Fast-path: put object into buffer.
+ __ bind(L_fast);
+ __ subptr(tmp, wordSize);
+ __ movptr(index, tmp);
+ __ addptr(tmp, buffer);
+ __ movptr(Address(tmp, 0), obj);
+
+ // Exit here.
+ __ bind(L_pack_and_done);
+
+ // Pack the object back if needed. We can skip this if we performed
+ // the load ourselves: the value is not used by the caller.
+ if (_narrow && !_do_load) {
+ __ encode_heap_oop_not_null(obj);
+ }
+ if (L_done != nullptr) {
+ __ jmp(*L_done);
+ } else {
+ // Fall-through
+ __ bind(L_through);
+ }
+ }
+
+ void ShenandoahBarrierStubC2::lrb(MacroAssembler& masm, Register obj, Address addr, Register tmp, Label* L_done) {
+ Label L_through, L_slow;
+
+ // If another barrier is enabled as well, do a runtime check for a specific barrier.
+ if (_needs_keep_alive_barrier) {
+ Address gc_state(r15_thread, in_bytes(ShenandoahThreadLocalData::gc_state_offset()));
+ __ testb(gc_state, ShenandoahHeap::HAS_FORWARDED | (_needs_load_ref_weak_barrier ? ShenandoahHeap::WEAK_ROOTS : 0));
+ __ jcc(Assembler::zero, (L_done != nullptr) ? *L_done : L_through);
+ }
+
+ // If weak references are being processed, weak/phantom loads need to go slow,
+ // regadless of their cset status.
+ if (_needs_load_ref_weak_barrier) {
+ Address gc_state(r15_thread, in_bytes(ShenandoahThreadLocalData::gc_state_offset()));
+ __ testb(gc_state, ShenandoahHeap::WEAK_ROOTS);
+ __ jccb(Assembler::notZero, L_slow);
+ }
+
+ // Compute the cset bitmap index
+ if (_narrow) {
+ __ decode_heap_oop_not_null(tmp, obj);
+ } else {
+ __ movptr(tmp, obj);
+ }
+ __ shrptr(tmp, ShenandoahHeapRegion::region_size_bytes_shift_jint());
+
+ // If cset address is in good spot to just use it as offset. It almost always is.
+ Address cset_addr_arg;
+ intptr_t cset_addr = (intptr_t) ShenandoahHeap::in_cset_fast_test_addr();
+ if ((cset_addr >> 3) < INT32_MAX) {
+ assert(is_aligned(cset_addr, 8), "Sanity");
+ cset_addr_arg = Address(tmp, checked_cast<int>(cset_addr >> 3), Address::times_8);
+ } else {
+ __ addptr(tmp, cset_addr);
+ cset_addr_arg = Address(tmp, 0);
+ }
+
+ // Cset-check. Fall-through to slow if in collection set.
+ __ cmpb(cset_addr_arg, 0);
+ __ jcc(Assembler::equal, (L_done != nullptr) ? *L_done : L_through);
+
+ // Slow path
+ __ bind(L_slow);
+ {
+ assert_different_registers(rax, c_rarg0, c_rarg1);
+
+ // Shuffle in the arguments. The end result should be:
+ // c_rarg0 <-- obj
+ // c_rarg1 <-- lea(addr)
+ //
+ // Save clobbered registers before overwriting them, unless they
+ // carry obj, which would be overwritten on return.
+ bool clobbered_c_rarg0 = false;
+ bool clobbered_c_rarg1 = false;
+ bool clobbered_rax = false;
+
+ if (obj == c_rarg0) {
+ clobbered_c_rarg1 = push_save_register_if_live(masm, c_rarg1);
+ __ lea(c_rarg1, addr);
+ } else if (obj == c_rarg1) {
+ // Set up arguments in reverse, and then flip them
+ clobbered_c_rarg0 = push_save_register_if_live(masm, c_rarg0);
+ __ lea(c_rarg0, addr);
+ __ xchgptr(c_rarg0, c_rarg1);
+ } else {
+ assert_different_registers(obj, c_rarg0, c_rarg1);
+ clobbered_c_rarg0 = push_save_register_if_live(masm, c_rarg0);
+ clobbered_c_rarg1 = push_save_register_if_live(masm, c_rarg1);
+ __ lea(c_rarg1, addr);
+ __ movptr(c_rarg0, obj);
+ }
+ if (obj != rax) {
+ clobbered_rax = push_save_register_if_live(masm, rax);
+ }
+
+ // Decode if needed.
+ if (_narrow) {
+ __ decode_heap_oop_not_null(c_rarg0);
+ }
+
+ // Go to runtime stub and handle the rest there.
+ if (!ShenandoahFasterRuntimeStubs || has_live_vector_registers()) {
+ __ call(RuntimeAddress(lrb_runtime_entry_addr(SaveMode::All)));
+ } else if (has_save_space_for_live_gp_registers(clobbered_c_rarg0, clobbered_c_rarg1, true)) {
+ save_live_gp_regs(masm, clobbered_c_rarg0, clobbered_c_rarg1, true);
+ __ call(RuntimeAddress(lrb_runtime_entry_addr(SaveMode::Nothing)));
+ restore_live_gp_regs(masm, clobbered_c_rarg0, clobbered_c_rarg1, true);
+ } else {
+ __ call(RuntimeAddress(lrb_runtime_entry_addr(SaveMode::GP)));
+ }
+
+ // Save the result where needed and restore the clobbered registers.
+ if (obj != rax) {
+ __ movptr(obj, rax);
+ }
+ // If object is narrow, we need to encode it before exiting.
+ // For encoding, dst can only turn null if we are dealing with weak loads.
+ // Otherwise, we have already null-checked. We can skip this if we performed
+ // the load ourselves: the value is not used by the caller.
+ if (_narrow && !_do_load) {
+ if (_needs_load_ref_weak_barrier) {
+ __ encode_heap_oop(obj);
+ } else {
+ __ encode_heap_oop_not_null(obj);
+ }
+ }
+ if (clobbered_rax) {
+ pop_save_register(masm, rax);
+ }
+ if (clobbered_c_rarg1) {
+ pop_save_register(masm, c_rarg1);
+ }
+ if (clobbered_c_rarg0) {
+ pop_save_register(masm, c_rarg0);
+ }
+ }
+
+ // Exit here
+ if (L_done != nullptr) {
+ __ jmp(*L_done);
+ } else {
+ // Fall-through.
+ __ bind(L_through);
+ }
+ }
+
+ bool ShenandoahBarrierStubC2::has_live_vector_registers() {
+ return _has_live_vector_registers;
+ }
+
+ bool ShenandoahBarrierStubC2::is_live(Register reg) {
+ return _live_gp.contains(reg);
+ }
+
+ bool ShenandoahBarrierStubC2::has_save_space_for_live_gp_registers(bool skip_crarg0, bool skip_crarg1, bool skip_rax) {
+ int c = 0;
+ for (int i = 0; i < _live_gp.length(); i++) {
+ Register r = _live_gp.at(i);
+ if (skip_rax && (r == rax)) continue;
+ if (skip_crarg0 && (r == c_rarg0)) continue;
+ if (skip_crarg1 && (r == c_rarg1)) continue;
+ c++;
+ }
+ return c <= (ShenandoahBarrierSetC2::bsc2()->reserved_slots() - _save_slots_idx);
+ }
+
+ void ShenandoahBarrierStubC2::save_live_gp_regs(MacroAssembler& masm, bool skip_crarg0, bool skip_crarg1, bool skip_rax) {
+ for (int i = 0; i < _live_gp.length(); i++) {
+ Register r = _live_gp.at(i);
+ if (skip_rax && (r == rax)) continue;
+ if (skip_crarg0 && (r == c_rarg0)) continue;
+ if (skip_crarg1 && (r == c_rarg1)) continue;
+ push_save_register(masm, r);
+ }
+ }
+
+ void ShenandoahBarrierStubC2::restore_live_gp_regs(MacroAssembler& masm, bool skip_crarg0, bool skip_crarg1, bool skip_rax) {
+ for (int i = _live_gp.length() - 1; i >= 0; i--) {
+ Register r = _live_gp.at(i);
+ if (skip_rax && (r == rax)) continue;
+ if (skip_crarg0 && (r == c_rarg0)) continue;
+ if (skip_crarg1 && (r == c_rarg1)) continue;
+ pop_save_register(masm, r);
+ }
+ }
+
+ bool ShenandoahBarrierStubC2::push_save_register_if_live(MacroAssembler& masm, Register reg) {
+ if (is_live(reg)) {
+ push_save_register(masm, reg);
+ return true;
+ } else {
+ return false;
+ }
+ }
+
+ void ShenandoahBarrierStubC2::push_save_register(MacroAssembler& masm, Register reg) {
+ __ movptr(Address(rsp, push_save_slot()), reg);
+ }
+
+ void ShenandoahBarrierStubC2::pop_save_register(MacroAssembler& masm, Register reg) {
+ __ movptr(reg, Address(rsp, pop_save_slot()));
+ }
+
+ Register ShenandoahBarrierStubC2::select_temp_register(bool& selected_live, Address addr, Register reg1) {
+ Register tmp = noreg;
+ Register fallback_live = noreg;
+
+ // Try to select non-live first:
+ for (int i = 0; i < Register::available_gp_registers(); i++) {
+ Register r = as_Register(i);
+ if (r != rsp && r != rbp && r != r12_heapbase && r != r15_thread &&
+ r != reg1 && r != addr.base() && r != addr.index()) {
+ if (!is_live(r)) {
+ tmp = r;
+ break;
+ } else if (fallback_live == noreg) {
+ fallback_live = r;
+ }
+ }
+ }
+
+ // If we could not find a non-live register, select the live fallback:
+ if (tmp == noreg) {
+ tmp = fallback_live;
+ selected_live = true;
+ } else {
+ selected_live = false;
+ }
+
+ assert(tmp != noreg, "successfully selected");
+ assert_different_registers(tmp, reg1);
+ assert_different_registers(tmp, addr.base());
+ assert_different_registers(tmp, addr.index());
+ return tmp;
+ }
+
+ void ShenandoahBarrierStubC2::post_init(int offset) {
+ // Precompute live registers.
+ assert(_live_gp.is_empty(), "sanity: initial state");
+ assert(!_has_live_vector_registers, "sanity: initial state");
+ RegMaskIterator rmi(preserve_set());
+ while (rmi.has_next()) {
+ const OptoReg::Name opto_reg = rmi.next();
+ const VMReg vm_reg = OptoReg::as_VMReg(opto_reg);
+ if (vm_reg->is_Register()) {
+ Register r = vm_reg->as_Register();
+ if (r == rsp) continue;
+ _live_gp.append_if_missing(r);
+ } else if (vm_reg->is_KRegister()) {
+ _has_live_vector_registers = true;
+ } else if (vm_reg->is_XMMRegister()) {
+ _has_live_vector_registers = true;
+ } else {
+ fatal("Unexpected register type");
+ }
+ }
+ }
+ #undef __
+ #endif
< prev index next >