< prev index next > src/hotspot/cpu/x86/gc/shenandoah/shenandoahBarrierSetAssembler_x86.cpp
Print this page
#include "gc/shenandoah/shenandoahHeap.inline.hpp"
#include "gc/shenandoah/shenandoahHeapRegion.hpp"
#include "gc/shenandoah/shenandoahRuntime.hpp"
#include "gc/shenandoah/shenandoahThreadLocalData.hpp"
#include "gc/shenandoah/heuristics/shenandoahHeuristics.hpp"
+ #include "gc/shenandoah/mode/shenandoahMode.hpp"
#include "interpreter/interpreter.hpp"
#include "runtime/javaThread.hpp"
#include "runtime/sharedRuntime.hpp"
#include "utilities/macros.hpp"
#ifdef COMPILER1
Register src, Register dst, Register count) {
bool dest_uninitialized = (decorators & IS_DEST_UNINITIALIZED) != 0;
if (is_reference_type(type)) {
+ if (ShenandoahHeap::heap()->mode()->is_generational()) {
+ bool checkcast = (decorators & ARRAYCOPY_CHECKCAST) != 0;
+ bool disjoint = (decorators & ARRAYCOPY_DISJOINT) != 0;
+ bool obj_int = type == T_OBJECT LP64_ONLY(&& UseCompressedOops);
+
+ // We need to squirrel away the original element count because the
+ // array copy assembly will destroy the value and we need it for the
+ // card marking barrier.
+ #ifdef _LP64
+ if (!checkcast) {
+ if (!obj_int) {
+ // Save count for barrier
+ __ movptr(r11, count);
+ } else if (disjoint) {
+ // Save dst in r11 in the disjoint case
+ __ movq(r11, dst);
+ }
+ }
+ #else
+ if (disjoint) {
+ __ mov(rdx, dst); // save 'to'
+ }
+ #endif
+ }
if ((ShenandoahSATBBarrier && !dest_uninitialized) || ShenandoahIUBarrier || ShenandoahLoadRefBarrier) {
#ifdef _LP64
Register thread = r15_thread;
#else
Address gc_state(thread, in_bytes(ShenandoahThreadLocalData::gc_state_offset()));
int flags;
if (ShenandoahSATBBarrier && dest_uninitialized) {
flags = ShenandoahHeap::HAS_FORWARDED;
} else {
! flags = ShenandoahHeap::HAS_FORWARDED | ShenandoahHeap::MARKING;
}
__ testb(gc_state, flags);
__ jcc(Assembler::zero, done);
save_machine_state(masm, /* handle_gpr = */ true, /* handle_fp = */ false);
Address gc_state(thread, in_bytes(ShenandoahThreadLocalData::gc_state_offset()));
int flags;
if (ShenandoahSATBBarrier && dest_uninitialized) {
flags = ShenandoahHeap::HAS_FORWARDED;
} else {
! flags = ShenandoahHeap::HAS_FORWARDED | ShenandoahHeap::YOUNG_MARKING | ShenandoahHeap::OLD_MARKING;
}
__ testb(gc_state, flags);
__ jcc(Assembler::zero, done);
save_machine_state(masm, /* handle_gpr = */ true, /* handle_fp = */ false);
}
}
}
+ void ShenandoahBarrierSetAssembler::arraycopy_epilogue(MacroAssembler* masm, DecoratorSet decorators, BasicType type,
+ Register src, Register dst, Register count) {
+ bool checkcast = (decorators & ARRAYCOPY_CHECKCAST) != 0;
+ bool disjoint = (decorators & ARRAYCOPY_DISJOINT) != 0;
+ bool obj_int = type == T_OBJECT LP64_ONLY(&& UseCompressedOops);
+ Register tmp = rax;
+
+ if (is_reference_type(type)) {
+ #ifdef _LP64
+ if (!checkcast) {
+ if (!obj_int) {
+ // Save count for barrier
+ count = r11;
+ } else if (disjoint) {
+ // Use the saved dst in the disjoint case
+ dst = r11;
+ }
+ } else {
+ tmp = rscratch1;
+ }
+ #else
+ if (disjoint) {
+ __ mov(dst, rdx); // restore 'to'
+ }
+ #endif
+ gen_write_ref_array_post_barrier(masm, decorators, dst, count, tmp);
+ }
+ }
+
void ShenandoahBarrierSetAssembler::shenandoah_write_barrier_pre(MacroAssembler* masm,
Register obj,
Register pre_val,
Register thread,
Register tmp,
Address in_progress(thread, in_bytes(ShenandoahThreadLocalData::satb_mark_queue_active_offset()));
Address index(thread, in_bytes(ShenandoahThreadLocalData::satb_mark_queue_index_offset()));
Address buffer(thread, in_bytes(ShenandoahThreadLocalData::satb_mark_queue_buffer_offset()));
Address gc_state(thread, in_bytes(ShenandoahThreadLocalData::gc_state_offset()));
! __ testb(gc_state, ShenandoahHeap::MARKING);
__ jcc(Assembler::zero, done);
// Do we need to load the previous value?
if (obj != noreg) {
__ load_heap_oop(pre_val, Address(obj, 0), noreg, noreg, AS_RAW);
Address in_progress(thread, in_bytes(ShenandoahThreadLocalData::satb_mark_queue_active_offset()));
Address index(thread, in_bytes(ShenandoahThreadLocalData::satb_mark_queue_index_offset()));
Address buffer(thread, in_bytes(ShenandoahThreadLocalData::satb_mark_queue_buffer_offset()));
Address gc_state(thread, in_bytes(ShenandoahThreadLocalData::gc_state_offset()));
! __ testb(gc_state, ShenandoahHeap::YOUNG_MARKING | ShenandoahHeap::OLD_MARKING);
__ jcc(Assembler::zero, done);
// Do we need to load the previous value?
if (obj != noreg) {
__ load_heap_oop(pre_val, Address(obj, 0), noreg, noreg, AS_RAW);
restore_machine_state(masm, /* handle_gpr = */ true, /* handle_fp = */ true);
}
}
+ void ShenandoahBarrierSetAssembler::store_check(MacroAssembler* masm, Register obj) {
+ if (!ShenandoahHeap::heap()->mode()->is_generational()) {
+ return;
+ }
+
+ // Does a store check for the oop in register obj. The content of
+ // register obj is destroyed afterwards.
+
+ ShenandoahBarrierSet* ctbs = ShenandoahBarrierSet::barrier_set();
+ CardTable* ct = ctbs->card_table();
+
+ __ shrptr(obj, CardTable::card_shift());
+
+ Address card_addr;
+
+ // The calculation for byte_map_base is as follows:
+ // byte_map_base = _byte_map - (uintptr_t(low_bound) >> card_shift);
+ // So this essentially converts an address to a displacement and it will
+ // never need to be relocated. On 64bit however the value may be too
+ // large for a 32bit displacement.
+ intptr_t byte_map_base = (intptr_t)ct->byte_map_base();
+ if (__ is_simm32(byte_map_base)) {
+ card_addr = Address(noreg, obj, Address::times_1, byte_map_base);
+ } else {
+ // By doing it as an ExternalAddress 'byte_map_base' could be converted to a rip-relative
+ // displacement and done in a single instruction given favorable mapping and a
+ // smarter version of as_Address. However, 'ExternalAddress' generates a relocation
+ // entry and that entry is not properly handled by the relocation code.
+ AddressLiteral cardtable((address)byte_map_base, relocInfo::none);
+ Address index(noreg, obj, Address::times_1);
+ card_addr = __ as_Address(ArrayAddress(cardtable, index), rscratch1);
+ }
+
+ int dirty = CardTable::dirty_card_val();
+ if (UseCondCardMark) {
+ Label L_already_dirty;
+ __ cmpb(card_addr, dirty);
+ __ jcc(Assembler::equal, L_already_dirty);
+ __ movb(card_addr, dirty);
+ __ bind(L_already_dirty);
+ } else {
+ __ movb(card_addr, dirty);
+ }
+ }
+
void ShenandoahBarrierSetAssembler::store_at(MacroAssembler* masm, DecoratorSet decorators, BasicType type,
Address dst, Register val, Register tmp1, Register tmp2, Register tmp3) {
bool on_oop = is_reference_type(type);
bool in_heap = (decorators & IN_HEAP) != 0;
}
if (val == noreg) {
BarrierSetAssembler::store_at(masm, decorators, type, Address(tmp1, 0), val, noreg, noreg, noreg);
} else {
iu_barrier(masm, val, tmp3);
+ // XXX: store_check missing from upstream
BarrierSetAssembler::store_at(masm, decorators, type, Address(tmp1, 0), val, noreg, noreg, noreg);
+ store_check(masm, tmp1);
}
NOT_LP64(imasm->restore_bcp());
} else {
BarrierSetAssembler::store_at(masm, decorators, type, dst, val, tmp1, tmp2, tmp3);
}
__ movptr(res, 1);
__ bind(exit);
}
}
+ #ifdef PRODUCT
+ #define BLOCK_COMMENT(str) /* nothing */
+ #else
+ #define BLOCK_COMMENT(str) __ block_comment(str)
+ #endif
+
+ #define BIND(label) bind(label); BLOCK_COMMENT(#label ":")
+
+ #define TIMES_OOP (UseCompressedOops ? Address::times_4 : Address::times_8)
+
+ void ShenandoahBarrierSetAssembler::gen_write_ref_array_post_barrier(MacroAssembler* masm, DecoratorSet decorators, Register addr, Register count, Register tmp) {
+ if (!ShenandoahHeap::heap()->mode()->is_generational()) {
+ return;
+ }
+
+ ShenandoahBarrierSet* bs = ShenandoahBarrierSet::barrier_set();
+ CardTable* ct = bs->card_table();
+ intptr_t disp = (intptr_t) ct->byte_map_base();
+
+ Label L_loop, L_done;
+ const Register end = count;
+ assert_different_registers(addr, end);
+
+ __ testl(count, count);
+ __ jcc(Assembler::zero, L_done); // zero count - nothing to do
+
+
+ #ifdef _LP64
+ __ leaq(end, Address(addr, count, TIMES_OOP, 0)); // end == addr+count*oop_size
+ __ subptr(end, BytesPerHeapOop); // end - 1 to make inclusive
+ __ shrptr(addr, CardTable::card_shift());
+ __ shrptr(end, CardTable::card_shift());
+ __ subptr(end, addr); // end --> cards count
+
+ __ mov64(tmp, disp);
+ __ addptr(addr, tmp);
+ __ BIND(L_loop);
+ __ movb(Address(addr, count, Address::times_1), 0);
+ __ decrement(count);
+ __ jcc(Assembler::greaterEqual, L_loop);
+ #else
+ __ lea(end, Address(addr, count, Address::times_ptr, -wordSize));
+ __ shrptr(addr, CardTable::card_shift());
+ __ shrptr(end, CardTable::card_shift());
+ __ subptr(end, addr); // end --> count
+ __ BIND(L_loop);
+ Address cardtable(addr, count, Address::times_1, disp);
+ __ movb(cardtable, 0);
+ __ decrement(count);
+ __ jcc(Assembler::greaterEqual, L_loop);
+ #endif
+
+ __ BIND(L_done);
+ }
+
#undef __
#ifdef COMPILER1
#define __ ce->masm()->
Label done;
Label runtime;
// Is SATB still active?
Address gc_state(thread, in_bytes(ShenandoahThreadLocalData::gc_state_offset()));
! __ testb(gc_state, ShenandoahHeap::MARKING);
__ jcc(Assembler::zero, done);
// Can we store original value in the thread's buffer?
__ movptr(tmp, queue_index);
Label done;
Label runtime;
// Is SATB still active?
Address gc_state(thread, in_bytes(ShenandoahThreadLocalData::gc_state_offset()));
! __ testb(gc_state, ShenandoahHeap::YOUNG_MARKING | ShenandoahHeap::OLD_MARKING);
__ jcc(Assembler::zero, done);
// Can we store original value in the thread's buffer?
__ movptr(tmp, queue_index);
< prev index next >