< prev index next > src/hotspot/cpu/x86/gc/shenandoah/shenandoahBarrierSetAssembler_x86.cpp
Print this page
/*
* Copyright (c) 2018, 2021, Red Hat, Inc. All rights reserved.
+ * Copyright Amazon.com Inc. or its affiliates. All Rights Reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
* under the terms of the GNU General Public License version 2 only, as
* published by the Free Software Foundation.
#include "gc/shenandoah/shenandoahHeap.inline.hpp"
#include "gc/shenandoah/shenandoahHeapRegion.hpp"
#include "gc/shenandoah/shenandoahRuntime.hpp"
#include "gc/shenandoah/shenandoahThreadLocalData.hpp"
#include "gc/shenandoah/heuristics/shenandoahHeuristics.hpp"
+ #include "gc/shenandoah/mode/shenandoahMode.hpp"
#include "interpreter/interpreter.hpp"
#include "runtime/javaThread.hpp"
#include "runtime/sharedRuntime.hpp"
#include "utilities/macros.hpp"
#ifdef COMPILER1
Register src, Register dst, Register count) {
bool dest_uninitialized = (decorators & IS_DEST_UNINITIALIZED) != 0;
if (is_reference_type(type)) {
+ if (ShenandoahCardBarrier) {
+ bool checkcast = (decorators & ARRAYCOPY_CHECKCAST) != 0;
+ bool disjoint = (decorators & ARRAYCOPY_DISJOINT) != 0;
+ bool obj_int = type == T_OBJECT LP64_ONLY(&& UseCompressedOops);
+
+ // We need to save the original element count because the array copy stub
+ // will destroy the value and we need it for the card marking barrier.
+ #ifdef _LP64
+ if (!checkcast) {
+ if (!obj_int) {
+ // Save count for barrier
+ __ movptr(r11, count);
+ } else if (disjoint) {
+ // Save dst in r11 in the disjoint case
+ __ movq(r11, dst);
+ }
+ }
+ #else
+ if (disjoint) {
+ __ mov(rdx, dst); // save 'to'
+ }
+ #endif
+ }
if ((ShenandoahSATBBarrier && !dest_uninitialized) || ShenandoahIUBarrier || ShenandoahLoadRefBarrier) {
#ifdef _LP64
Register thread = r15_thread;
#else
__ push(thread);
__ get_thread(thread);
#endif
assert_different_registers(src, dst, count, thread);
! Label done;
// Short-circuit if count == 0.
__ testptr(count, count);
! __ jcc(Assembler::zero, done);
// Avoid runtime call when not active.
Address gc_state(thread, in_bytes(ShenandoahThreadLocalData::gc_state_offset()));
int flags;
if (ShenandoahSATBBarrier && dest_uninitialized) {
flags = ShenandoahHeap::HAS_FORWARDED;
} else {
flags = ShenandoahHeap::HAS_FORWARDED | ShenandoahHeap::MARKING;
}
__ testb(gc_state, flags);
! __ jcc(Assembler::zero, done);
save_machine_state(masm, /* handle_gpr = */ true, /* handle_fp = */ false);
#ifdef _LP64
assert(src == rdi, "expected");
__ push(thread);
__ get_thread(thread);
#endif
assert_different_registers(src, dst, count, thread);
! Label L_done;
// Short-circuit if count == 0.
__ testptr(count, count);
! __ jcc(Assembler::zero, L_done);
// Avoid runtime call when not active.
Address gc_state(thread, in_bytes(ShenandoahThreadLocalData::gc_state_offset()));
int flags;
if (ShenandoahSATBBarrier && dest_uninitialized) {
flags = ShenandoahHeap::HAS_FORWARDED;
} else {
flags = ShenandoahHeap::HAS_FORWARDED | ShenandoahHeap::MARKING;
}
__ testb(gc_state, flags);
! __ jcc(Assembler::zero, L_done);
save_machine_state(masm, /* handle_gpr = */ true, /* handle_fp = */ false);
#ifdef _LP64
assert(src == rdi, "expected");
src, dst, count);
}
restore_machine_state(masm, /* handle_gpr = */ true, /* handle_fp = */ false);
! __ bind(done);
NOT_LP64(__ pop(thread);)
}
}
}
void ShenandoahBarrierSetAssembler::shenandoah_write_barrier_pre(MacroAssembler* masm,
Register obj,
Register pre_val,
Register thread,
Register tmp,
src, dst, count);
}
restore_machine_state(masm, /* handle_gpr = */ true, /* handle_fp = */ false);
! __ bind(L_done);
NOT_LP64(__ pop(thread);)
}
}
}
+ void ShenandoahBarrierSetAssembler::arraycopy_epilogue(MacroAssembler* masm, DecoratorSet decorators, BasicType type,
+ Register src, Register dst, Register count) {
+
+ if (ShenandoahCardBarrier && is_reference_type(type)) {
+ bool checkcast = (decorators & ARRAYCOPY_CHECKCAST) != 0;
+ bool disjoint = (decorators & ARRAYCOPY_DISJOINT) != 0;
+ bool obj_int = type == T_OBJECT LP64_ONLY(&& UseCompressedOops);
+ Register tmp = rax;
+
+ #ifdef _LP64
+ if (!checkcast) {
+ if (!obj_int) {
+ // Save count for barrier
+ count = r11;
+ } else if (disjoint) {
+ // Use the saved dst in the disjoint case
+ dst = r11;
+ }
+ } else {
+ tmp = rscratch1;
+ }
+ #else
+ if (disjoint) {
+ __ mov(dst, rdx); // restore 'to'
+ }
+ #endif
+ gen_write_ref_array_post_barrier(masm, decorators, dst, count, tmp);
+ }
+ }
+
void ShenandoahBarrierSetAssembler::shenandoah_write_barrier_pre(MacroAssembler* masm,
Register obj,
Register pre_val,
Register thread,
Register tmp,
restore_machine_state(masm, /* handle_gpr = */ true, /* handle_fp = */ true);
}
}
+ void ShenandoahBarrierSetAssembler::store_check(MacroAssembler* masm, Register obj) {
+ assert(ShenandoahCardBarrier, "Should have been checked by caller");
+
+ // Does a store check for the oop in register obj. The content of
+ // register obj is destroyed afterwards.
+
+ ShenandoahBarrierSet* ctbs = ShenandoahBarrierSet::barrier_set();
+ CardTable* ct = ctbs->card_table();
+
+ __ shrptr(obj, CardTable::card_shift());
+
+ Address card_addr;
+
+ // The calculation for byte_map_base is as follows:
+ // byte_map_base = _byte_map - (uintptr_t(low_bound) >> card_shift);
+ // So this essentially converts an address to a displacement and it will
+ // never need to be relocated. On 64-bit however the value may be too
+ // large for a 32-bit displacement.
+ intptr_t byte_map_base = (intptr_t)ct->byte_map_base();
+ if (__ is_simm32(byte_map_base)) {
+ card_addr = Address(noreg, obj, Address::times_1, byte_map_base);
+ } else {
+ // By doing it as an ExternalAddress 'byte_map_base' could be converted to a rip-relative
+ // displacement and done in a single instruction given favorable mapping and a
+ // smarter version of as_Address. However, 'ExternalAddress' generates a relocation
+ // entry and that entry is not properly handled by the relocation code.
+ AddressLiteral cardtable((address)byte_map_base, relocInfo::none);
+ Address index(noreg, obj, Address::times_1);
+ card_addr = __ as_Address(ArrayAddress(cardtable, index), rscratch1);
+ }
+
+ int dirty = CardTable::dirty_card_val();
+ if (UseCondCardMark) {
+ Label L_already_dirty;
+ __ cmpb(card_addr, dirty);
+ __ jccb(Assembler::equal, L_already_dirty);
+ __ movb(card_addr, dirty);
+ __ bind(L_already_dirty);
+ } else {
+ __ movb(card_addr, dirty);
+ }
+ }
+
void ShenandoahBarrierSetAssembler::store_at(MacroAssembler* masm, DecoratorSet decorators, BasicType type,
Address dst, Register val, Register tmp1, Register tmp2, Register tmp3) {
bool on_oop = is_reference_type(type);
bool in_heap = (decorators & IN_HEAP) != 0;
if (val == noreg) {
BarrierSetAssembler::store_at(masm, decorators, type, Address(tmp1, 0), val, noreg, noreg, noreg);
} else {
iu_barrier(masm, val, tmp3);
BarrierSetAssembler::store_at(masm, decorators, type, Address(tmp1, 0), val, noreg, noreg, noreg);
+ if (ShenandoahCardBarrier) {
+ store_check(masm, tmp1);
+ }
}
NOT_LP64(imasm->restore_bcp());
} else {
BarrierSetAssembler::store_at(masm, decorators, type, dst, val, tmp1, tmp2, tmp3);
}
__ movptr(res, 1);
__ bind(exit);
}
}
+ #ifdef PRODUCT
+ #define BLOCK_COMMENT(str) /* nothing */
+ #else
+ #define BLOCK_COMMENT(str) __ block_comment(str)
+ #endif
+
+ #define BIND(label) bind(label); BLOCK_COMMENT(#label ":")
+
+ #define TIMES_OOP (UseCompressedOops ? Address::times_4 : Address::times_8)
+
+ void ShenandoahBarrierSetAssembler::gen_write_ref_array_post_barrier(MacroAssembler* masm, DecoratorSet decorators,
+ Register addr, Register count,
+ Register tmp) {
+ assert(ShenandoahCardBarrier, "Should have been checked by caller");
+
+ ShenandoahBarrierSet* bs = ShenandoahBarrierSet::barrier_set();
+ CardTable* ct = bs->card_table();
+ intptr_t disp = (intptr_t) ct->byte_map_base();
+
+ Label L_loop, L_done;
+ const Register end = count;
+ assert_different_registers(addr, end);
+
+ // Zero count? Nothing to do.
+ __ testl(count, count);
+ __ jccb(Assembler::zero, L_done);
+
+ #ifdef _LP64
+ __ leaq(end, Address(addr, count, TIMES_OOP, 0)); // end == addr+count*oop_size
+ __ subptr(end, BytesPerHeapOop); // end - 1 to make inclusive
+ __ shrptr(addr, CardTable::card_shift());
+ __ shrptr(end, CardTable::card_shift());
+ __ subptr(end, addr); // end --> cards count
+
+ __ mov64(tmp, disp);
+ __ addptr(addr, tmp);
+
+ __ BIND(L_loop);
+ __ movb(Address(addr, count, Address::times_1), 0);
+ __ decrement(count);
+ __ jccb(Assembler::greaterEqual, L_loop);
+ #else
+ __ lea(end, Address(addr, count, Address::times_ptr, -wordSize));
+ __ shrptr(addr, CardTable::card_shift());
+ __ shrptr(end, CardTable::card_shift());
+ __ subptr(end, addr); // end --> count
+
+ __ BIND(L_loop);
+ Address cardtable(addr, count, Address::times_1, disp);
+ __ movb(cardtable, 0);
+ __ decrement(count);
+ __ jccb(Assembler::greaterEqual, L_loop);
+ #endif
+
+ __ BIND(L_done);
+ }
+
#undef __
#ifdef COMPILER1
#define __ ce->masm()->
< prev index next >