< prev index next > src/hotspot/share/gc/shenandoah/c2/shenandoahBarrierSetC2.cpp
Print this page
/*
! * Copyright (c) 2018, 2021, Red Hat, Inc. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
* under the terms of the GNU General Public License version 2 only, as
* published by the Free Software Foundation.
/*
! * Copyright (c) 2018, 2023, Red Hat, Inc. All rights reserved.
+ * Copyright Amazon.com Inc. or its affiliates. All Rights Reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
* under the terms of the GNU General Public License version 2 only, as
* published by the Free Software Foundation.
#include "precompiled.hpp"
#include "classfile/javaClasses.hpp"
#include "gc/shared/barrierSet.hpp"
#include "gc/shenandoah/shenandoahBarrierSet.hpp"
+ #include "gc/shenandoah/shenandoahCardTable.hpp"
#include "gc/shenandoah/shenandoahForwarding.hpp"
#include "gc/shenandoah/shenandoahHeap.hpp"
#include "gc/shenandoah/shenandoahRuntime.hpp"
#include "gc/shenandoah/shenandoahThreadLocalData.hpp"
#include "gc/shenandoah/c2/shenandoahBarrierSetC2.hpp"
ShenandoahBarrierSetC2* ShenandoahBarrierSetC2::bsc2() {
return reinterpret_cast<ShenandoahBarrierSetC2*>(BarrierSet::barrier_set()->barrier_set_c2());
}
ShenandoahBarrierSetC2State::ShenandoahBarrierSetC2State(Arena* comp_arena)
! : _iu_barriers(new (comp_arena) GrowableArray<ShenandoahIUBarrierNode*>(comp_arena, 8, 0, nullptr)),
- _load_reference_barriers(new (comp_arena) GrowableArray<ShenandoahLoadReferenceBarrierNode*>(comp_arena, 8, 0, nullptr)) {
- }
-
- int ShenandoahBarrierSetC2State::iu_barriers_count() const {
- return _iu_barriers->length();
- }
-
- ShenandoahIUBarrierNode* ShenandoahBarrierSetC2State::iu_barrier(int idx) const {
- return _iu_barriers->at(idx);
- }
-
- void ShenandoahBarrierSetC2State::add_iu_barrier(ShenandoahIUBarrierNode* n) {
- assert(!_iu_barriers->contains(n), "duplicate entry in barrier list");
- _iu_barriers->append(n);
- }
-
- void ShenandoahBarrierSetC2State::remove_iu_barrier(ShenandoahIUBarrierNode* n) {
- _iu_barriers->remove_if_existing(n);
}
int ShenandoahBarrierSetC2State::load_reference_barriers_count() const {
return _load_reference_barriers->length();
}
ShenandoahBarrierSetC2* ShenandoahBarrierSetC2::bsc2() {
return reinterpret_cast<ShenandoahBarrierSetC2*>(BarrierSet::barrier_set()->barrier_set_c2());
}
ShenandoahBarrierSetC2State::ShenandoahBarrierSetC2State(Arena* comp_arena)
! : _load_reference_barriers(new (comp_arena) GrowableArray<ShenandoahLoadReferenceBarrierNode*>(comp_arena, 8, 0, nullptr)) {
}
int ShenandoahBarrierSetC2State::load_reference_barriers_count() const {
return _load_reference_barriers->length();
}
if (_load_reference_barriers->contains(n)) {
_load_reference_barriers->remove(n);
}
}
- Node* ShenandoahBarrierSetC2::shenandoah_iu_barrier(GraphKit* kit, Node* obj) const {
- if (ShenandoahIUBarrier) {
- return kit->gvn().transform(new ShenandoahIUBarrierNode(obj));
- }
- return obj;
- }
-
#define __ kit->
bool ShenandoahBarrierSetC2::satb_can_remove_pre_barrier(GraphKit* kit, PhaseValues* phase, Node* adr,
BasicType bt, uint adr_idx) const {
intptr_t offset = 0;
bool ShenandoahBarrierSetC2::is_shenandoah_wb_pre_call(Node* call) {
return call->is_CallLeaf() &&
call->as_CallLeaf()->entry_point() == CAST_FROM_FN_PTR(address, ShenandoahRuntime::write_ref_field_pre_entry);
}
bool ShenandoahBarrierSetC2::is_shenandoah_lrb_call(Node* call) {
if (!call->is_CallLeaf()) {
return false;
}
address entry_point = call->as_CallLeaf()->entry_point();
return (entry_point == CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_strong)) ||
(entry_point == CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_strong_narrow)) ||
(entry_point == CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_weak)) ||
(entry_point == CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_weak_narrow)) ||
! (entry_point == CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_phantom));
}
bool ShenandoahBarrierSetC2::is_shenandoah_marking_if(PhaseValues* phase, Node* n) {
if (n->Opcode() != Op_If) {
return false;
bool ShenandoahBarrierSetC2::is_shenandoah_wb_pre_call(Node* call) {
return call->is_CallLeaf() &&
call->as_CallLeaf()->entry_point() == CAST_FROM_FN_PTR(address, ShenandoahRuntime::write_ref_field_pre_entry);
}
+ bool ShenandoahBarrierSetC2::is_shenandoah_clone_call(Node* call) {
+ return call->is_CallLeaf() &&
+ call->as_CallLeaf()->entry_point() == CAST_FROM_FN_PTR(address, ShenandoahRuntime::shenandoah_clone_barrier);
+ }
+
bool ShenandoahBarrierSetC2::is_shenandoah_lrb_call(Node* call) {
if (!call->is_CallLeaf()) {
return false;
}
address entry_point = call->as_CallLeaf()->entry_point();
return (entry_point == CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_strong)) ||
(entry_point == CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_strong_narrow)) ||
(entry_point == CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_weak)) ||
(entry_point == CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_weak_narrow)) ||
! (entry_point == CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_phantom)) ||
+ (entry_point == CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_phantom_narrow));
}
bool ShenandoahBarrierSetC2::is_shenandoah_marking_if(PhaseValues* phase, Node* n) {
if (n->Opcode() != Op_If) {
return false;
// Final sync IdealKit and GraphKit.
kit->final_sync(ideal);
}
+ Node* ShenandoahBarrierSetC2::byte_map_base_node(GraphKit* kit) const {
+ BarrierSet* bs = BarrierSet::barrier_set();
+ ShenandoahBarrierSet* ctbs = barrier_set_cast<ShenandoahBarrierSet>(bs);
+ CardTable::CardValue* card_table_base = ctbs->card_table()->byte_map_base();
+ if (card_table_base != nullptr) {
+ return kit->makecon(TypeRawPtr::make((address)card_table_base));
+ } else {
+ return kit->null();
+ }
+ }
+
+ void ShenandoahBarrierSetC2::post_barrier(GraphKit* kit,
+ Node* ctl,
+ Node* oop_store,
+ Node* obj,
+ Node* adr,
+ uint adr_idx,
+ Node* val,
+ BasicType bt,
+ bool use_precise) const {
+ assert(ShenandoahCardBarrier, "Should have been checked by caller");
+
+ // No store check needed if we're storing a null.
+ if (val != nullptr && val->is_Con()) {
+ // must be either an oop or NULL
+ const Type* t = val->bottom_type();
+ if (t == TypePtr::NULL_PTR || t == Type::TOP)
+ return;
+ }
+
+ if (ReduceInitialCardMarks && obj == kit->just_allocated_object(kit->control())) {
+ // We can skip marks on a freshly-allocated object in Eden.
+ // Keep this code in sync with new_deferred_store_barrier() in runtime.cpp.
+ // That routine informs GC to take appropriate compensating steps,
+ // upon a slow-path allocation, so as to make this card-mark
+ // elision safe.
+ return;
+ }
+
+ if (!use_precise) {
+ // All card marks for a (non-array) instance are in one place:
+ adr = obj;
+ }
+ // (Else it's an array (or unknown), and we want more precise card marks.)
+ assert(adr != nullptr, "");
+
+ IdealKit ideal(kit, true);
+
+ // Convert the pointer to an int prior to doing math on it
+ Node* cast = __ CastPX(__ ctrl(), adr);
+
+ // Divide by card size
+ Node* card_offset = __ URShiftX( cast, __ ConI(CardTable::card_shift()) );
+
+ // Combine card table base and card offset
+ Node* card_adr = __ AddP(__ top(), byte_map_base_node(kit), card_offset );
+
+ // Get the alias_index for raw card-mark memory
+ int adr_type = Compile::AliasIdxRaw;
+ Node* zero = __ ConI(0); // Dirty card value
+
+ if (UseCondCardMark) {
+ // The classic GC reference write barrier is typically implemented
+ // as a store into the global card mark table. Unfortunately
+ // unconditional stores can result in false sharing and excessive
+ // coherence traffic as well as false transactional aborts.
+ // UseCondCardMark enables MP "polite" conditional card mark
+ // stores. In theory we could relax the load from ctrl() to
+ // no_ctrl, but that doesn't buy much latitude.
+ Node* card_val = __ load( __ ctrl(), card_adr, TypeInt::BYTE, T_BYTE, adr_type);
+ __ if_then(card_val, BoolTest::ne, zero);
+ }
+
+ // Smash zero into card
+ __ store(__ ctrl(), card_adr, zero, T_BYTE, adr_type, MemNode::unordered);
+
+ if (UseCondCardMark) {
+ __ end_if();
+ }
+
+ // Final sync IdealKit and GraphKit.
+ kit->final_sync(ideal);
+ }
+
#undef __
const TypeFunc* ShenandoahBarrierSetC2::write_ref_field_pre_entry_Type() {
const Type **fields = TypeTuple::fields(2);
fields[TypeFunc::Parms+0] = TypeInstPtr::NOTNULL; // original field value
C2ParseAccess& parse_access = static_cast<C2ParseAccess&>(access);
GraphKit* kit = parse_access.kit();
uint adr_idx = kit->C->get_alias_index(adr_type);
assert(adr_idx != Compile::AliasIdxTop, "use other store_to_memory factory" );
- Node* value = val.node();
- value = shenandoah_iu_barrier(kit, value);
- val.set_node(value);
shenandoah_write_barrier_pre(kit, true /* do_load */, /*kit->control(),*/ access.base(), adr, adr_idx, val.node(),
static_cast<const TypeOopPtr*>(val.type()), nullptr /* pre_val */, access.type());
} else {
assert(access.is_opt_access(), "only for optimization passes");
assert(((decorators & C2_TIGHTLY_COUPLED_ALLOC) != 0 || !ShenandoahSATBBarrier) && (decorators & C2_ARRAY_COPY) != 0, "unexpected caller of this code");
! C2OptAccess& opt_access = static_cast<C2OptAccess&>(access);
- PhaseGVN& gvn = opt_access.gvn();
-
- if (ShenandoahIUBarrier) {
- Node* enqueue = gvn.transform(new ShenandoahIUBarrierNode(val.node()));
- val.set_node(enqueue);
- }
}
- return BarrierSetC2::store_at_resolved(access, val);
}
Node* ShenandoahBarrierSetC2::load_at_resolved(C2Access& access, const Type* val_type) const {
// 1: non-reference load, no additional barrier is needed
if (!access.is_oop()) {
C2ParseAccess& parse_access = static_cast<C2ParseAccess&>(access);
GraphKit* kit = parse_access.kit();
uint adr_idx = kit->C->get_alias_index(adr_type);
assert(adr_idx != Compile::AliasIdxTop, "use other store_to_memory factory" );
shenandoah_write_barrier_pre(kit, true /* do_load */, /*kit->control(),*/ access.base(), adr, adr_idx, val.node(),
static_cast<const TypeOopPtr*>(val.type()), nullptr /* pre_val */, access.type());
+
+ Node* result = BarrierSetC2::store_at_resolved(access, val);
+
+ if (ShenandoahCardBarrier) {
+ const bool anonymous = (decorators & ON_UNKNOWN_OOP_REF) != 0;
+ const bool is_array = (decorators & IS_ARRAY) != 0;
+ const bool use_precise = is_array || anonymous;
+ post_barrier(kit, kit->control(), access.raw_access(), access.base(),
+ adr, adr_idx, val.node(), access.type(), use_precise);
+ }
+ return result;
} else {
assert(access.is_opt_access(), "only for optimization passes");
assert(((decorators & C2_TIGHTLY_COUPLED_ALLOC) != 0 || !ShenandoahSATBBarrier) && (decorators & C2_ARRAY_COPY) != 0, "unexpected caller of this code");
! return BarrierSetC2::store_at_resolved(access, val);
}
}
Node* ShenandoahBarrierSetC2::load_at_resolved(C2Access& access, const Type* val_type) const {
// 1: non-reference load, no additional barrier is needed
if (!access.is_oop()) {
return load;
}
Node* ShenandoahBarrierSetC2::atomic_cmpxchg_val_at_resolved(C2AtomicParseAccess& access, Node* expected_val,
! Node* new_val, const Type* value_type) const {
GraphKit* kit = access.kit();
if (access.is_oop()) {
- new_val = shenandoah_iu_barrier(kit, new_val);
shenandoah_write_barrier_pre(kit, false /* do_load */,
nullptr, nullptr, max_juint, nullptr, nullptr,
expected_val /* pre_val */, T_OBJECT);
MemNode::MemOrd mo = access.mem_node_mo();
return load;
}
Node* ShenandoahBarrierSetC2::atomic_cmpxchg_val_at_resolved(C2AtomicParseAccess& access, Node* expected_val,
! Node* new_val, const Type* value_type) const {
GraphKit* kit = access.kit();
if (access.is_oop()) {
shenandoah_write_barrier_pre(kit, false /* do_load */,
nullptr, nullptr, max_juint, nullptr, nullptr,
expected_val /* pre_val */, T_OBJECT);
MemNode::MemOrd mo = access.mem_node_mo();
if (adr->bottom_type()->is_ptr_to_narrowoop()) {
load_store = kit->gvn().transform(new DecodeNNode(load_store, load_store->get_ptr_type()));
}
#endif
load_store = kit->gvn().transform(new ShenandoahLoadReferenceBarrierNode(nullptr, load_store, access.decorators()));
return load_store;
}
return BarrierSetC2::atomic_cmpxchg_val_at_resolved(access, expected_val, new_val, value_type);
}
Node* ShenandoahBarrierSetC2::atomic_cmpxchg_bool_at_resolved(C2AtomicParseAccess& access, Node* expected_val,
Node* new_val, const Type* value_type) const {
GraphKit* kit = access.kit();
if (access.is_oop()) {
- new_val = shenandoah_iu_barrier(kit, new_val);
shenandoah_write_barrier_pre(kit, false /* do_load */,
nullptr, nullptr, max_juint, nullptr, nullptr,
expected_val /* pre_val */, T_OBJECT);
DecoratorSet decorators = access.decorators();
MemNode::MemOrd mo = access.mem_node_mo();
if (adr->bottom_type()->is_ptr_to_narrowoop()) {
load_store = kit->gvn().transform(new DecodeNNode(load_store, load_store->get_ptr_type()));
}
#endif
load_store = kit->gvn().transform(new ShenandoahLoadReferenceBarrierNode(nullptr, load_store, access.decorators()));
+ if (ShenandoahCardBarrier) {
+ post_barrier(kit, kit->control(), access.raw_access(), access.base(),
+ access.addr().node(), access.alias_idx(), new_val, T_OBJECT, true);
+ }
return load_store;
}
return BarrierSetC2::atomic_cmpxchg_val_at_resolved(access, expected_val, new_val, value_type);
}
Node* ShenandoahBarrierSetC2::atomic_cmpxchg_bool_at_resolved(C2AtomicParseAccess& access, Node* expected_val,
Node* new_val, const Type* value_type) const {
GraphKit* kit = access.kit();
if (access.is_oop()) {
shenandoah_write_barrier_pre(kit, false /* do_load */,
nullptr, nullptr, max_juint, nullptr, nullptr,
expected_val /* pre_val */, T_OBJECT);
DecoratorSet decorators = access.decorators();
MemNode::MemOrd mo = access.mem_node_mo();
}
}
}
access.set_raw_access(load_store);
pin_atomic_op(access);
return load_store;
}
return BarrierSetC2::atomic_cmpxchg_bool_at_resolved(access, expected_val, new_val, value_type);
}
Node* ShenandoahBarrierSetC2::atomic_xchg_at_resolved(C2AtomicParseAccess& access, Node* val, const Type* value_type) const {
GraphKit* kit = access.kit();
- if (access.is_oop()) {
- val = shenandoah_iu_barrier(kit, val);
- }
Node* result = BarrierSetC2::atomic_xchg_at_resolved(access, val, value_type);
if (access.is_oop()) {
result = kit->gvn().transform(new ShenandoahLoadReferenceBarrierNode(nullptr, result, access.decorators()));
shenandoah_write_barrier_pre(kit, false /* do_load */,
nullptr, nullptr, max_juint, nullptr, nullptr,
result /* pre_val */, T_OBJECT);
}
return result;
}
bool ShenandoahBarrierSetC2::is_gc_pre_barrier_node(Node* node) const {
return is_shenandoah_wb_pre_call(node);
}
- // Support for GC barriers emitted during parsing
bool ShenandoahBarrierSetC2::is_gc_barrier_node(Node* node) const {
! if (node->Opcode() == Op_ShenandoahLoadReferenceBarrier || node->Opcode() == Op_ShenandoahIUBarrier) return true;
! if (node->Opcode() != Op_CallLeaf && node->Opcode() != Op_CallLeafNoFP) {
! return false;
! }
- CallLeafNode *call = node->as_CallLeaf();
- if (call->_name == nullptr) {
- return false;
- }
-
- return strcmp(call->_name, "shenandoah_clone_barrier") == 0 ||
- strcmp(call->_name, "shenandoah_cas_obj") == 0 ||
- strcmp(call->_name, "shenandoah_wb_pre") == 0;
}
Node* ShenandoahBarrierSetC2::step_over_gc_barrier(Node* c) const {
if (c == nullptr) {
return c;
}
if (c->Opcode() == Op_ShenandoahLoadReferenceBarrier) {
return c->in(ShenandoahLoadReferenceBarrierNode::ValueIn);
}
- if (c->Opcode() == Op_ShenandoahIUBarrier) {
- c = c->in(1);
- }
return c;
}
bool ShenandoahBarrierSetC2::expand_barriers(Compile* C, PhaseIterGVN& igvn) const {
return !ShenandoahBarrierC2Support::expand(C, igvn);
}
}
}
access.set_raw_access(load_store);
pin_atomic_op(access);
+ if (ShenandoahCardBarrier) {
+ post_barrier(kit, kit->control(), access.raw_access(), access.base(),
+ access.addr().node(), access.alias_idx(), new_val, T_OBJECT, true);
+ }
return load_store;
}
return BarrierSetC2::atomic_cmpxchg_bool_at_resolved(access, expected_val, new_val, value_type);
}
Node* ShenandoahBarrierSetC2::atomic_xchg_at_resolved(C2AtomicParseAccess& access, Node* val, const Type* value_type) const {
GraphKit* kit = access.kit();
Node* result = BarrierSetC2::atomic_xchg_at_resolved(access, val, value_type);
if (access.is_oop()) {
result = kit->gvn().transform(new ShenandoahLoadReferenceBarrierNode(nullptr, result, access.decorators()));
shenandoah_write_barrier_pre(kit, false /* do_load */,
nullptr, nullptr, max_juint, nullptr, nullptr,
result /* pre_val */, T_OBJECT);
+ if (ShenandoahCardBarrier) {
+ post_barrier(kit, kit->control(), access.raw_access(), access.base(),
+ access.addr().node(), access.alias_idx(), val, T_OBJECT, true);
+ }
}
return result;
}
bool ShenandoahBarrierSetC2::is_gc_pre_barrier_node(Node* node) const {
return is_shenandoah_wb_pre_call(node);
}
bool ShenandoahBarrierSetC2::is_gc_barrier_node(Node* node) const {
! return (node->Opcode() == Op_ShenandoahLoadReferenceBarrier) ||
! is_shenandoah_lrb_call(node) ||
! is_shenandoah_wb_pre_call(node) ||
! is_shenandoah_clone_call(node);
}
Node* ShenandoahBarrierSetC2::step_over_gc_barrier(Node* c) const {
if (c == nullptr) {
return c;
}
if (c->Opcode() == Op_ShenandoahLoadReferenceBarrier) {
return c->in(ShenandoahLoadReferenceBarrierNode::ValueIn);
}
return c;
}
bool ShenandoahBarrierSetC2::expand_barriers(Compile* C, PhaseIterGVN& igvn) const {
return !ShenandoahBarrierC2Support::expand(C, igvn);
if (phase == Optimization) {
return false;
}
return !is_clone;
}
- if (phase == Optimization) {
- return !ShenandoahIUBarrier;
- }
return true;
}
bool ShenandoahBarrierSetC2::clone_needs_barrier(Node* src, PhaseGVN& gvn) {
const TypeOopPtr* src_type = gvn.type(src)->is_oopptr();
uint gc_state_idx = Compile::AliasIdxRaw;
const TypePtr* gc_state_adr_type = nullptr; // debug-mode-only argument
debug_only(gc_state_adr_type = phase->C->get_adr_type(gc_state_idx));
Node* gc_state = phase->transform_later(new LoadBNode(ctrl, mem, gc_state_addr, gc_state_adr_type, TypeInt::BYTE, MemNode::unordered));
! int flags = ShenandoahHeap::HAS_FORWARDED;
- if (ShenandoahIUBarrier) {
- flags |= ShenandoahHeap::MARKING;
- }
- Node* stable_and = phase->transform_later(new AndINode(gc_state, phase->igvn().intcon(flags)));
Node* stable_cmp = phase->transform_later(new CmpINode(stable_and, phase->igvn().zerocon(T_INT)));
Node* stable_test = phase->transform_later(new BoolNode(stable_cmp, BoolTest::ne));
IfNode* stable_iff = phase->transform_later(new IfNode(ctrl, stable_test, PROB_UNLIKELY(0.999), COUNT_UNKNOWN))->as_If();
Node* stable_ctrl = phase->transform_later(new IfFalseNode(stable_iff));
uint gc_state_idx = Compile::AliasIdxRaw;
const TypePtr* gc_state_adr_type = nullptr; // debug-mode-only argument
debug_only(gc_state_adr_type = phase->C->get_adr_type(gc_state_idx));
Node* gc_state = phase->transform_later(new LoadBNode(ctrl, mem, gc_state_addr, gc_state_adr_type, TypeInt::BYTE, MemNode::unordered));
! Node* stable_and = phase->transform_later(new AndINode(gc_state, phase->igvn().intcon(ShenandoahHeap::HAS_FORWARDED)));
Node* stable_cmp = phase->transform_later(new CmpINode(stable_and, phase->igvn().zerocon(T_INT)));
Node* stable_test = phase->transform_later(new BoolNode(stable_cmp, BoolTest::ne));
IfNode* stable_iff = phase->transform_later(new IfNode(ctrl, stable_test, PROB_UNLIKELY(0.999), COUNT_UNKNOWN))->as_If();
Node* stable_ctrl = phase->transform_later(new IfFalseNode(stable_iff));
}
// Support for macro expanded GC barriers
void ShenandoahBarrierSetC2::register_potential_barrier_node(Node* node) const {
- if (node->Opcode() == Op_ShenandoahIUBarrier) {
- state()->add_iu_barrier((ShenandoahIUBarrierNode*) node);
- }
if (node->Opcode() == Op_ShenandoahLoadReferenceBarrier) {
state()->add_load_reference_barrier((ShenandoahLoadReferenceBarrierNode*) node);
}
}
void ShenandoahBarrierSetC2::unregister_potential_barrier_node(Node* node) const {
- if (node->Opcode() == Op_ShenandoahIUBarrier) {
- state()->remove_iu_barrier((ShenandoahIUBarrierNode*) node);
- }
if (node->Opcode() == Op_ShenandoahLoadReferenceBarrier) {
state()->remove_load_reference_barrier((ShenandoahLoadReferenceBarrierNode*) node);
}
}
! void ShenandoahBarrierSetC2::eliminate_gc_barrier(PhaseMacroExpand* macro, Node* n) const {
! if (is_shenandoah_wb_pre_call(n)) {
! shenandoah_eliminate_wb_pre(n, ¯o->igvn());
}
}
void ShenandoahBarrierSetC2::shenandoah_eliminate_wb_pre(Node* call, PhaseIterGVN* igvn) const {
assert(UseShenandoahGC && is_shenandoah_wb_pre_call(call), "");
}
// Support for macro expanded GC barriers
void ShenandoahBarrierSetC2::register_potential_barrier_node(Node* node) const {
if (node->Opcode() == Op_ShenandoahLoadReferenceBarrier) {
state()->add_load_reference_barrier((ShenandoahLoadReferenceBarrierNode*) node);
}
}
void ShenandoahBarrierSetC2::unregister_potential_barrier_node(Node* node) const {
if (node->Opcode() == Op_ShenandoahLoadReferenceBarrier) {
state()->remove_load_reference_barrier((ShenandoahLoadReferenceBarrierNode*) node);
}
}
! void ShenandoahBarrierSetC2::eliminate_gc_barrier(PhaseMacroExpand* macro, Node* node) const {
! if (is_shenandoah_wb_pre_call(node)) {
! shenandoah_eliminate_wb_pre(node, ¯o->igvn());
+ }
+ if (ShenandoahCardBarrier && node->Opcode() == Op_CastP2X) {
+ Node* shift = node->unique_out();
+ Node* addp = shift->unique_out();
+ for (DUIterator_Last jmin, j = addp->last_outs(jmin); j >= jmin; --j) {
+ Node* mem = addp->last_out(j);
+ if (UseCondCardMark && mem->is_Load()) {
+ assert(mem->Opcode() == Op_LoadB, "unexpected code shape");
+ // The load is checking if the card has been written so
+ // replace it with zero to fold the test.
+ macro->replace_node(mem, macro->intcon(0));
+ continue;
+ }
+ assert(mem->is_Store(), "store required");
+ macro->replace_node(mem, mem->in(MemNode::Memory));
+ }
}
}
void ShenandoahBarrierSetC2::shenandoah_eliminate_wb_pre(Node* call, PhaseIterGVN* igvn) const {
assert(UseShenandoahGC && is_shenandoah_wb_pre_call(call), "");
for (DUIterator_Fast imax, i = n->fast_outs(imax); i < imax; i++) {
C->record_for_igvn(n->fast_out(i));
}
}
}
! for (int i = state()->iu_barriers_count() - 1; i >= 0; i--) {
- ShenandoahIUBarrierNode* n = state()->iu_barrier(i);
- if (!useful.member(n)) {
- state()->remove_iu_barrier(n);
- }
- }
for (int i = state()->load_reference_barriers_count() - 1; i >= 0; i--) {
ShenandoahLoadReferenceBarrierNode* n = state()->load_reference_barrier(i);
if (!useful.member(n)) {
state()->remove_load_reference_barrier(n);
}
for (DUIterator_Fast imax, i = n->fast_outs(imax); i < imax; i++) {
C->record_for_igvn(n->fast_out(i));
}
}
}
!
for (int i = state()->load_reference_barriers_count() - 1; i >= 0; i--) {
ShenandoahLoadReferenceBarrierNode* n = state()->load_reference_barrier(i);
if (!useful.member(n)) {
state()->remove_load_reference_barrier(n);
}
}
}
}
return false;
}
- case Op_ShenandoahIUBarrier:
- conn_graph->add_local_var_and_edge(n, PointsToNode::NoEscape, n->in(1), delayed_worklist);
- break;
case Op_ShenandoahLoadReferenceBarrier:
conn_graph->add_local_var_and_edge(n, PointsToNode::NoEscape, n->in(ShenandoahLoadReferenceBarrierNode::ValueIn), delayed_worklist);
return true;
default:
// Nothing
case Op_ShenandoahCompareAndSwapP:
case Op_ShenandoahCompareAndSwapN:
case Op_ShenandoahWeakCompareAndSwapP:
case Op_ShenandoahWeakCompareAndSwapN:
return conn_graph->add_final_edges_unsafe_access(n, opcode);
- case Op_ShenandoahIUBarrier:
- conn_graph->add_local_var_and_edge(n, PointsToNode::NoEscape, n->in(1), nullptr);
- return true;
case Op_ShenandoahLoadReferenceBarrier:
conn_graph->add_local_var_and_edge(n, PointsToNode::NoEscape, n->in(ShenandoahLoadReferenceBarrierNode::ValueIn), nullptr);
return true;
default:
// Nothing
< prev index next >