< prev index next > src/hotspot/share/opto/cfgnode.cpp
Print this page
#include "opto/addnode.hpp"
#include "opto/castnode.hpp"
#include "opto/cfgnode.hpp"
#include "opto/connode.hpp"
#include "opto/convertnode.hpp"
+ #include "opto/inlinetypenode.hpp"
#include "opto/loopnode.hpp"
#include "opto/machnode.hpp"
#include "opto/movenode.hpp"
#include "opto/mulnode.hpp"
#include "opto/narrowptrnode.hpp"
if (!cmp->is_Cmp()) {
return false;
}
return true;
}
+
//------------------------------Ideal------------------------------------------
// Return a node which is more "ideal" than the current node. Must preserve
// the CFG, but we can still strip out dead paths.
Node *RegionNode::Ideal(PhaseGVN *phase, bool can_reshape) {
if( !can_reshape && !in(0) ) return nullptr; // Already degraded to a Copy
return false; // No comparison
} else if (cmp1->Opcode() == Op_CmpF || cmp1->Opcode() == Op_CmpD ||
cmp2->Opcode() == Op_CmpF || cmp2->Opcode() == Op_CmpD ||
cmp1->Opcode() == Op_CmpP || cmp1->Opcode() == Op_CmpN ||
cmp2->Opcode() == Op_CmpP || cmp2->Opcode() == Op_CmpN ||
- cmp1->is_SubTypeCheck() || cmp2->is_SubTypeCheck()) {
+ cmp1->is_SubTypeCheck() || cmp2->is_SubTypeCheck() ||
+ cmp1->is_FlatArrayCheck() || cmp2->is_FlatArrayCheck()) {
// Floats and pointers don't exactly obey trichotomy. To be on the safe side, don't transform their tests.
// SubTypeCheck is not commutative
return false;
} else if (cmp1 != cmp2) {
if (cmp1->in(1) == cmp2->in(2) &&
return nullptr;
}
//=============================================================================
- // note that these functions assume that the _adr_type field is flattened
+ // note that these functions assume that the _adr_type field is flat
uint PhiNode::hash() const {
const Type* at = _adr_type;
return TypeNode::hash() + (at ? at->hash() : 0);
}
bool PhiNode::cmp( const Node &n ) const {
//----------------------------make---------------------------------------------
// create a new phi with edges matching r and set (initially) to x
PhiNode* PhiNode::make(Node* r, Node* x, const Type *t, const TypePtr* at) {
uint preds = r->req(); // Number of predecessor paths
- assert(t != Type::MEMORY || at == flatten_phi_adr_type(at), "flatten at");
+ assert(t != Type::MEMORY || at == flatten_phi_adr_type(at) || (flatten_phi_adr_type(at) == TypeAryPtr::INLINES && Compile::current()->flat_accesses_share_alias()), "flatten at");
PhiNode* p = new PhiNode(r, t, at);
for (uint j = 1; j < preds; j++) {
// Fill in all inputs, except those which the region does not yet have
if (r->in(j) != nullptr)
p->init_req(j, x);
void PhiNode::verify_adr_type(bool recursive) const {
if (VMError::is_error_reported()) return; // muzzle asserts when debugging an error
if (Node::in_dump()) return; // muzzle asserts when printing
assert((_type == Type::MEMORY) == (_adr_type != nullptr), "adr_type for memory phis only");
+ // Flat array elements shouldn't get their own memory slice until flat_accesses_share_alias is cleared.
+ // It could be the graph has no loads/stores and flat_accesses_share_alias is never cleared. EA could still
+ // create per-element Phis but that wouldn't be a problem as there are no memory accesses for that array.
+ assert(_adr_type == nullptr || _adr_type->isa_aryptr() == nullptr ||
+ _adr_type->is_aryptr()->is_known_instance() ||
+ !_adr_type->is_aryptr()->is_flat() ||
+ !Compile::current()->flat_accesses_share_alias() ||
+ _adr_type == TypeAryPtr::INLINES, "flat array element shouldn't get its own slice yet");
if (!VerifyAliases) return; // verify thoroughly only if requested
assert(_adr_type == flatten_phi_adr_type(_adr_type),
"Phi::adr_type must be pre-normalized");
}
}
}
return false;
}
+
//----------------------------check_cmove_id-----------------------------------
// Check for CMove'ing a constant after comparing against the constant.
// Happens all the time now, since if we compare equality vs a constant in
// the parser, we "know" the variable is constant on one path and we force
// it. Thus code like "if( x==0 ) {/*EMPTY*/}" ends up inserting a
// does all this and more, by reducing such tributaries to 'this'.)
Node* uin = unique_input(phase, false);
if (uin != nullptr) {
return uin;
}
+ uin = unique_constant_input_recursive(phase);
+ if (uin != nullptr) {
+ return uin;
+ }
int true_path = is_diamond_phi();
// Delay CMove'ing identity if Ideal has not had the chance to handle unsafe cases, yet.
if (true_path != 0 && !(phase->is_IterGVN() && wait_for_region_igvn(phase))) {
Node* id = is_cmove_id(phase, true_path);
// Nothing.
return nullptr;
}
+ // Find the unique input, try to look recursively through input Phis
+ Node* PhiNode::unique_constant_input_recursive(PhaseGVN* phase) {
+ if (!phase->is_IterGVN()) {
+ return nullptr;
+ }
+
+ ResourceMark rm;
+ Node* unique = nullptr;
+ Unique_Node_List visited;
+ visited.push(this);
+
+ for (uint visited_idx = 0; visited_idx < visited.size(); visited_idx++) {
+ Node* current = visited.at(visited_idx);
+ for (uint i = 1; i < current->req(); i++) {
+ Node* phi_in = current->in(i);
+ if (phi_in == nullptr) {
+ continue;
+ }
+
+ if (phi_in->is_Phi()) {
+ visited.push(phi_in);
+ } else {
+ if (unique == nullptr) {
+ if (!phi_in->is_Con()) {
+ return nullptr;
+ }
+ unique = phi_in;
+ } else if (unique != phi_in) {
+ return nullptr;
+ }
+ }
+ }
+ }
+ return unique;
+ }
+
//------------------------------is_x2logic-------------------------------------
// Check for simple convert-to-boolean pattern
// If:(C Bool) Region:(IfF IfT) Phi:(Region 0 1)
// Convert Phi to an ConvIB.
static Node *is_x2logic( PhaseGVN *phase, PhiNode *phi, int true_path ) {
worklist.push(this);
}
return delay;
}
+
// If the Phi's Region is in an irreducible loop, and the Region
// has had an input removed, but not yet transformed, it could be
// that the Region (and this Phi) are not reachable from Root.
// If we allow the Phi to collapse before the Region, this may lead
// to dead-loop data. Wait for the Region to check for reachability,
// MergeMem
//
// This split breaks the circularity and consequently does not lead to
// non-termination.
uint merge_width = 0;
+ // TODO revisit this with JDK-8247216
+ bool mergemem_only = true;
bool split_always_terminates = false; // Is splitting guaranteed to terminate?
for( uint i=1; i<req(); ++i ) {// For all paths in
Node *ii = in(i);
// TOP inputs should not be counted as safe inputs because if the
// Phi references itself through all other inputs then splitting the
MergeMemNode* n = ii->as_MergeMem();
merge_width = MAX2(merge_width, n->req());
if (n->base_memory() == this) {
split_always_terminates = true;
}
+ } else {
+ mergemem_only = false;
}
}
// There are cases with circular dependencies between bottom Phis
// and MergeMems. Below is a minimal example.
//
// Here, we cannot break the circularity through a self-loop as there
// are two Phis involved. Repeatedly splitting the Phis through the
// MergeMem leads to non-termination. We check for non-termination below.
// Only check for non-termination if necessary.
- if (!split_always_terminates && adr_type() == TypePtr::BOTTOM &&
+ if (!mergemem_only && !split_always_terminates && adr_type() == TypePtr::BOTTOM &&
merge_width > Compile::AliasIdxRaw) {
split_always_terminates = is_split_through_mergemem_terminating();
}
if (merge_width > Compile::AliasIdxRaw) {
set_req_X(i, new_mem, phase->is_IterGVN());
progress = this;
}
}
}
- } else if (split_always_terminates) {
+ } else if (mergemem_only || split_always_terminates) {
// If all inputs reference this phi (directly or through data nodes) -
// it is a dead loop.
bool saw_safe_input = false;
for (uint j = 1; j < req(); ++j) {
Node* n = in(j);
MergeMemNode* result = MergeMemNode::make(new_base);
for (uint i = 1; i < req(); ++i) {
Node *ii = in(i);
if (ii->is_MergeMem()) {
MergeMemNode* n = ii->as_MergeMem();
+ if (igvn) {
+ // TODO revisit this with JDK-8247216
+ // Put 'n' on the worklist because it might be modified by MergeMemStream::iteration_setup
+ igvn->_worklist.push(n);
+ }
for (MergeMemStream mms(result, n); mms.next_non_empty2(); ) {
// If we have not seen this slice yet, make a phi for it.
bool made_new_phi = false;
if (mms.is_empty()) {
Node* new_phi = new_base->slice_memory(mms.adr_type(phase->C));
// MemNode::optimize_memory_chain above may kill us!
if (outcnt() == 0) {
return top;
}
if (ii != new_in ) {
- set_req_X(i, new_in, phase);
+ set_req_X(i, new_in, phase->is_IterGVN());
progress = this;
}
}
}
}
}
}
#endif
+ Node* inline_type = try_push_inline_types_down(phase, can_reshape);
+ if (inline_type != this) {
+ return inline_type;
+ }
+
// Try to convert a Phi with two duplicated convert nodes into a phi of the pre-conversion type and the convert node
// proceeding the phi, to de-duplicate the convert node and compact the IR.
if (can_reshape && progress == nullptr) {
ConvertNode* convert = in(1)->isa_Convert();
if (convert != nullptr) {
}
return progress; // Return any progress
}
+ // If an InlineType node references itself through a Phi (oop input):
+ //
+ // /------ |
+ // InlineType |
+ // \ / |
+ // Phi |
+ // ^____________|
+ //
+ // and is pushed down through the Phi, the result is a new InlineType node that can be pushed down through the Phi
+ // etc.
+ //
+ // To solve that problem, the code below finds InlineType nodes that are reachable from another InlineType node by
+ // following the inline type node's oop inputs through Phis and casts such as, for instance:
+ // (InlineType (Cast (Phi (InlineType oop
+ // and replaces it with:
+ // (InlineType (Cast (Phi oop
+ // which requires cloning every Phi and cast nodes in the subgraph between the root InlineType and the leaf
+ // InlineType node in this example.
+ class PushInlineTypeDown {
+ private:
+ // Find the inlineType nodes that can be reached from this Phi without going through another InlineType (those that
+ // must not reference another inline type node from their oop input through Phis and cast nodes)
+ void collect_nodes_from_phi() {
+ _nodes_from_phi.push(_root_phi);
+ for (uint i = 0; i < _nodes_from_phi.size(); ++i) {
+ Node* n = _nodes_from_phi.at(i);
+ if (n->is_Phi()) {
+ for (uint j = 1; j < n->req(); ++j) {
+ Node* in = n->in(j);
+ if (in != nullptr) {
+ _nodes_from_phi.push(in);
+ }
+ }
+ } else if (n->is_ConstraintCast()) {
+ Node* in = n->in(1);
+ if (in != nullptr) {
+ _nodes_from_phi.push(in);
+ }
+ }
+ }
+ }
+
+ void collect_nodes_from_inline_types() {
+ // Only keep InlineType nodes
+ for (int i = _nodes_from_phi.size() - 1; i >= 0; i--) {
+ Node* n = _nodes_from_phi.at(i);
+ if (!n->is_InlineType()) {
+ _nodes_from_phi.remove(i);
+ }
+ }
+ DEBUG_ONLY(_init_nodes = _nodes_from_phi.size());
+ // Find the InlineType nodes reachable from the current set of inline type nodes.
+ for (uint i = 0; i < _nodes_from_phi.size(); ++i) {
+ Node* n = _nodes_from_phi.at(i);
+ if (n->is_Phi()) {
+ for (uint j = 1; j < n->req(); ++j) {
+ Node* in = n->in(j);
+ if (in != nullptr) {
+ _nodes_from_phi.push(in);
+ }
+ }
+ } else if (n->is_ConstraintCast()) {
+ Node* in = n->in(1);
+ if (in != nullptr) {
+ _nodes_from_phi.push(in);
+ }
+ } else if (n->is_InlineType()) {
+ Node* buf = n->as_InlineType()->get_oop();
+ if (buf != nullptr) {
+ _nodes_from_phi.push(buf);
+ _subgraph_to_clone.push(n);
+ }
+ }
+ }
+ }
+
+ void collect_nodes_to_clone() {
+ collect_nodes_from_inline_types();
+
+ // Find the subgraph that must be cloned by following uses from inline types.
+ for (uint i = 0; i < _subgraph_to_clone.size(); ++i) {
+ Node* n = _subgraph_to_clone.at(i);
+ for (DUIterator_Fast imax, i = n->fast_outs(imax); i < imax; i++) {
+ Node* u = n->fast_out(i);
+ if (_nodes_from_phi.member(u)) {
+ _subgraph_to_clone.push(u);
+ }
+ }
+ }
+ }
+
+ void clone_nodes() {
+ for (uint i = 0; i < _subgraph_to_clone.size(); ++i) {
+ Node* n = _subgraph_to_clone.at(i);
+ assert(_clones[n->_idx] == nullptr, "shouldn't be cloned yet");
+ if (n->is_InlineType()) {
+ _clones.map(n->_idx, n->as_InlineType()->get_oop());
+ } else {
+ Node* clone = n->clone();
+ _phase->is_IterGVN()->register_new_node_with_optimizer(clone);
+ _clones.map(n->_idx, clone);
+ }
+ }
+ }
+
+ Node* get_clone(Node* n) const {
+ Node* clone = nullptr;
+ while (true) {
+ Node* m = _clones[n->_idx];
+ if (m == nullptr) {
+ return clone;
+ }
+ clone = m;
+ n = clone;
+ }
+ }
+
+ void update_clone_node_edges() {
+ for (uint i = 0; i < _subgraph_to_clone.size(); ++i) {
+ Node* n = _subgraph_to_clone.at(i);
+ Node* n_clone = get_clone(n);
+ assert(n_clone != nullptr, "must be cloned");
+ if (n->is_Phi()) {
+ for (uint j = 1; j < n->req(); ++j) {
+ Node* in = n->in(j);
+ if (in != nullptr) {
+ Node* in_clone = get_clone(in);
+ if (in_clone != nullptr) {
+ n_clone->set_req(j, in_clone);
+ }
+ }
+ }
+ } else if (n->is_ConstraintCast()) {
+ Node* in = n->in(1);
+ Node* in_clone = get_clone(in);
+ assert(in_clone != nullptr, "must be cloned");
+ n_clone->set_req(1, in_clone);
+ } else if (n->is_InlineType()) {
+ Node* in = n->as_InlineType()->get_oop();
+ Node* in_clone = get_clone(in);
+ if (in_clone != nullptr) {
+ _phase->is_IterGVN()->rehash_node_delayed(n);
+ n->as_InlineType()->set_oop(*_phase, in_clone);
+ }
+ }
+ }
+ }
+
+ void clone_subgraph() {
+ clone_nodes();
+ update_clone_node_edges();
+ }
+
+ #ifdef ASSERT
+ void verify_clone() {
+ uint vts_to_skip = 0;
+ uint before_phis = 0;
+ uint before_casts = 0;
+ for (uint i = 0; i < _nodes_from_phi.size(); ++i) {
+ Node *n = _nodes_from_phi.at(i);
+ if (n->is_Phi()) {
+ before_phis++;
+ } else if (n->is_ConstraintCast()) {
+ before_casts++;
+ } else if (n->is_InlineType()) {
+ Node* buf = n->as_InlineType()->get_oop();
+ if (buf != nullptr && i >= _init_nodes) {
+ vts_to_skip++;
+ }
+ }
+ }
+
+ Unique_Node_List after;
+ after.push(_root_phi);
+ for (uint i = 0; i < after.size(); ++i) {
+ Node* n = after.at(i);
+ if (n->is_Phi()) {
+ for (uint j = 1; j < n->req(); ++j) {
+ Node* in = n->in(j);
+ if (in != nullptr) {
+ after.push(in);
+ }
+ }
+ } else if (n->is_ConstraintCast()) {
+ Node* in = n->in(1);
+ if (in != nullptr) {
+ after.push(in);
+ }
+ }
+ }
+ for (int i = after.size() - 1; i >= 0; i--) {
+ Node* n = after.at(i);
+ if (!n->is_InlineType()) {
+ after.remove(i);
+ }
+ }
+ uint after_phis = 0;
+ uint after_casts = 0;
+ uint init_nodes = after.size();
+ for (uint i = 0; i < after.size(); ++i) {
+ Node* n = after.at(i);
+ if (n->is_Phi()) {
+ after_phis++;
+ for (uint j = 1; j < n->req(); ++j) {
+ Node *in = n->in(j);
+ if (in != nullptr) {
+ after.push(in);
+ }
+ }
+ } else if (n->is_ConstraintCast()) {
+ after_casts++;
+ Node* in = n->in(1);
+ if (in != nullptr) {
+ after.push(in);
+ }
+ } else if (n->is_InlineType()) {
+ assert(i < init_nodes, "");
+ Node* buf = n->as_InlineType()->get_oop();
+ if (buf != nullptr) {
+ after.push(buf);
+ }
+ }
+ }
+ assert(after.size() + vts_to_skip == _nodes_from_phi.size(), "");
+ assert(before_casts == after_casts, "no cast should have been dropped");
+ assert(before_phis == after_phis, "no phi should");
+ }
+ #endif
+
+ Node* do_transform(PhiNode* phi) {
+ assert(_inline_klass != nullptr, "must be");
+ InlineTypeNode *vt = InlineTypeNode::make_null(*_phase, _inline_klass, /* transform = */ false)->clone_with_phis(
+ _phase, phi->in(0), nullptr, !phi->type()->maybe_null(), true);
+ if (_can_reshape) {
+ // Replace phi right away to be able to use the inline
+ // type node when reaching the phi again through data loops.
+ PhaseIterGVN* igvn = _phase->is_IterGVN();
+ for (DUIterator_Fast imax, i = phi->fast_outs(imax); i < imax; i++) {
+ Node* u = phi->fast_out(i);
+ igvn->rehash_node_delayed(u);
+ imax -= u->replace_edge(phi, vt);
+ --i;
+ }
+ igvn->rehash_node_delayed(phi);
+ assert(phi->outcnt() == 0, "should be dead now");
+ }
+ ResourceMark rm;
+ Node_List casts;
+ for (uint i = 1; i < phi->req(); ++i) {
+ Node* n = phi->in(i);
+ if (n == nullptr) {
+ continue;
+ }
+ while (n->is_ConstraintCast()) {
+ casts.push(n);
+ n = n->in(1);
+ }
+ if (_phase->type(n)->is_zero_type()) {
+ n = InlineTypeNode::make_null(*_phase, _inline_klass);
+ } else if (n->is_Phi()) {
+ assert(_can_reshape, "can only handle phis during IGVN");
+ n = _phase->transform(do_transform(n->as_Phi()));
+ }
+ while (casts.size() != 0) {
+ // Push the cast(s) through the InlineTypeNode
+ Node *cast = casts.pop()->clone();
+ cast->set_req_X(1, n->as_InlineType()->get_oop(), _phase);
+ n = n->clone();
+ n->as_InlineType()->set_oop(*_phase, _phase->transform(cast));
+ n = _phase->transform(n);
+ if (n->is_top()) {
+ break;
+ }
+ }
+ bool transform = !_can_reshape && (i == (phi->req() - 1)); // Transform phis on last merge
+ assert(n->is_top() || n->is_InlineType(), "Only InlineType or top at this point.");
+ if (n->is_InlineType()) {
+ vt->merge_with(_phase, n->as_InlineType(), i, transform);
+ } // else nothing to do: phis above vt created by clone_with_phis are initialized to top already.
+ }
+ return vt;
+
+ }
+
+ PhiNode* _root_phi;
+ PhaseGVN* _phase;
+ bool _can_reshape;
+ ciInlineKlass* _inline_klass;
+ Unique_Node_List _nodes_from_phi;
+ Unique_Node_List _subgraph_to_clone;
+ Node_List _clones;
+ DEBUG_ONLY(uint _init_nodes);
+
+ public:
+
+ PushInlineTypeDown(PhiNode *root_phi, PhaseGVN *phase, bool can_reshape)
+ : _root_phi(root_phi), _phase(phase), _can_reshape(can_reshape), _inline_klass(nullptr) {
+ collect_nodes_from_phi();
+ }
+
+ bool can_do_it() {
+ if (_root_phi->req() <= 2) {
+ // Dead phi.
+ return false;
+ }
+
+ for (uint next = 0; next < _nodes_from_phi.size(); next++) {
+ Node* n = _nodes_from_phi.at(next);
+ if (n->is_Phi()) {
+ assert(n->bottom_type()->isa_ptr(), "broken graph");
+ if (n != _root_phi && !_can_reshape) {
+ return false;
+ }
+ continue;
+ }
+ if (n->is_ConstraintCast()) {
+ if (n->in(0) != nullptr && n->in(0)->is_top()) {
+ // Will die, don't optimize
+ return false;
+ }
+ continue;
+ }
+ const Type* type = _phase->type(n);
+ if (n->is_InlineType()) {
+ if (_inline_klass == nullptr) {
+ _inline_klass = type->inline_klass();
+ } else if (_inline_klass != type->inline_klass()) {
+ return false;
+ }
+ continue;
+ }
+ if (!type->is_zero_type()) {
+ return false;
+ }
+ }
+
+ if (_inline_klass == nullptr) {
+ return false;
+ }
+
+ // Check if cast nodes can be pushed through
+ const Type* t = Type::get_const_type(_inline_klass);
+ for (uint next = 0; next < _nodes_from_phi.size(); next++) {
+ Node* n = _nodes_from_phi.at(next);
+ if (n->is_ConstraintCast()) {
+ if (t->filter(n->bottom_type()) == Type::TOP) {
+ return false;
+ }
+ }
+ }
+ return true;
+ }
+
+
+ Node* do_it() {
+ if (_can_reshape) {
+ Node_List clones;
+ collect_nodes_to_clone();
+ clone_subgraph();
+ DEBUG_ONLY(verify_clone());
+ }
+ return do_transform(_root_phi);
+ }
+ };
+
+
+ // Check recursively if inputs are either an inline type, constant null
+ // or another Phi (including self references through data loops). If so,
+ // push the inline types down through the phis to enable folding of loads.
+ Node* PhiNode::try_push_inline_types_down(PhaseGVN* phase, const bool can_reshape) {
+ if (!can_be_inline_type()) {
+ return this;
+ }
+
+ ResourceMark rm;
+ PushInlineTypeDown push_inline_type_down(this, phase, can_reshape);
+ if (push_inline_type_down.can_do_it()) {
+ return push_inline_type_down.do_it();
+ }
+ return this;
+ }
+
+ #ifdef ASSERT
+ bool PhiNode::can_push_inline_types_down(PhaseGVN* phase) {
+ if (!can_be_inline_type()) {
+ return false;
+ }
+
+ ResourceMark rm;
+ PushInlineTypeDown push_inline_type_down(this, phase, false);
+ return push_inline_type_down.can_do_it();
+ }
+ #endif // ASSERT
+
static int compare_types(const Type* const& e1, const Type* const& e2) {
return (intptr_t)e1 - (intptr_t)e2;
}
// Collect types at casts that are going to be eliminated at that Phi and store them in a TypeTuple.
return phase->C->top(); // dead code
}
// We only come from CatchProj, unless the CatchProj goes away.
// If the CatchProj is optimized away, then we just carry the
// exception oop through.
+
+ // CheckCastPPNode::Ideal() for inline types reuses the exception
+ // paths of a call to perform an allocation: we can see a Phi here.
+ if (in(1)->is_Phi()) {
+ return this;
+ }
CallNode *call = in(1)->in(0)->as_Call();
return (in(0)->is_CatchProj() && in(0)->in(0)->is_Catch() &&
in(0)->in(0)->in(1) == in(1)) ? this : call->in(TypeFunc::Parms);
}
< prev index next >