< prev index next > src/hotspot/share/opto/cfgnode.cpp
Print this page
/*
- * Copyright (c) 1997, 2022, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 1997, 2023, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
* under the terms of the GNU General Public License version 2 only, as
* published by the Free Software Foundation.
#include "opto/addnode.hpp"
#include "opto/castnode.hpp"
#include "opto/cfgnode.hpp"
#include "opto/connode.hpp"
#include "opto/convertnode.hpp"
+ #include "opto/inlinetypenode.hpp"
#include "opto/loopnode.hpp"
#include "opto/machnode.hpp"
#include "opto/movenode.hpp"
#include "opto/narrowptrnode.hpp"
#include "opto/mulnode.hpp"
// No exit found for any loop -> all are infinite
return true;
}
#endif //ASSERT
- bool RegionNode::try_clean_mem_phi(PhaseGVN *phase) {
+ Node* PhiNode::try_clean_mem_phi(PhaseGVN *phase) {
// Incremental inlining + PhaseStringOpts sometimes produce:
//
// cmpP with 1 top input
// |
// If
// the Region stays in the graph. The top input from the cmpP is
// propagated forward and a subgraph that is useful goes away. The
// code below replaces the Phi with the MergeMem so that the Region
// is simplified.
- PhiNode* phi = has_unique_phi();
- if (phi && phi->type() == Type::MEMORY && req() == 3 && phi->is_diamond_phi(true)) {
+ if (type() == Type::MEMORY && is_diamond_phi(true)) {
MergeMemNode* m = NULL;
- assert(phi->req() == 3, "same as region");
+ assert(req() == 3, "same as region");
+ Node* r = in(0);
for (uint i = 1; i < 3; ++i) {
- Node *mem = phi->in(i);
- if (mem && mem->is_MergeMem() && in(i)->outcnt() == 1) {
+ Node *mem = in(i);
+ if (mem && mem->is_MergeMem() && r->in(i)->outcnt() == 1) {
// Nothing is control-dependent on path #i except the region itself.
m = mem->as_MergeMem();
uint j = 3 - i;
- Node* other = phi->in(j);
+ Node* other = in(j);
if (other && other == m->base_memory()) {
// m is a successor memory to other, and is not pinned inside the diamond, so push it out.
// This will allow the diamond to collapse completely.
- phase->is_IterGVN()->replace_node(phi, m);
- return true;
+ return m;
}
}
}
}
- return false;
+ return NULL;
}
//------------------------------Ideal------------------------------------------
// Return a node which is more "ideal" than the current node. Must preserve
// the CFG, but we can still strip out dead paths.
// Check for RegionNode with no Phi users and both inputs come from either
// arm of the same IF. If found, then the control-flow split is useless.
bool has_phis = false;
if (can_reshape) { // Need DU info to check for Phi users
has_phis = (has_phi() != NULL); // Cache result
- if (has_phis && try_clean_mem_phi(phase)) {
- has_phis = false;
+ if (has_phis) {
+ PhiNode* phi = has_unique_phi();
+ if (phi != NULL) {
+ Node* m = phi->try_clean_mem_phi(phase);
+ if (m != NULL) {
+ phase->is_IterGVN()->replace_node(phi, m);
+ has_phis = false;
+ }
+ }
}
if (!has_phis) { // No Phi users? Nothing merging?
for (uint i = 1; i < req()-1; i++) {
Node *if1 = in(i);
return false; // No comparison
} else if (cmp1->Opcode() == Op_CmpF || cmp1->Opcode() == Op_CmpD ||
cmp2->Opcode() == Op_CmpF || cmp2->Opcode() == Op_CmpD ||
cmp1->Opcode() == Op_CmpP || cmp1->Opcode() == Op_CmpN ||
cmp2->Opcode() == Op_CmpP || cmp2->Opcode() == Op_CmpN ||
- cmp1->is_SubTypeCheck() || cmp2->is_SubTypeCheck()) {
+ cmp1->is_SubTypeCheck() || cmp2->is_SubTypeCheck() ||
+ cmp1->is_FlatArrayCheck() || cmp2->is_FlatArrayCheck()) {
// Floats and pointers don't exactly obey trichotomy. To be on the safe side, don't transform their tests.
// SubTypeCheck is not commutative
return false;
} else if (cmp1 != cmp2) {
if (cmp1->in(1) == cmp2->in(2) &&
//----------------------------make---------------------------------------------
// create a new phi with edges matching r and set (initially) to x
PhiNode* PhiNode::make(Node* r, Node* x, const Type *t, const TypePtr* at) {
uint preds = r->req(); // Number of predecessor paths
- assert(t != Type::MEMORY || at == flatten_phi_adr_type(at), "flatten at");
+ assert(t != Type::MEMORY || at == flatten_phi_adr_type(at) || (flatten_phi_adr_type(at) == TypeAryPtr::INLINES && Compile::current()->flattened_accesses_share_alias()), "flatten at");
PhiNode* p = new PhiNode(r, t, at);
for (uint j = 1; j < preds; j++) {
// Fill in all inputs, except those which the region does not yet have
if (r->in(j) != NULL)
p->init_req(j, x);
void PhiNode::verify_adr_type(bool recursive) const {
if (VMError::is_error_reported()) return; // muzzle asserts when debugging an error
if (Node::in_dump()) return; // muzzle asserts when printing
assert((_type == Type::MEMORY) == (_adr_type != NULL), "adr_type for memory phis only");
+ // Flat array element shouldn't get their own memory slice until flattened_accesses_share_alias is cleared.
+ // It could be the graph has no loads/stores and flattened_accesses_share_alias is never cleared. EA could still
+ // creates per element Phis but that wouldn't be a problem as there are no memory accesses for that array.
+ assert(_adr_type == NULL || _adr_type->isa_aryptr() == NULL ||
+ _adr_type->is_aryptr()->is_known_instance() ||
+ !_adr_type->is_aryptr()->is_flat() ||
+ !Compile::current()->flattened_accesses_share_alias() ||
+ _adr_type == TypeAryPtr::INLINES, "flat array element shouldn't get its own slice yet");
if (!VerifyAliases) return; // verify thoroughly only if requested
assert(_adr_type == flatten_phi_adr_type(_adr_type),
"Phi::adr_type must be pre-normalized");
if (id != NULL) {
return id;
}
}
+ if (phase->is_IterGVN()) {
+ Node* m = try_clean_mem_phi(phase);
+ if (m != NULL) {
+ return m;
+ }
+ }
+
+
// Looking for phis with identical inputs. If we find one that has
// type TypePtr::BOTTOM, replace the current phi with the bottom phi.
if (phase->is_IterGVN() && type() == Type::MEMORY && adr_type() !=
TypePtr::BOTTOM && !adr_type()->is_known_instance()) {
uint phi_len = req();
worklist.push(this);
}
return delay;
}
+ // Push inline type input nodes (and null) down through the phi recursively (can handle data loops).
+ InlineTypeNode* PhiNode::push_inline_types_through(PhaseGVN* phase, bool can_reshape, ciInlineKlass* vk, bool is_init) {
+ InlineTypeNode* vt = InlineTypeNode::make_null(*phase, vk)->clone_with_phis(phase, in(0), is_init);
+ if (can_reshape) {
+ // Replace phi right away to be able to use the inline
+ // type node when reaching the phi again through data loops.
+ PhaseIterGVN* igvn = phase->is_IterGVN();
+ for (DUIterator_Fast imax, i = fast_outs(imax); i < imax; i++) {
+ Node* u = fast_out(i);
+ igvn->rehash_node_delayed(u);
+ imax -= u->replace_edge(this, vt);
+ --i;
+ }
+ igvn->rehash_node_delayed(this);
+ assert(outcnt() == 0, "should be dead now");
+ }
+ ResourceMark rm;
+ Node_List casts;
+ for (uint i = 1; i < req(); ++i) {
+ Node* n = in(i);
+ while (n->is_ConstraintCast()) {
+ casts.push(n);
+ n = n->in(1);
+ }
+ if (phase->type(n)->is_zero_type()) {
+ n = InlineTypeNode::make_null(*phase, vk);
+ } else if (n->is_Phi()) {
+ assert(can_reshape, "can only handle phis during IGVN");
+ n = phase->transform(n->as_Phi()->push_inline_types_through(phase, can_reshape, vk, is_init));
+ }
+ while (casts.size() != 0) {
+ // Push the cast(s) through the InlineTypeNode
+ Node* cast = casts.pop()->clone();
+ cast->set_req_X(1, n->as_InlineType()->get_oop(), phase);
+ n = n->clone();
+ n->as_InlineType()->set_oop(phase->transform(cast));
+ n = phase->transform(n);
+ }
+ bool transform = !can_reshape && (i == (req()-1)); // Transform phis on last merge
+ vt->merge_with(phase, n->as_InlineType(), i, transform);
+ }
+ return vt;
+ }
+
//------------------------------Ideal------------------------------------------
// Return a node which is more "ideal" than the current node. Must preserve
// the CFG, but we can still strip out dead paths.
Node *PhiNode::Ideal(PhaseGVN *phase, bool can_reshape) {
Node *r = in(0); // RegionNode
// (MergeMemNode is not dead_loop_safe - need to check for dead loop.)
if (progress == NULL && can_reshape && type() == Type::MEMORY) {
// see if this phi should be sliced
uint merge_width = 0;
bool saw_self = false;
+ // TODO revisit this with JDK-8247216
+ bool mergemem_only = true;
for( uint i=1; i<req(); ++i ) {// For all paths in
Node *ii = in(i);
// TOP inputs should not be counted as safe inputs because if the
// Phi references itself through all other inputs then splitting the
// Phi through memory merges would create dead loop at later stage.
}
if (ii->is_MergeMem()) {
MergeMemNode* n = ii->as_MergeMem();
merge_width = MAX2(merge_width, n->req());
saw_self = saw_self || (n->base_memory() == this);
+ } else {
+ mergemem_only = false;
}
}
// This restriction is temporarily necessary to ensure termination:
- if (!saw_self && adr_type() == TypePtr::BOTTOM) merge_width = 0;
+ if (!mergemem_only && !saw_self && adr_type() == TypePtr::BOTTOM) merge_width = 0;
if (merge_width > Compile::AliasIdxRaw) {
// found at least one non-empty MergeMem
const TypePtr* at = adr_type();
if (at != TypePtr::BOTTOM) {
MergeMemNode* result = MergeMemNode::make(new_base);
for (uint i = 1; i < req(); ++i) {
Node *ii = in(i);
if (ii->is_MergeMem()) {
MergeMemNode* n = ii->as_MergeMem();
+ if (igvn) {
+ // TODO revisit this with JDK-8247216
+ // Put 'n' on the worklist because it might be modified by MergeMemStream::iteration_setup
+ igvn->_worklist.push(n);
+ }
for (MergeMemStream mms(result, n); mms.next_non_empty2(); ) {
// If we have not seen this slice yet, make a phi for it.
bool made_new_phi = false;
if (mms.is_empty()) {
Node* new_phi = new_base->slice_memory(mms.adr_type(phase->C));
}
}
}
#endif
+ // Check recursively if inputs are either an inline type, constant null
+ // or another Phi (including self references through data loops). If so,
+ // push the inline types down through the phis to enable folding of loads.
+ if (EnableValhalla && _type->isa_ptr() && req() > 2) {
+ ResourceMark rm;
+ Unique_Node_List worklist;
+ worklist.push(this);
+ bool can_optimize = true;
+ ciInlineKlass* vk = NULL;
+ // true if all IsInit inputs of all InlineType* nodes are true
+ bool is_init = true;
+ Node_List casts;
+
+ // TODO 8302217 We need to prevent endless pushing through
+ bool only_phi = (outcnt() != 0);
+ for (DUIterator_Fast imax, i = fast_outs(imax); i < imax; i++) {
+ Node* n = fast_out(i);
+ if (n->is_InlineType() && n->in(1) == this) {
+ can_optimize = false;
+ break;
+ }
+ if (!n->is_Phi()) {
+ only_phi = false;
+ }
+ }
+ if (only_phi) {
+ can_optimize = false;
+ }
+ for (uint next = 0; next < worklist.size() && can_optimize; next++) {
+ Node* phi = worklist.at(next);
+ for (uint i = 1; i < phi->req() && can_optimize; i++) {
+ Node* n = phi->in(i);
+ if (n == NULL) {
+ can_optimize = false;
+ break;
+ }
+ while (n->is_ConstraintCast()) {
+ if (n->in(0) != NULL && n->in(0)->is_top()) {
+ // Will die, don't optimize
+ can_optimize = false;
+ break;
+ }
+ casts.push(n);
+ n = n->in(1);
+ }
+ const Type* t = phase->type(n);
+ if (n->is_InlineType() && (vk == NULL || vk == t->inline_klass())) {
+ vk = (vk == NULL) ? t->inline_klass() : vk;
+ if (phase->find_int_con(n->as_InlineType()->get_is_init(), 0) != 1) {
+ is_init = false;
+ }
+ } else if (n->is_Phi() && can_reshape && n->bottom_type()->isa_ptr()) {
+ worklist.push(n);
+ } else if (t->is_zero_type()) {
+ is_init = false;
+ } else {
+ can_optimize = false;
+ }
+ }
+ }
+ // Check if cast nodes can be pushed through
+ const Type* t = Type::get_const_type(vk);
+ while (casts.size() != 0 && can_optimize && t != NULL) {
+ Node* cast = casts.pop();
+ if (t->filter(cast->bottom_type()) == Type::TOP) {
+ can_optimize = false;
+ }
+ }
+ if (can_optimize && vk != NULL) {
+ // TODO 8302217
+ // assert(!_type->isa_ptr() || _type->maybe_null() || is_init, "Phi not null but a possible null was seen");
+ return push_inline_types_through(phase, can_reshape, vk, is_init);
+ }
+ }
+
// Phi (VB ... VB) => VB (Phi ...) (Phi ...)
if (EnableVectorReboxing && can_reshape && progress == NULL && type()->isa_oopptr()) {
progress = merge_through_phi(this, phase->is_IterGVN());
}
}
} else {
stack.pop();
}
}
+
assert(cached_vbox != NULL, "sanity");
const TypeInstPtr* btype = cached_vbox->box_type();
const TypeVect* vtype = cached_vbox->vec_type();
Node* new_vbox_phi = clone_through_phi(root_phi, btype, VectorBoxNode::Box, igvn);
Node* new_vect_phi = clone_through_phi(root_phi, vtype, VectorBoxNode::Value, igvn);
if( phase->type(in(1)) == Type::TOP ) return in(1);
if( phase->type(in(0)) == Type::TOP ) return in(0);
// We only come from CatchProj, unless the CatchProj goes away.
// If the CatchProj is optimized away, then we just carry the
// exception oop through.
+
+ // CheckCastPPNode::Ideal() for inline types reuses the exception
+ // paths of a call to perform an allocation: we can see a Phi here.
+ if (in(1)->is_Phi()) {
+ return this;
+ }
CallNode *call = in(1)->in(0)->as_Call();
return (in(0)->is_CatchProj() && in(0)->in(0)->is_Catch() &&
in(0)->in(0)->in(1) == in(1)) ? this : call->in(TypeFunc::Parms);
}
< prev index next >