< prev index next >

src/hotspot/share/opto/escape.cpp

Print this page

  11  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  12  * version 2 for more details (a copy is included in the LICENSE file that
  13  * accompanied this code).
  14  *
  15  * You should have received a copy of the GNU General Public License version
  16  * 2 along with this work; if not, write to the Free Software Foundation,
  17  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
  18  *
  19  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
  20  * or visit www.oracle.com if you need additional information or have any
  21  * questions.
  22  *
  23  */
  24 
  25 #include "ci/bcEscapeAnalyzer.hpp"
  26 #include "compiler/compileLog.hpp"
  27 #include "gc/shared/barrierSet.hpp"
  28 #include "gc/shared/c2/barrierSetC2.hpp"
  29 #include "libadt/vectset.hpp"
  30 #include "memory/allocation.hpp"

  31 #include "memory/resourceArea.hpp"
  32 #include "opto/arraycopynode.hpp"
  33 #include "opto/c2compiler.hpp"
  34 #include "opto/callnode.hpp"
  35 #include "opto/castnode.hpp"
  36 #include "opto/cfgnode.hpp"
  37 #include "opto/compile.hpp"
  38 #include "opto/escape.hpp"

  39 #include "opto/locknode.hpp"
  40 #include "opto/macro.hpp"
  41 #include "opto/movenode.hpp"
  42 #include "opto/narrowptrnode.hpp"
  43 #include "opto/phaseX.hpp"
  44 #include "opto/rootnode.hpp"
  45 #include "utilities/macros.hpp"
  46 
  47 ConnectionGraph::ConnectionGraph(Compile * C, PhaseIterGVN *igvn, int invocation) :
  48   // If ReduceAllocationMerges is enabled we might call split_through_phi during
  49   // split_unique_types and that will create additional nodes that need to be
  50   // pushed to the ConnectionGraph. The code below bumps the initial capacity of
  51   // _nodes by 10% to account for these additional nodes. If capacity is exceeded
  52   // the array will be reallocated.
  53   _nodes(C->comp_arena(), C->do_reduce_allocation_merges() ? C->unique()*1.10 : C->unique(), C->unique(), nullptr),
  54   _in_worklist(C->comp_arena()),
  55   _next_pidx(0),
  56   _collecting(true),
  57   _verify(false),
  58   _compile(C),

 146   GrowableArray<SafePointNode*>  sfn_worklist;
 147   GrowableArray<MergeMemNode*>   mergemem_worklist;
 148   DEBUG_ONLY( GrowableArray<Node*> addp_worklist; )
 149 
 150   { Compile::TracePhase tp(Phase::_t_connectionGraph);
 151 
 152   // 1. Populate Connection Graph (CG) with PointsTo nodes.
 153   ideal_nodes.map(C->live_nodes(), nullptr);  // preallocate space
 154   // Initialize worklist
 155   if (C->root() != nullptr) {
 156     ideal_nodes.push(C->root());
 157   }
 158   // Processed ideal nodes are unique on ideal_nodes list
 159   // but several ideal nodes are mapped to the phantom_obj.
 160   // To avoid duplicated entries on the following worklists
 161   // add the phantom_obj only once to them.
 162   ptnodes_worklist.append(phantom_obj);
 163   java_objects_worklist.append(phantom_obj);
 164   for( uint next = 0; next < ideal_nodes.size(); ++next ) {
 165     Node* n = ideal_nodes.at(next);










 166     // Create PointsTo nodes and add them to Connection Graph. Called
 167     // only once per ideal node since ideal_nodes is Unique_Node list.
 168     add_node_to_connection_graph(n, &delayed_worklist);
 169     PointsToNode* ptn = ptnode_adr(n->_idx);
 170     if (ptn != nullptr && ptn != phantom_obj) {
 171       ptnodes_worklist.append(ptn);
 172       if (ptn->is_JavaObject()) {
 173         java_objects_worklist.append(ptn->as_JavaObject());
 174         if ((n->is_Allocate() || n->is_CallStaticJava()) &&
 175             (ptn->escape_state() < PointsToNode::GlobalEscape)) {
 176           // Only allocations and java static calls results are interesting.
 177           non_escaped_allocs_worklist.append(ptn->as_JavaObject());
 178         }
 179       } else if (ptn->is_Field() && ptn->as_Field()->is_oop()) {
 180         oop_fields_worklist.append(ptn->as_Field());
 181       }
 182     }
 183     // Collect some interesting nodes for further use.
 184     switch (n->Opcode()) {
 185       case Op_MergeMem:

 396     split_unique_types(alloc_worklist, arraycopy_worklist, mergemem_worklist, reducible_merges);
 397     if (C->failing()) {
 398       NOT_PRODUCT(escape_state_statistics(java_objects_worklist);)
 399       return false;
 400     }
 401     C->print_method(PHASE_AFTER_EA, 2);
 402 
 403 #ifdef ASSERT
 404   } else if (Verbose && (PrintEscapeAnalysis || PrintEliminateAllocations)) {
 405     tty->print("=== No allocations eliminated for ");
 406     C->method()->print_short_name();
 407     if (!EliminateAllocations) {
 408       tty->print(" since EliminateAllocations is off ===");
 409     } else if(!has_scalar_replaceable_candidates) {
 410       tty->print(" since there are no scalar replaceable candidates ===");
 411     }
 412     tty->cr();
 413 #endif
 414   }
 415 
 416   // 6. Reduce allocation merges used as debug information. This is done after








 417   // split_unique_types because the methods used to create SafePointScalarObject
 418   // need to traverse the memory graph to find values for object fields. We also
 419   // set to null the scalarized inputs of reducible Phis so that the Allocate
 420   // that they point can be later scalar replaced.
 421   bool delay = _igvn->delay_transform();
 422   _igvn->set_delay_transform(true);
 423   for (uint i = 0; i < reducible_merges.size(); i++) {
 424     Node* n = reducible_merges.at(i);
 425     if (n->outcnt() > 0) {
 426       if (!reduce_phi_on_safepoints(n->as_Phi())) {
 427         NOT_PRODUCT(escape_state_statistics(java_objects_worklist);)
 428         C->record_failure(C2Compiler::retry_no_reduce_allocation_merges());
 429         return false;
 430       }
 431 
 432       // Now we set the scalar replaceable inputs of ophi to null, which is
 433       // the last piece that would prevent it from being scalar replaceable.
 434       reset_scalar_replaceable_entries(n->as_Phi());
 435     }
 436   }

1235 
1236     // The next two inputs are:
1237     //  (1) A copy of the original pointer to NSR objects.
1238     //  (2) A selector, used to decide if we need to rematerialize an object
1239     //      or use the pointer to a NSR object.
1240     // See more details of these fields in the declaration of SafePointScalarMergeNode
1241     sfpt->add_req(nsr_merge_pointer);
1242     sfpt->add_req(selector);
1243 
1244     for (uint i = 1; i < ophi->req(); i++) {
1245       Node* base = ophi->in(i);
1246       JavaObjectNode* ptn = unique_java_object(base);
1247 
1248       // If the base is not scalar replaceable we don't need to register information about
1249       // it at this time.
1250       if (ptn == nullptr || !ptn->scalar_replaceable()) {
1251         continue;
1252       }
1253 
1254       AllocateNode* alloc = ptn->ideal_node()->as_Allocate();
1255       SafePointScalarObjectNode* sobj = mexp.create_scalarized_object_description(alloc, sfpt);








1256       if (sobj == nullptr) {

1257         return false;
1258       }
1259 
1260       // Now make a pass over the debug information replacing any references
1261       // to the allocated object with "sobj"
1262       Node* ccpp = alloc->result_cast();
1263       sfpt->replace_edges_in_range(ccpp, sobj, debug_start, jvms->debug_end(), _igvn);
1264 
1265       // Register the scalarized object as a candidate for reallocation
1266       smerge->add_req(sobj);









1267     }
1268 
1269     // Replaces debug information references to "original_sfpt_parent" in "sfpt" with references to "smerge"
1270     sfpt->replace_edges_in_range(original_sfpt_parent, smerge, debug_start, jvms->debug_end(), _igvn);
1271 
1272     // The call to 'replace_edges_in_range' above might have removed the
1273     // reference to ophi that we need at _merge_pointer_idx. The line below make
1274     // sure the reference is maintained.
1275     sfpt->set_req(smerge->merge_pointer_idx(jvms), nsr_merge_pointer);
1276     _igvn->_worklist.push(sfpt);
1277   }
1278 
1279   return true;
1280 }
1281 
1282 void ConnectionGraph::reduce_phi(PhiNode* ophi, GrowableArray<Node *>  &alloc_worklist, GrowableArray<Node *>  &memnode_worklist) {
1283   bool delay = _igvn->delay_transform();
1284   _igvn->set_delay_transform(true);
1285   _igvn->hash_delete(ophi);
1286 

1445   return false;
1446 }
1447 
1448 // Returns true if at least one of the arguments to the call is an object
1449 // that does not escape globally.
1450 bool ConnectionGraph::has_arg_escape(CallJavaNode* call) {
1451   if (call->method() != nullptr) {
1452     uint max_idx = TypeFunc::Parms + call->method()->arg_size();
1453     for (uint idx = TypeFunc::Parms; idx < max_idx; idx++) {
1454       Node* p = call->in(idx);
1455       if (not_global_escape(p)) {
1456         return true;
1457       }
1458     }
1459   } else {
1460     const char* name = call->as_CallStaticJava()->_name;
1461     assert(name != nullptr, "no name");
1462     // no arg escapes through uncommon traps
1463     if (strcmp(name, "uncommon_trap") != 0) {
1464       // process_call_arguments() assumes that all arguments escape globally
1465       const TypeTuple* d = call->tf()->domain();
1466       for (uint i = TypeFunc::Parms; i < d->cnt(); i++) {
1467         const Type* at = d->field_at(i);
1468         if (at->isa_oopptr() != nullptr) {
1469           return true;
1470         }
1471       }
1472     }
1473   }
1474   return false;
1475 }
1476 
1477 
1478 
1479 // Utility function for nodes that load an object
1480 void ConnectionGraph::add_objload_to_connection_graph(Node *n, Unique_Node_List *delayed_worklist) {
1481   // Using isa_ptr() instead of isa_oopptr() for LoadP and Phi because
1482   // ThreadLocal has RawPtr type.
1483   const Type* t = _igvn->type(n);
1484   if (t->make_ptr() != nullptr) {
1485     Node* adr = n->in(MemNode::Address);

1519       // first IGVN optimization when escape information is still available.
1520       record_for_optimizer(n);
1521     } else if (n->is_Allocate()) {
1522       add_call_node(n->as_Call());
1523       record_for_optimizer(n);
1524     } else {
1525       if (n->is_CallStaticJava()) {
1526         const char* name = n->as_CallStaticJava()->_name;
1527         if (name != nullptr && strcmp(name, "uncommon_trap") == 0) {
1528           return; // Skip uncommon traps
1529         }
1530       }
1531       // Don't mark as processed since call's arguments have to be processed.
1532       delayed_worklist->push(n);
1533       // Check if a call returns an object.
1534       if ((n->as_Call()->returns_pointer() &&
1535            n->as_Call()->proj_out_or_null(TypeFunc::Parms) != nullptr) ||
1536           (n->is_CallStaticJava() &&
1537            n->as_CallStaticJava()->is_boxing_method())) {
1538         add_call_node(n->as_Call());











1539       }
1540     }
1541     return;
1542   }
1543   // Put this check here to process call arguments since some call nodes
1544   // point to phantom_obj.
1545   if (n_ptn == phantom_obj || n_ptn == null_obj) {
1546     return; // Skip predefined nodes.
1547   }
1548   switch (opcode) {
1549     case Op_AddP: {
1550       Node* base = get_addp_base(n);
1551       PointsToNode* ptn_base = ptnode_adr(base->_idx);
1552       // Field nodes are created for all field types. They are used in
1553       // adjust_scalar_replaceable_state() and split_unique_types().
1554       // Note, non-oop fields will have only base edges in Connection
1555       // Graph because such fields are not used for oop loads and stores.
1556       int offset = address_offset(n, igvn);
1557       add_field(n, PointsToNode::NoEscape, offset);
1558       if (ptn_base == nullptr) {
1559         delayed_worklist->push(n); // Process it later.
1560       } else {
1561         n_ptn = ptnode_adr(n_idx);
1562         add_base(n_ptn->as_Field(), ptn_base);
1563       }
1564       break;
1565     }
1566     case Op_CastX2P: {

1567       map_ideal_node(n, phantom_obj);
1568       break;
1569     }

1570     case Op_CastPP:
1571     case Op_CheckCastPP:
1572     case Op_EncodeP:
1573     case Op_DecodeN:
1574     case Op_EncodePKlass:
1575     case Op_DecodeNKlass: {
1576       add_local_var_and_edge(n, PointsToNode::NoEscape, n->in(1), delayed_worklist);
1577       break;
1578     }
1579     case Op_CMoveP: {
1580       add_local_var(n, PointsToNode::NoEscape);
1581       // Do not add edges during first iteration because some could be
1582       // not defined yet.
1583       delayed_worklist->push(n);
1584       break;
1585     }
1586     case Op_ConP:
1587     case Op_ConN:
1588     case Op_ConNKlass: {
1589       // assume all oop constants globally escape except for null

1619       break;
1620     }
1621     case Op_PartialSubtypeCheck: {
1622       // Produces Null or notNull and is used in only in CmpP so
1623       // phantom_obj could be used.
1624       map_ideal_node(n, phantom_obj); // Result is unknown
1625       break;
1626     }
1627     case Op_Phi: {
1628       // Using isa_ptr() instead of isa_oopptr() for LoadP and Phi because
1629       // ThreadLocal has RawPtr type.
1630       const Type* t = n->as_Phi()->type();
1631       if (t->make_ptr() != nullptr) {
1632         add_local_var(n, PointsToNode::NoEscape);
1633         // Do not add edges during first iteration because some could be
1634         // not defined yet.
1635         delayed_worklist->push(n);
1636       }
1637       break;
1638     }








1639     case Op_Proj: {
1640       // we are only interested in the oop result projection from a call
1641       if (n->as_Proj()->_con == TypeFunc::Parms && n->in(0)->is_Call() &&
1642           n->in(0)->as_Call()->returns_pointer()) {





1643         add_local_var_and_edge(n, PointsToNode::NoEscape, n->in(0), delayed_worklist);
1644       }
1645       break;
1646     }
1647     case Op_Rethrow: // Exception object escapes
1648     case Op_Return: {
1649       if (n->req() > TypeFunc::Parms &&
1650           igvn->type(n->in(TypeFunc::Parms))->isa_oopptr()) {
1651         // Treat Return value as LocalVar with GlobalEscape escape state.
1652         add_local_var_and_edge(n, PointsToNode::GlobalEscape, n->in(TypeFunc::Parms), delayed_worklist);
1653       }
1654       break;
1655     }
1656     case Op_CompareAndExchangeP:
1657     case Op_CompareAndExchangeN:
1658     case Op_GetAndSetP:
1659     case Op_GetAndSetN: {
1660       add_objload_to_connection_graph(n, delayed_worklist);
1661       // fall-through
1662     }

1708       break;
1709     }
1710     default:
1711       ; // Do nothing for nodes not related to EA.
1712   }
1713   return;
1714 }
1715 
1716 // Add final simple edges to graph.
1717 void ConnectionGraph::add_final_edges(Node *n) {
1718   PointsToNode* n_ptn = ptnode_adr(n->_idx);
1719 #ifdef ASSERT
1720   if (_verify && n_ptn->is_JavaObject())
1721     return; // This method does not change graph for JavaObject.
1722 #endif
1723 
1724   if (n->is_Call()) {
1725     process_call_arguments(n->as_Call());
1726     return;
1727   }
1728   assert(n->is_Store() || n->is_LoadStore() ||
1729          ((n_ptn != nullptr) && (n_ptn->ideal_node() != nullptr)),
1730          "node should be registered already");
1731   int opcode = n->Opcode();
1732   bool gc_handled = BarrierSet::barrier_set()->barrier_set_c2()->escape_add_final_edges(this, _igvn, n, opcode);
1733   if (gc_handled) {
1734     return; // Ignore node if already handled by GC.
1735   }
1736   switch (opcode) {
1737     case Op_AddP: {
1738       Node* base = get_addp_base(n);
1739       PointsToNode* ptn_base = ptnode_adr(base->_idx);
1740       assert(ptn_base != nullptr, "field's base should be registered");
1741       add_base(n_ptn->as_Field(), ptn_base);
1742       break;
1743     }

1744     case Op_CastPP:
1745     case Op_CheckCastPP:
1746     case Op_EncodeP:
1747     case Op_DecodeN:
1748     case Op_EncodePKlass:
1749     case Op_DecodeNKlass: {
1750       add_local_var_and_edge(n, PointsToNode::NoEscape, n->in(1), nullptr);
1751       break;
1752     }
1753     case Op_CMoveP: {
1754       for (uint i = CMoveNode::IfFalse; i < n->req(); i++) {
1755         Node* in = n->in(i);
1756         if (in == nullptr) {
1757           continue;  // ignore null
1758         }
1759         Node* uncast_in = in->uncast();
1760         if (uncast_in->is_top() || uncast_in == n) {
1761           continue;  // ignore top or inputs which go back this node
1762         }
1763         PointsToNode* ptn = ptnode_adr(in->_idx);

1776     }
1777     case Op_Phi: {
1778       // Using isa_ptr() instead of isa_oopptr() for LoadP and Phi because
1779       // ThreadLocal has RawPtr type.
1780       assert(n->as_Phi()->type()->make_ptr() != nullptr, "Unexpected node type");
1781       for (uint i = 1; i < n->req(); i++) {
1782         Node* in = n->in(i);
1783         if (in == nullptr) {
1784           continue;  // ignore null
1785         }
1786         Node* uncast_in = in->uncast();
1787         if (uncast_in->is_top() || uncast_in == n) {
1788           continue;  // ignore top or inputs which go back this node
1789         }
1790         PointsToNode* ptn = ptnode_adr(in->_idx);
1791         assert(ptn != nullptr, "node should be registered");
1792         add_edge(n_ptn, ptn);
1793       }
1794       break;
1795     }
















1796     case Op_Proj: {
1797       // we are only interested in the oop result projection from a call
1798       assert(n->as_Proj()->_con == TypeFunc::Parms && n->in(0)->is_Call() &&
1799              n->in(0)->as_Call()->returns_pointer(), "Unexpected node type");
1800       add_local_var_and_edge(n, PointsToNode::NoEscape, n->in(0), nullptr);





1801       break;
1802     }
1803     case Op_Rethrow: // Exception object escapes
1804     case Op_Return: {
1805       assert(n->req() > TypeFunc::Parms && _igvn->type(n->in(TypeFunc::Parms))->isa_oopptr(),
1806              "Unexpected node type");
1807       // Treat Return value as LocalVar with GlobalEscape escape state.
1808       add_local_var_and_edge(n, PointsToNode::GlobalEscape, n->in(TypeFunc::Parms), nullptr);
1809       break;
1810     }
1811     case Op_CompareAndExchangeP:
1812     case Op_CompareAndExchangeN:
1813     case Op_GetAndSetP:
1814     case Op_GetAndSetN:{
1815       assert(_igvn->type(n)->make_ptr() != nullptr, "Unexpected node type");
1816       add_local_var_and_edge(n, PointsToNode::NoEscape, n->in(MemNode::Address), nullptr);
1817       // fall-through
1818     }
1819     case Op_CompareAndSwapP:
1820     case Op_CompareAndSwapN:

1955     PointsToNode* ptn = ptnode_adr(val->_idx);
1956     assert(ptn != nullptr, "node should be registered");
1957     set_escape_state(ptn, PointsToNode::GlobalEscape NOT_PRODUCT(COMMA "stored at raw address"));
1958     // Add edge to object for unsafe access with offset.
1959     PointsToNode* adr_ptn = ptnode_adr(adr->_idx);
1960     assert(adr_ptn != nullptr, "node should be registered");
1961     if (adr_ptn->is_Field()) {
1962       assert(adr_ptn->as_Field()->is_oop(), "should be oop field");
1963       add_edge(adr_ptn, ptn);
1964     }
1965     return true;
1966   }
1967 #ifdef ASSERT
1968   n->dump(1);
1969   assert(false, "not unsafe");
1970 #endif
1971   return false;
1972 }
1973 
1974 void ConnectionGraph::add_call_node(CallNode* call) {
1975   assert(call->returns_pointer(), "only for call which returns pointer");
1976   uint call_idx = call->_idx;
1977   if (call->is_Allocate()) {
1978     Node* k = call->in(AllocateNode::KlassNode);
1979     const TypeKlassPtr* kt = k->bottom_type()->isa_klassptr();
1980     assert(kt != nullptr, "TypeKlassPtr  required.");
1981     PointsToNode::EscapeState es = PointsToNode::NoEscape;
1982     bool scalar_replaceable = true;
1983     NOT_PRODUCT(const char* nsr_reason = "");
1984     if (call->is_AllocateArray()) {
1985       if (!kt->isa_aryklassptr()) { // StressReflectiveCode
1986         es = PointsToNode::GlobalEscape;
1987       } else {
1988         int length = call->in(AllocateNode::ALength)->find_int_con(-1);
1989         if (length < 0) {
1990           // Not scalar replaceable if the length is not constant.
1991           scalar_replaceable = false;
1992           NOT_PRODUCT(nsr_reason = "has a non-constant length");
1993         } else if (length > EliminateAllocationArraySizeLimit) {
1994           // Not scalar replaceable if the length is too big.
1995           scalar_replaceable = false;

2031     //
2032     //    - all oop arguments are escaping globally;
2033     //
2034     // 2. CallStaticJavaNode (execute bytecode analysis if possible):
2035     //
2036     //    - the same as CallDynamicJavaNode if can't do bytecode analysis;
2037     //
2038     //    - mapped to GlobalEscape JavaObject node if unknown oop is returned;
2039     //    - mapped to NoEscape JavaObject node if non-escaping object allocated
2040     //      during call is returned;
2041     //    - mapped to ArgEscape LocalVar node pointed to object arguments
2042     //      which are returned and does not escape during call;
2043     //
2044     //    - oop arguments escaping status is defined by bytecode analysis;
2045     //
2046     // For a static call, we know exactly what method is being called.
2047     // Use bytecode estimator to record whether the call's return value escapes.
2048     ciMethod* meth = call->as_CallJava()->method();
2049     if (meth == nullptr) {
2050       const char* name = call->as_CallStaticJava()->_name;
2051       assert(strncmp(name, "C2 Runtime multianewarray", 25) == 0, "TODO: add failed case check");


2052       // Returns a newly allocated non-escaped object.
2053       add_java_object(call, PointsToNode::NoEscape);
2054       set_not_scalar_replaceable(ptnode_adr(call_idx) NOT_PRODUCT(COMMA "is result of multinewarray"));
2055     } else if (meth->is_boxing_method()) {
2056       // Returns boxing object
2057       PointsToNode::EscapeState es;
2058       vmIntrinsics::ID intr = meth->intrinsic_id();
2059       if (intr == vmIntrinsics::_floatValue || intr == vmIntrinsics::_doubleValue) {
2060         // It does not escape if object is always allocated.
2061         es = PointsToNode::NoEscape;
2062       } else {
2063         // It escapes globally if object could be loaded from cache.
2064         es = PointsToNode::GlobalEscape;
2065       }
2066       add_java_object(call, es);
2067       if (es == PointsToNode::GlobalEscape) {
2068         set_not_scalar_replaceable(ptnode_adr(call->_idx) NOT_PRODUCT(COMMA "object can be loaded from boxing cache"));
2069       }
2070     } else {
2071       BCEscapeAnalyzer* call_analyzer = meth->get_bcea();
2072       call_analyzer->copy_dependencies(_compile->dependencies());
2073       if (call_analyzer->is_return_allocated()) {
2074         // Returns a newly allocated non-escaped object, simply
2075         // update dependency information.
2076         // Mark it as NoEscape so that objects referenced by
2077         // it's fields will be marked as NoEscape at least.
2078         add_java_object(call, PointsToNode::NoEscape);
2079         set_not_scalar_replaceable(ptnode_adr(call_idx) NOT_PRODUCT(COMMA "is result of call"));
2080       } else {
2081         // Determine whether any arguments are returned.
2082         const TypeTuple* d = call->tf()->domain();
2083         bool ret_arg = false;
2084         for (uint i = TypeFunc::Parms; i < d->cnt(); i++) {
2085           if (d->field_at(i)->isa_ptr() != nullptr &&
2086               call_analyzer->is_arg_returned(i - TypeFunc::Parms)) {
2087             ret_arg = true;
2088             break;
2089           }
2090         }
2091         if (ret_arg) {
2092           add_local_var(call, PointsToNode::ArgEscape);
2093         } else {
2094           // Returns unknown object.
2095           map_ideal_node(call, phantom_obj);
2096         }
2097       }
2098     }
2099   } else {
2100     // An other type of call, assume the worst case:
2101     // returned value is unknown and globally escapes.
2102     assert(call->Opcode() == Op_CallDynamicJava, "add failed case check");

2110 #ifdef ASSERT
2111     case Op_Allocate:
2112     case Op_AllocateArray:
2113     case Op_Lock:
2114     case Op_Unlock:
2115       assert(false, "should be done already");
2116       break;
2117 #endif
2118     case Op_ArrayCopy:
2119     case Op_CallLeafNoFP:
2120       // Most array copies are ArrayCopy nodes at this point but there
2121       // are still a few direct calls to the copy subroutines (See
2122       // PhaseStringOpts::copy_string())
2123       is_arraycopy = (call->Opcode() == Op_ArrayCopy) ||
2124         call->as_CallLeaf()->is_call_to_arraycopystub();
2125       // fall through
2126     case Op_CallLeafVector:
2127     case Op_CallLeaf: {
2128       // Stub calls, objects do not escape but they are not scale replaceable.
2129       // Adjust escape state for outgoing arguments.
2130       const TypeTuple * d = call->tf()->domain();
2131       bool src_has_oops = false;
2132       for (uint i = TypeFunc::Parms; i < d->cnt(); i++) {
2133         const Type* at = d->field_at(i);
2134         Node *arg = call->in(i);
2135         if (arg == nullptr) {
2136           continue;
2137         }
2138         const Type *aat = _igvn->type(arg);
2139         if (arg->is_top() || !at->isa_ptr() || !aat->isa_ptr()) {
2140           continue;
2141         }
2142         if (arg->is_AddP()) {
2143           //
2144           // The inline_native_clone() case when the arraycopy stub is called
2145           // after the allocation before Initialize and CheckCastPP nodes.
2146           // Or normal arraycopy for object arrays case.
2147           //
2148           // Set AddP's base (Allocate) as not scalar replaceable since
2149           // pointer to the base (with offset) is passed as argument.
2150           //
2151           arg = get_addp_base(arg);
2152         }
2153         PointsToNode* arg_ptn = ptnode_adr(arg->_idx);
2154         assert(arg_ptn != nullptr, "should be registered");
2155         PointsToNode::EscapeState arg_esc = arg_ptn->escape_state();
2156         if (is_arraycopy || arg_esc < PointsToNode::ArgEscape) {
2157           assert(aat == Type::TOP || aat == TypePtr::NULL_PTR ||
2158                  aat->isa_ptr() != nullptr, "expecting an Ptr");
2159           bool arg_has_oops = aat->isa_oopptr() &&
2160                               (aat->isa_instptr() ||
2161                                (aat->isa_aryptr() && (aat->isa_aryptr()->elem() == Type::BOTTOM || aat->isa_aryptr()->elem()->make_oopptr() != nullptr)));



2162           if (i == TypeFunc::Parms) {
2163             src_has_oops = arg_has_oops;
2164           }
2165           //
2166           // src or dst could be j.l.Object when other is basic type array:
2167           //
2168           //   arraycopy(char[],0,Object*,0,size);
2169           //   arraycopy(Object*,0,char[],0,size);
2170           //
2171           // Don't add edges in such cases.
2172           //
2173           bool arg_is_arraycopy_dest = src_has_oops && is_arraycopy &&
2174                                        arg_has_oops && (i > TypeFunc::Parms);
2175 #ifdef ASSERT
2176           if (!(is_arraycopy ||
2177                 BarrierSet::barrier_set()->barrier_set_c2()->is_gc_barrier_node(call) ||
2178                 (call->as_CallLeaf()->_name != nullptr &&
2179                  (strcmp(call->as_CallLeaf()->_name, "updateBytesCRC32") == 0 ||
2180                   strcmp(call->as_CallLeaf()->_name, "updateBytesCRC32C") == 0 ||
2181                   strcmp(call->as_CallLeaf()->_name, "updateBytesAdler32") == 0 ||

2205                   strcmp(call->as_CallLeaf()->_name, "dilithiumMontMulByConstant") == 0 ||
2206                   strcmp(call->as_CallLeaf()->_name, "dilithiumDecomposePoly") == 0 ||
2207                   strcmp(call->as_CallLeaf()->_name, "encodeBlock") == 0 ||
2208                   strcmp(call->as_CallLeaf()->_name, "decodeBlock") == 0 ||
2209                   strcmp(call->as_CallLeaf()->_name, "md5_implCompress") == 0 ||
2210                   strcmp(call->as_CallLeaf()->_name, "md5_implCompressMB") == 0 ||
2211                   strcmp(call->as_CallLeaf()->_name, "sha1_implCompress") == 0 ||
2212                   strcmp(call->as_CallLeaf()->_name, "sha1_implCompressMB") == 0 ||
2213                   strcmp(call->as_CallLeaf()->_name, "sha256_implCompress") == 0 ||
2214                   strcmp(call->as_CallLeaf()->_name, "sha256_implCompressMB") == 0 ||
2215                   strcmp(call->as_CallLeaf()->_name, "sha512_implCompress") == 0 ||
2216                   strcmp(call->as_CallLeaf()->_name, "sha512_implCompressMB") == 0 ||
2217                   strcmp(call->as_CallLeaf()->_name, "sha3_implCompress") == 0 ||
2218                   strcmp(call->as_CallLeaf()->_name, "double_keccak") == 0 ||
2219                   strcmp(call->as_CallLeaf()->_name, "sha3_implCompressMB") == 0 ||
2220                   strcmp(call->as_CallLeaf()->_name, "multiplyToLen") == 0 ||
2221                   strcmp(call->as_CallLeaf()->_name, "squareToLen") == 0 ||
2222                   strcmp(call->as_CallLeaf()->_name, "mulAdd") == 0 ||
2223                   strcmp(call->as_CallLeaf()->_name, "montgomery_multiply") == 0 ||
2224                   strcmp(call->as_CallLeaf()->_name, "montgomery_square") == 0 ||




2225                   strcmp(call->as_CallLeaf()->_name, "bigIntegerRightShiftWorker") == 0 ||
2226                   strcmp(call->as_CallLeaf()->_name, "bigIntegerLeftShiftWorker") == 0 ||
2227                   strcmp(call->as_CallLeaf()->_name, "vectorizedMismatch") == 0 ||
2228                   strcmp(call->as_CallLeaf()->_name, "stringIndexOf") == 0 ||
2229                   strcmp(call->as_CallLeaf()->_name, "arraysort_stub") == 0 ||
2230                   strcmp(call->as_CallLeaf()->_name, "array_partition_stub") == 0 ||
2231                   strcmp(call->as_CallLeaf()->_name, "get_class_id_intrinsic") == 0 ||
2232                   strcmp(call->as_CallLeaf()->_name, "unsafe_setmemory") == 0)
2233                  ))) {
2234             call->dump();
2235             fatal("EA unexpected CallLeaf %s", call->as_CallLeaf()->_name);
2236           }
2237 #endif
2238           // Always process arraycopy's destination object since
2239           // we need to add all possible edges to references in
2240           // source object.
2241           if (arg_esc >= PointsToNode::ArgEscape &&
2242               !arg_is_arraycopy_dest) {
2243             continue;
2244           }

2267           }
2268         }
2269       }
2270       break;
2271     }
2272     case Op_CallStaticJava: {
2273       // For a static call, we know exactly what method is being called.
2274       // Use bytecode estimator to record the call's escape affects
2275 #ifdef ASSERT
2276       const char* name = call->as_CallStaticJava()->_name;
2277       assert((name == nullptr || strcmp(name, "uncommon_trap") != 0), "normal calls only");
2278 #endif
2279       ciMethod* meth = call->as_CallJava()->method();
2280       if ((meth != nullptr) && meth->is_boxing_method()) {
2281         break; // Boxing methods do not modify any oops.
2282       }
2283       BCEscapeAnalyzer* call_analyzer = (meth !=nullptr) ? meth->get_bcea() : nullptr;
2284       // fall-through if not a Java method or no analyzer information
2285       if (call_analyzer != nullptr) {
2286         PointsToNode* call_ptn = ptnode_adr(call->_idx);
2287         const TypeTuple* d = call->tf()->domain();
2288         for (uint i = TypeFunc::Parms; i < d->cnt(); i++) {
2289           const Type* at = d->field_at(i);
2290           int k = i - TypeFunc::Parms;
2291           Node* arg = call->in(i);
2292           PointsToNode* arg_ptn = ptnode_adr(arg->_idx);
2293           if (at->isa_ptr() != nullptr &&
2294               call_analyzer->is_arg_returned(k)) {
2295             // The call returns arguments.
2296             if (call_ptn != nullptr) { // Is call's result used?
2297               assert(call_ptn->is_LocalVar(), "node should be registered");
2298               assert(arg_ptn != nullptr, "node should be registered");
2299               add_edge(call_ptn, arg_ptn);
2300             }
2301           }
2302           if (at->isa_oopptr() != nullptr &&
2303               arg_ptn->escape_state() < PointsToNode::GlobalEscape) {
2304             if (!call_analyzer->is_arg_stack(k)) {
2305               // The argument global escapes
2306               set_escape_state(arg_ptn, PointsToNode::GlobalEscape NOT_PRODUCT(COMMA trace_arg_escape_message(call)));
2307             } else {

2311                 set_fields_escape_state(arg_ptn, PointsToNode::GlobalEscape NOT_PRODUCT(COMMA trace_arg_escape_message(call)));
2312               }
2313             }
2314           }
2315         }
2316         if (call_ptn != nullptr && call_ptn->is_LocalVar()) {
2317           // The call returns arguments.
2318           assert(call_ptn->edge_count() > 0, "sanity");
2319           if (!call_analyzer->is_return_local()) {
2320             // Returns also unknown object.
2321             add_edge(call_ptn, phantom_obj);
2322           }
2323         }
2324         break;
2325       }
2326     }
2327     default: {
2328       // Fall-through here if not a Java method or no analyzer information
2329       // or some other type of call, assume the worst case: all arguments
2330       // globally escape.
2331       const TypeTuple* d = call->tf()->domain();
2332       for (uint i = TypeFunc::Parms; i < d->cnt(); i++) {
2333         const Type* at = d->field_at(i);
2334         if (at->isa_oopptr() != nullptr) {
2335           Node* arg = call->in(i);
2336           if (arg->is_AddP()) {
2337             arg = get_addp_base(arg);
2338           }
2339           assert(ptnode_adr(arg->_idx) != nullptr, "should be defined already");
2340           set_escape_state(ptnode_adr(arg->_idx), PointsToNode::GlobalEscape NOT_PRODUCT(COMMA trace_arg_escape_message(call)));
2341         }
2342       }
2343     }
2344   }
2345 }
2346 
2347 
2348 // Finish Graph construction.
2349 bool ConnectionGraph::complete_connection_graph(
2350                          GrowableArray<PointsToNode*>&   ptnodes_worklist,
2351                          GrowableArray<JavaObjectNode*>& non_escaped_allocs_worklist,

2724     PointsToNode* base = i.get();
2725     if (base->is_JavaObject()) {
2726       // Skip Allocate's fields which will be processed later.
2727       if (base->ideal_node()->is_Allocate()) {
2728         return 0;
2729       }
2730       assert(base == null_obj, "only null ptr base expected here");
2731     }
2732   }
2733   if (add_edge(field, phantom_obj)) {
2734     // New edge was added
2735     new_edges++;
2736     add_field_uses_to_worklist(field);
2737   }
2738   return new_edges;
2739 }
2740 
2741 // Find fields initializing values for allocations.
2742 int ConnectionGraph::find_init_values_phantom(JavaObjectNode* pta) {
2743   assert(pta->escape_state() == PointsToNode::NoEscape, "Not escaped Allocate nodes only");

2744   Node* alloc = pta->ideal_node();
2745 
2746   // Do nothing for Allocate nodes since its fields values are
2747   // "known" unless they are initialized by arraycopy/clone.
2748   if (alloc->is_Allocate() && !pta->arraycopy_dst()) {
2749     return 0;






2750   }
2751   assert(pta->arraycopy_dst() || alloc->as_CallStaticJava(), "sanity");

2752 #ifdef ASSERT
2753   if (!pta->arraycopy_dst() && alloc->as_CallStaticJava()->method() == nullptr) {
2754     const char* name = alloc->as_CallStaticJava()->_name;
2755     assert(strncmp(name, "C2 Runtime multianewarray", 25) == 0, "sanity");


2756   }
2757 #endif
2758   // Non-escaped allocation returned from Java or runtime call have unknown values in fields.
2759   int new_edges = 0;
2760   for (EdgeIterator i(pta); i.has_next(); i.next()) {
2761     PointsToNode* field = i.get();
2762     if (field->is_Field() && field->as_Field()->is_oop()) {
2763       if (add_edge(field, phantom_obj)) {
2764         // New edge was added
2765         new_edges++;
2766         add_field_uses_to_worklist(field->as_Field());
2767       }
2768     }
2769   }
2770   return new_edges;
2771 }
2772 
2773 // Find fields initializing values for allocations.
2774 int ConnectionGraph::find_init_values_null(JavaObjectNode* pta, PhaseValues* phase) {
2775   assert(pta->escape_state() == PointsToNode::NoEscape, "Not escaped Allocate nodes only");
2776   Node* alloc = pta->ideal_node();
2777   // Do nothing for Call nodes since its fields values are unknown.
2778   if (!alloc->is_Allocate()) {
2779     return 0;
2780   }
2781   InitializeNode* ini = alloc->as_Allocate()->initialization();
2782   bool visited_bottom_offset = false;
2783   GrowableArray<int> offsets_worklist;
2784   int new_edges = 0;
2785 
2786   // Check if an oop field's initializing value is recorded and add
2787   // a corresponding null if field's value if it is not recorded.
2788   // Connection Graph does not record a default initialization by null
2789   // captured by Initialize node.
2790   //
2791   for (EdgeIterator i(pta); i.has_next(); i.next()) {
2792     PointsToNode* field = i.get(); // Field (AddP)
2793     if (!field->is_Field() || !field->as_Field()->is_oop()) {
2794       continue; // Not oop field
2795     }
2796     int offset = field->as_Field()->offset();
2797     if (offset == Type::OffsetBot) {
2798       if (!visited_bottom_offset) {

2844               } else {
2845                 if (!val->is_LocalVar() || (val->edge_count() == 0)) {
2846                   tty->print_cr("----------init store has invalid value -----");
2847                   store->dump();
2848                   val->dump();
2849                   assert(val->is_LocalVar() && (val->edge_count() > 0), "should be processed already");
2850                 }
2851                 for (EdgeIterator j(val); j.has_next(); j.next()) {
2852                   PointsToNode* obj = j.get();
2853                   if (obj->is_JavaObject()) {
2854                     if (!field->points_to(obj->as_JavaObject())) {
2855                       missed_obj = obj;
2856                       break;
2857                     }
2858                   }
2859                 }
2860               }
2861               if (missed_obj != nullptr) {
2862                 tty->print_cr("----------field---------------------------------");
2863                 field->dump();
2864                 tty->print_cr("----------missed referernce to object-----------");
2865                 missed_obj->dump();
2866                 tty->print_cr("----------object referernced by init store -----");
2867                 store->dump();
2868                 val->dump();
2869                 assert(!field->points_to(missed_obj->as_JavaObject()), "missed JavaObject reference");
2870               }
2871             }
2872 #endif
2873           } else {
2874             // There could be initializing stores which follow allocation.
2875             // For example, a volatile field store is not collected
2876             // by Initialize node.
2877             //
2878             // Need to check for dependent loads to separate such stores from
2879             // stores which follow loads. For now, add initial value null so
2880             // that compare pointers optimization works correctly.
2881           }
2882         }
2883         if (value == nullptr) {
2884           // A field's initializing value was not recorded. Add null.
2885           if (add_edge(field, null_obj)) {
2886             // New edge was added

3210         assert(field->edge_count() > 0, "sanity");
3211       }
3212     }
3213   }
3214 }
3215 #endif
3216 
3217 // Optimize ideal graph.
3218 void ConnectionGraph::optimize_ideal_graph(GrowableArray<Node*>& ptr_cmp_worklist,
3219                                            GrowableArray<MemBarStoreStoreNode*>& storestore_worklist) {
3220   Compile* C = _compile;
3221   PhaseIterGVN* igvn = _igvn;
3222   if (EliminateLocks) {
3223     // Mark locks before changing ideal graph.
3224     int cnt = C->macro_count();
3225     for (int i = 0; i < cnt; i++) {
3226       Node *n = C->macro_node(i);
3227       if (n->is_AbstractLock()) { // Lock and Unlock nodes
3228         AbstractLockNode* alock = n->as_AbstractLock();
3229         if (!alock->is_non_esc_obj()) {
3230           if (can_eliminate_lock(alock)) {

3231             assert(!alock->is_eliminated() || alock->is_coarsened(), "sanity");
3232             // The lock could be marked eliminated by lock coarsening
3233             // code during first IGVN before EA. Replace coarsened flag
3234             // to eliminate all associated locks/unlocks.
3235 #ifdef ASSERT
3236             alock->log_lock_optimization(C, "eliminate_lock_set_non_esc3");
3237 #endif
3238             alock->set_non_esc_obj();
3239           }
3240         }
3241       }
3242     }
3243   }
3244 
3245   if (OptimizePtrCompare) {
3246     for (int i = 0; i < ptr_cmp_worklist.length(); i++) {
3247       Node *n = ptr_cmp_worklist.at(i);
3248       assert(n->Opcode() == Op_CmpN || n->Opcode() == Op_CmpP, "must be");
3249       const TypeInt* tcmp = optimize_ptr_compare(n->in(1), n->in(2));
3250       if (tcmp->singleton()) {

3252 #ifndef PRODUCT
3253         if (PrintOptimizePtrCompare) {
3254           tty->print_cr("++++ Replaced: %d %s(%d,%d) --> %s", n->_idx, (n->Opcode() == Op_CmpP ? "CmpP" : "CmpN"), n->in(1)->_idx, n->in(2)->_idx, (tcmp == TypeInt::CC_EQ ? "EQ" : "NotEQ"));
3255           if (Verbose) {
3256             n->dump(1);
3257           }
3258         }
3259 #endif
3260         igvn->replace_node(n, cmp);
3261       }
3262     }
3263   }
3264 
3265   // For MemBarStoreStore nodes added in library_call.cpp, check
3266   // escape status of associated AllocateNode and optimize out
3267   // MemBarStoreStore node if the allocated object never escapes.
3268   for (int i = 0; i < storestore_worklist.length(); i++) {
3269     Node* storestore = storestore_worklist.at(i);
3270     Node* alloc = storestore->in(MemBarNode::Precedent)->in(0);
3271     if (alloc->is_Allocate() && not_global_escape(alloc)) {
3272       MemBarNode* mb = MemBarNode::make(C, Op_MemBarCPUOrder, Compile::AliasIdxBot);
3273       mb->init_req(TypeFunc::Memory,  storestore->in(TypeFunc::Memory));
3274       mb->init_req(TypeFunc::Control, storestore->in(TypeFunc::Control));
3275       igvn->register_new_node_with_optimizer(mb);
3276       igvn->replace_node(storestore, mb);





3277     }
3278   }
3279 }
3280 

























3281 // Optimize objects compare.
3282 const TypeInt* ConnectionGraph::optimize_ptr_compare(Node* left, Node* right) {
3283   const TypeInt* UNKNOWN = TypeInt::CC;    // [-1, 0,1]
3284   if (!OptimizePtrCompare) {
3285     return UNKNOWN;
3286   }
3287   const TypeInt* EQ = TypeInt::CC_EQ; // [0] == ZERO
3288   const TypeInt* NE = TypeInt::CC_GT; // [1] == ONE
3289 
3290   PointsToNode* ptn1 = ptnode_adr(left->_idx);
3291   PointsToNode* ptn2 = ptnode_adr(right->_idx);
3292   JavaObjectNode* jobj1 = unique_java_object(left);
3293   JavaObjectNode* jobj2 = unique_java_object(right);
3294 
3295   // The use of this method during allocation merge reduction may cause 'left'
3296   // or 'right' be something (e.g., a Phi) that isn't in the connection graph or
3297   // that doesn't reference an unique java object.
3298   if (ptn1 == nullptr || ptn2 == nullptr ||
3299       jobj1 == nullptr || jobj2 == nullptr) {
3300     return UNKNOWN;

3420   assert(!src->is_Field() && !dst->is_Field(), "only for JavaObject and LocalVar");
3421   assert((src != null_obj) && (dst != null_obj), "not for ConP null");
3422   PointsToNode* ptadr = _nodes.at(n->_idx);
3423   if (ptadr != nullptr) {
3424     assert(ptadr->is_Arraycopy() && ptadr->ideal_node() == n, "sanity");
3425     return;
3426   }
3427   Compile* C = _compile;
3428   ptadr = new (C->comp_arena()) ArraycopyNode(this, n, es);
3429   map_ideal_node(n, ptadr);
3430   // Add edge from arraycopy node to source object.
3431   (void)add_edge(ptadr, src);
3432   src->set_arraycopy_src();
3433   // Add edge from destination object to arraycopy node.
3434   (void)add_edge(dst, ptadr);
3435   dst->set_arraycopy_dst();
3436 }
3437 
3438 bool ConnectionGraph::is_oop_field(Node* n, int offset, bool* unsafe) {
3439   const Type* adr_type = n->as_AddP()->bottom_type();

3440   BasicType bt = T_INT;
3441   if (offset == Type::OffsetBot) {
3442     // Check only oop fields.
3443     if (!adr_type->isa_aryptr() ||
3444         adr_type->isa_aryptr()->elem() == Type::BOTTOM ||
3445         adr_type->isa_aryptr()->elem()->make_oopptr() != nullptr) {
3446       // OffsetBot is used to reference array's element. Ignore first AddP.
3447       if (find_second_addp(n, n->in(AddPNode::Base)) == nullptr) {
3448         bt = T_OBJECT;
3449       }
3450     }
3451   } else if (offset != oopDesc::klass_offset_in_bytes()) {
3452     if (adr_type->isa_instptr()) {
3453       ciField* field = _compile->alias_type(adr_type->isa_instptr())->field();
3454       if (field != nullptr) {
3455         bt = field->layout_type();
3456       } else {
3457         // Check for unsafe oop field access
3458         if (n->has_out_with(Op_StoreP, Op_LoadP, Op_StoreN, Op_LoadN) ||
3459             n->has_out_with(Op_GetAndSetP, Op_GetAndSetN, Op_CompareAndExchangeP, Op_CompareAndExchangeN) ||
3460             n->has_out_with(Op_CompareAndSwapP, Op_CompareAndSwapN, Op_WeakCompareAndSwapP, Op_WeakCompareAndSwapN) ||
3461             BarrierSet::barrier_set()->barrier_set_c2()->escape_has_out_with_unsafe_object(n)) {
3462           bt = T_OBJECT;
3463           (*unsafe) = true;
3464         }
3465       }
3466     } else if (adr_type->isa_aryptr()) {
3467       if (offset == arrayOopDesc::length_offset_in_bytes()) {
3468         // Ignore array length load.
3469       } else if (find_second_addp(n, n->in(AddPNode::Base)) != nullptr) {
3470         // Ignore first AddP.
3471       } else {
3472         const Type* elemtype = adr_type->isa_aryptr()->elem();
3473         bt = elemtype->array_element_basic_type();












3474       }
3475     } else if (adr_type->isa_rawptr() || adr_type->isa_klassptr()) {
3476       // Allocation initialization, ThreadLocal field access, unsafe access
3477       if (n->has_out_with(Op_StoreP, Op_LoadP, Op_StoreN, Op_LoadN) ||
3478           n->has_out_with(Op_GetAndSetP, Op_GetAndSetN, Op_CompareAndExchangeP, Op_CompareAndExchangeN) ||
3479           n->has_out_with(Op_CompareAndSwapP, Op_CompareAndSwapN, Op_WeakCompareAndSwapP, Op_WeakCompareAndSwapN) ||
3480           BarrierSet::barrier_set()->barrier_set_c2()->escape_has_out_with_unsafe_object(n)) {
3481         bt = T_OBJECT;
3482       }
3483     }
3484   }
3485   // Note: T_NARROWOOP is not classed as a real reference type
3486   return (is_reference_type(bt) || bt == T_NARROWOOP);
3487 }
3488 
3489 // Returns unique pointed java object or null.
3490 JavaObjectNode* ConnectionGraph::unique_java_object(Node *n) const {
3491   // If the node was created after the escape computation we can't answer.
3492   uint idx = n->_idx;
3493   if (idx >= nodes_size()) {

3650             return true;
3651           }
3652         }
3653       }
3654     }
3655   }
3656   return false;
3657 }
3658 
3659 int ConnectionGraph::address_offset(Node* adr, PhaseValues* phase) {
3660   const Type *adr_type = phase->type(adr);
3661   if (adr->is_AddP() && adr_type->isa_oopptr() == nullptr && is_captured_store_address(adr)) {
3662     // We are computing a raw address for a store captured by an Initialize
3663     // compute an appropriate address type. AddP cases #3 and #5 (see below).
3664     int offs = (int)phase->find_intptr_t_con(adr->in(AddPNode::Offset), Type::OffsetBot);
3665     assert(offs != Type::OffsetBot ||
3666            adr->in(AddPNode::Address)->in(0)->is_AllocateArray(),
3667            "offset must be a constant or it is initialization of array");
3668     return offs;
3669   }
3670   const TypePtr *t_ptr = adr_type->isa_ptr();
3671   assert(t_ptr != nullptr, "must be a pointer type");
3672   return t_ptr->offset();
3673 }
3674 
3675 Node* ConnectionGraph::get_addp_base(Node *addp) {
3676   assert(addp->is_AddP(), "must be AddP");
3677   //
3678   // AddP cases for Base and Address inputs:
3679   // case #1. Direct object's field reference:
3680   //     Allocate
3681   //       |
3682   //     Proj #5 ( oop result )
3683   //       |
3684   //     CheckCastPP (cast to instance type)
3685   //      | |
3686   //     AddP  ( base == address )
3687   //
3688   // case #2. Indirect object's field reference:
3689   //      Phi
3690   //       |
3691   //     CastPP (cast to instance type)
3692   //      | |

3806   }
3807   return nullptr;
3808 }
3809 
3810 //
3811 // Adjust the type and inputs of an AddP which computes the
3812 // address of a field of an instance
3813 //
3814 bool ConnectionGraph::split_AddP(Node *addp, Node *base) {
3815   PhaseGVN* igvn = _igvn;
3816   const TypeOopPtr *base_t = igvn->type(base)->isa_oopptr();
3817   assert(base_t != nullptr && base_t->is_known_instance(), "expecting instance oopptr");
3818   const TypeOopPtr *t = igvn->type(addp)->isa_oopptr();
3819   if (t == nullptr) {
3820     // We are computing a raw address for a store captured by an Initialize
3821     // compute an appropriate address type (cases #3 and #5).
3822     assert(igvn->type(addp) == TypeRawPtr::NOTNULL, "must be raw pointer");
3823     assert(addp->in(AddPNode::Address)->is_Proj(), "base of raw address must be result projection from allocation");
3824     intptr_t offs = (int)igvn->find_intptr_t_con(addp->in(AddPNode::Offset), Type::OffsetBot);
3825     assert(offs != Type::OffsetBot, "offset must be a constant");
3826     t = base_t->add_offset(offs)->is_oopptr();







3827   }
3828   int inst_id =  base_t->instance_id();
3829   assert(!t->is_known_instance() || t->instance_id() == inst_id,
3830                              "old type must be non-instance or match new type");
3831 
3832   // The type 't' could be subclass of 'base_t'.
3833   // As result t->offset() could be large then base_t's size and it will
3834   // cause the failure in add_offset() with narrow oops since TypeOopPtr()
3835   // constructor verifies correctness of the offset.
3836   //
3837   // It could happened on subclass's branch (from the type profiling
3838   // inlining) which was not eliminated during parsing since the exactness
3839   // of the allocation type was not propagated to the subclass type check.
3840   //
3841   // Or the type 't' could be not related to 'base_t' at all.
3842   // It could happened when CHA type is different from MDO type on a dead path
3843   // (for example, from instanceof check) which is not collapsed during parsing.
3844   //
3845   // Do nothing for such AddP node and don't process its users since
3846   // this code branch will go away.
3847   //
3848   if (!t->is_known_instance() &&
3849       !base_t->maybe_java_subtype_of(t)) {
3850      return false; // bail out
3851   }
3852   const TypeOopPtr *tinst = base_t->add_offset(t->offset())->is_oopptr();











3853   // Do NOT remove the next line: ensure a new alias index is allocated
3854   // for the instance type. Note: C++ will not remove it since the call
3855   // has side effect.
3856   int alias_idx = _compile->get_alias_index(tinst);
3857   igvn->set_type(addp, tinst);
3858   // record the allocation in the node map
3859   set_map(addp, get_map(base->_idx));
3860   // Set addp's Base and Address to 'base'.
3861   Node *abase = addp->in(AddPNode::Base);
3862   Node *adr   = addp->in(AddPNode::Address);
3863   if (adr->is_Proj() && adr->in(0)->is_Allocate() &&
3864       adr->in(0)->_idx == (uint)inst_id) {
3865     // Skip AddP cases #3 and #5.
3866   } else {
3867     assert(!abase->is_top(), "sanity"); // AddP case #3
3868     if (abase != base) {
3869       igvn->hash_delete(addp);
3870       addp->set_req(AddPNode::Base, base);
3871       if (abase == adr) {
3872         addp->set_req(AddPNode::Address, base);

4538         ptnode_adr(n->_idx)->dump();
4539         assert(jobj != nullptr && jobj != phantom_obj, "escaped allocation");
4540 #endif
4541         _compile->record_failure(_invocation > 0 ? C2Compiler::retry_no_iterative_escape_analysis() : C2Compiler::retry_no_escape_analysis());
4542         return;
4543       } else {
4544         Node *val = get_map(jobj->idx());   // CheckCastPP node
4545         TypeNode *tn = n->as_Type();
4546         const TypeOopPtr* tinst = igvn->type(val)->isa_oopptr();
4547         assert(tinst != nullptr && tinst->is_known_instance() &&
4548                tinst->instance_id() == jobj->idx() , "instance type expected.");
4549 
4550         const Type *tn_type = igvn->type(tn);
4551         const TypeOopPtr *tn_t;
4552         if (tn_type->isa_narrowoop()) {
4553           tn_t = tn_type->make_ptr()->isa_oopptr();
4554         } else {
4555           tn_t = tn_type->isa_oopptr();
4556         }
4557         if (tn_t != nullptr && tinst->maybe_java_subtype_of(tn_t)) {







4558           if (tn_type->isa_narrowoop()) {
4559             tn_type = tinst->make_narrowoop();
4560           } else {
4561             tn_type = tinst;
4562           }
4563           igvn->hash_delete(tn);
4564           igvn->set_type(tn, tn_type);
4565           tn->set_type(tn_type);
4566           igvn->hash_insert(tn);
4567           record_for_optimizer(n);
4568         } else {
4569           assert(tn_type == TypePtr::NULL_PTR ||
4570                  (tn_t != nullptr && !tinst->maybe_java_subtype_of(tn_t)),
4571                  "unexpected type");
4572           continue; // Skip dead path with different type
4573         }
4574       }
4575     } else {
4576       DEBUG_ONLY(n->dump();)
4577       assert(false, "EA: unexpected node");
4578       continue;
4579     }
4580     // push allocation's users on appropriate worklist
4581     for (DUIterator_Fast imax, i = n->fast_outs(imax); i < imax; i++) {
4582       Node *use = n->fast_out(i);
4583       if(use->is_Mem() && use->in(MemNode::Address) == n) {
4584         // Load/store to instance's field
4585         memnode_worklist.append_if_missing(use);
4586       } else if (use->is_MemBar()) {
4587         if (use->in(TypeFunc::Memory) == n) { // Ignore precedent edge
4588           memnode_worklist.append_if_missing(use);
4589         }
4590       } else if (use->is_AddP() && use->outcnt() > 0) { // No dead nodes
4591         Node* addp2 = find_second_addp(use, n);
4592         if (addp2 != nullptr) {
4593           alloc_worklist.append_if_missing(addp2);
4594         }
4595         alloc_worklist.append_if_missing(use);
4596       } else if (use->is_Phi() ||
4597                  use->is_CheckCastPP() ||
4598                  use->is_EncodeNarrowPtr() ||
4599                  use->is_DecodeNarrowPtr() ||
4600                  (use->is_ConstraintCast() && use->Opcode() == Op_CastPP)) {
4601         alloc_worklist.append_if_missing(use);
4602 #ifdef ASSERT
4603       } else if (use->is_Mem()) {
4604         assert(use->in(MemNode::Address) != n, "EA: missing allocation reference path");
4605       } else if (use->is_MergeMem()) {
4606         assert(mergemem_worklist.contains(use->as_MergeMem()), "EA: missing MergeMem node in the worklist");
4607       } else if (use->is_SafePoint()) {
4608         // Look for MergeMem nodes for calls which reference unique allocation
4609         // (through CheckCastPP nodes) even for debug info.
4610         Node* m = use->in(TypeFunc::Memory);
4611         if (m->is_MergeMem()) {
4612           assert(mergemem_worklist.contains(m->as_MergeMem()), "EA: missing MergeMem node in the worklist");
4613         }
4614       } else if (use->Opcode() == Op_EncodeISOArray) {
4615         if (use->in(MemNode::Memory) == n || use->in(3) == n) {
4616           // EncodeISOArray overwrites destination array
4617           memnode_worklist.append_if_missing(use);
4618         }



4619       } else {
4620         uint op = use->Opcode();
4621         if ((op == Op_StrCompressedCopy || op == Op_StrInflatedCopy) &&
4622             (use->in(MemNode::Memory) == n)) {
4623           // They overwrite memory edge corresponding to destination array,
4624           memnode_worklist.append_if_missing(use);
4625         } else if (!(op == Op_CmpP || op == Op_Conv2B ||
4626               op == Op_CastP2X ||
4627               op == Op_FastLock || op == Op_AryEq ||
4628               op == Op_StrComp || op == Op_CountPositives ||
4629               op == Op_StrCompressedCopy || op == Op_StrInflatedCopy ||
4630               op == Op_StrEquals || op == Op_VectorizedHashCode ||
4631               op == Op_StrIndexOf || op == Op_StrIndexOfChar ||
4632               op == Op_SubTypeCheck ||
4633               op == Op_ReinterpretS2HF ||
4634               BarrierSet::barrier_set()->barrier_set_c2()->is_gc_barrier_node(use))) {
4635           n->dump();
4636           use->dump();
4637           assert(false, "EA: missing allocation reference path");
4638         }
4639 #endif
4640       }
4641     }
4642 
4643   }
4644 
4645 #ifdef ASSERT
4646   if (VerifyReduceAllocationMerges) {
4647     for (uint i = 0; i < reducible_merges.size(); i++) {
4648       Node* phi = reducible_merges.at(i);
4649 
4650       if (!reduced_merges.member(phi)) {
4651         phi->dump(2);
4652         phi->dump(-2);

4716       // we don't need to do anything, but the users must be pushed
4717       n = n->as_MemBar()->proj_out_or_null(TypeFunc::Memory);
4718       if (n == nullptr) {
4719         continue;
4720       }
4721     } else if (n->is_CallLeaf()) {
4722       // Runtime calls with narrow memory input (no MergeMem node)
4723       // get the memory projection
4724       n = n->as_Call()->proj_out_or_null(TypeFunc::Memory);
4725       if (n == nullptr) {
4726         continue;
4727       }
4728     } else if (n->Opcode() == Op_StrInflatedCopy) {
4729       // Check direct uses of StrInflatedCopy.
4730       // It is memory type Node - no special SCMemProj node.
4731     } else if (n->Opcode() == Op_StrCompressedCopy ||
4732                n->Opcode() == Op_EncodeISOArray) {
4733       // get the memory projection
4734       n = n->find_out_with(Op_SCMemProj);
4735       assert(n != nullptr && n->Opcode() == Op_SCMemProj, "memory projection required");



4736     } else {
4737 #ifdef ASSERT
4738       if (!n->is_Mem()) {
4739         n->dump();
4740       }
4741       assert(n->is_Mem(), "memory node required.");
4742 #endif
4743       Node *addr = n->in(MemNode::Address);
4744       const Type *addr_t = igvn->type(addr);
4745       if (addr_t == Type::TOP) {
4746         continue;
4747       }
4748       assert (addr_t->isa_ptr() != nullptr, "pointer type required.");
4749       int alias_idx = _compile->get_alias_index(addr_t->is_ptr());
4750       assert ((uint)alias_idx < new_index_end, "wrong alias index");
4751       Node *mem = find_inst_mem(n->in(MemNode::Memory), alias_idx, orig_phis);
4752       if (_compile->failing()) {
4753         return;
4754       }
4755       if (mem != n->in(MemNode::Memory)) {

4760       if (n->is_Load()) {
4761         continue;  // don't push users
4762       } else if (n->is_LoadStore()) {
4763         // get the memory projection
4764         n = n->find_out_with(Op_SCMemProj);
4765         assert(n != nullptr && n->Opcode() == Op_SCMemProj, "memory projection required");
4766       }
4767     }
4768     // push user on appropriate worklist
4769     for (DUIterator_Fast imax, i = n->fast_outs(imax); i < imax; i++) {
4770       Node *use = n->fast_out(i);
4771       if (use->is_Phi() || use->is_ClearArray()) {
4772         memnode_worklist.append_if_missing(use);
4773       } else if (use->is_Mem() && use->in(MemNode::Memory) == n) {
4774         memnode_worklist.append_if_missing(use);
4775       } else if (use->is_MemBar() || use->is_CallLeaf()) {
4776         if (use->in(TypeFunc::Memory) == n) { // Ignore precedent edge
4777           memnode_worklist.append_if_missing(use);
4778         }
4779 #ifdef ASSERT
4780       } else if(use->is_Mem()) {
4781         assert(use->in(MemNode::Memory) != n, "EA: missing memory path");
4782       } else if (use->is_MergeMem()) {
4783         assert(mergemem_worklist.contains(use->as_MergeMem()), "EA: missing MergeMem node in the worklist");
4784       } else if (use->Opcode() == Op_EncodeISOArray) {
4785         if (use->in(MemNode::Memory) == n || use->in(3) == n) {
4786           // EncodeISOArray overwrites destination array
4787           memnode_worklist.append_if_missing(use);
4788         }




4789       } else {
4790         uint op = use->Opcode();
4791         if ((use->in(MemNode::Memory) == n) &&
4792             (op == Op_StrCompressedCopy || op == Op_StrInflatedCopy)) {
4793           // They overwrite memory edge corresponding to destination array,
4794           memnode_worklist.append_if_missing(use);
4795         } else if (!(BarrierSet::barrier_set()->barrier_set_c2()->is_gc_barrier_node(use) ||
4796               op == Op_AryEq || op == Op_StrComp || op == Op_CountPositives ||
4797               op == Op_StrCompressedCopy || op == Op_StrInflatedCopy || op == Op_VectorizedHashCode ||
4798               op == Op_StrEquals || op == Op_StrIndexOf || op == Op_StrIndexOfChar)) {
4799           n->dump();
4800           use->dump();
4801           assert(false, "EA: missing memory path");
4802         }
4803 #endif
4804       }
4805     }
4806   }
4807 
4808   //  Phase 3:  Process MergeMem nodes from mergemem_worklist.
4809   //            Walk each memory slice moving the first node encountered of each
4810   //            instance type to the input corresponding to its alias index.
4811   uint length = mergemem_worklist.length();
4812   for( uint next = 0; next < length; ++next ) {
4813     MergeMemNode* nmm = mergemem_worklist.at(next);
4814     assert(!visited.test_set(nmm->_idx), "should not be visited before");
4815     // Note: we don't want to use MergeMemStream here because we only want to
4816     // scan inputs which exist at the start, not ones we add during processing.
4817     // Note 2: MergeMem may already contains instance memory slices added
4818     // during find_inst_mem() call when memory nodes were processed above.

4879     if (_compile->live_nodes() >= _compile->max_node_limit() * 0.75) {
4880       if (_compile->do_reduce_allocation_merges()) {
4881         _compile->record_failure(C2Compiler::retry_no_reduce_allocation_merges());
4882       } else if (_invocation > 0) {
4883         _compile->record_failure(C2Compiler::retry_no_iterative_escape_analysis());
4884       } else {
4885         _compile->record_failure(C2Compiler::retry_no_escape_analysis());
4886       }
4887       return;
4888     }
4889 
4890     igvn->hash_insert(nmm);
4891     record_for_optimizer(nmm);
4892   }
4893 
4894   //  Phase 4:  Update the inputs of non-instance memory Phis and
4895   //            the Memory input of memnodes
4896   // First update the inputs of any non-instance Phi's from
4897   // which we split out an instance Phi.  Note we don't have
4898   // to recursively process Phi's encountered on the input memory
4899   // chains as is done in split_memory_phi() since they  will
4900   // also be processed here.
4901   for (int j = 0; j < orig_phis.length(); j++) {
4902     PhiNode *phi = orig_phis.at(j);
4903     int alias_idx = _compile->get_alias_index(phi->adr_type());
4904     igvn->hash_delete(phi);
4905     for (uint i = 1; i < phi->req(); i++) {
4906       Node *mem = phi->in(i);
4907       Node *new_mem = find_inst_mem(mem, alias_idx, orig_phis);
4908       if (_compile->failing()) {
4909         return;
4910       }
4911       if (mem != new_mem) {
4912         phi->set_req(i, new_mem);
4913       }
4914     }
4915     igvn->hash_insert(phi);
4916     record_for_optimizer(phi);
4917   }
4918 
4919   // Update the memory inputs of MemNodes with the value we computed

  11  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  12  * version 2 for more details (a copy is included in the LICENSE file that
  13  * accompanied this code).
  14  *
  15  * You should have received a copy of the GNU General Public License version
  16  * 2 along with this work; if not, write to the Free Software Foundation,
  17  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
  18  *
  19  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
  20  * or visit www.oracle.com if you need additional information or have any
  21  * questions.
  22  *
  23  */
  24 
  25 #include "ci/bcEscapeAnalyzer.hpp"
  26 #include "compiler/compileLog.hpp"
  27 #include "gc/shared/barrierSet.hpp"
  28 #include "gc/shared/c2/barrierSetC2.hpp"
  29 #include "libadt/vectset.hpp"
  30 #include "memory/allocation.hpp"
  31 #include "memory/metaspace.hpp"
  32 #include "memory/resourceArea.hpp"
  33 #include "opto/arraycopynode.hpp"
  34 #include "opto/c2compiler.hpp"
  35 #include "opto/callnode.hpp"
  36 #include "opto/castnode.hpp"
  37 #include "opto/cfgnode.hpp"
  38 #include "opto/compile.hpp"
  39 #include "opto/escape.hpp"
  40 #include "opto/inlinetypenode.hpp"
  41 #include "opto/locknode.hpp"
  42 #include "opto/macro.hpp"
  43 #include "opto/movenode.hpp"
  44 #include "opto/narrowptrnode.hpp"
  45 #include "opto/phaseX.hpp"
  46 #include "opto/rootnode.hpp"
  47 #include "utilities/macros.hpp"
  48 
  49 ConnectionGraph::ConnectionGraph(Compile * C, PhaseIterGVN *igvn, int invocation) :
  50   // If ReduceAllocationMerges is enabled we might call split_through_phi during
  51   // split_unique_types and that will create additional nodes that need to be
  52   // pushed to the ConnectionGraph. The code below bumps the initial capacity of
  53   // _nodes by 10% to account for these additional nodes. If capacity is exceeded
  54   // the array will be reallocated.
  55   _nodes(C->comp_arena(), C->do_reduce_allocation_merges() ? C->unique()*1.10 : C->unique(), C->unique(), nullptr),
  56   _in_worklist(C->comp_arena()),
  57   _next_pidx(0),
  58   _collecting(true),
  59   _verify(false),
  60   _compile(C),

 148   GrowableArray<SafePointNode*>  sfn_worklist;
 149   GrowableArray<MergeMemNode*>   mergemem_worklist;
 150   DEBUG_ONLY( GrowableArray<Node*> addp_worklist; )
 151 
 152   { Compile::TracePhase tp(Phase::_t_connectionGraph);
 153 
 154   // 1. Populate Connection Graph (CG) with PointsTo nodes.
 155   ideal_nodes.map(C->live_nodes(), nullptr);  // preallocate space
 156   // Initialize worklist
 157   if (C->root() != nullptr) {
 158     ideal_nodes.push(C->root());
 159   }
 160   // Processed ideal nodes are unique on ideal_nodes list
 161   // but several ideal nodes are mapped to the phantom_obj.
 162   // To avoid duplicated entries on the following worklists
 163   // add the phantom_obj only once to them.
 164   ptnodes_worklist.append(phantom_obj);
 165   java_objects_worklist.append(phantom_obj);
 166   for( uint next = 0; next < ideal_nodes.size(); ++next ) {
 167     Node* n = ideal_nodes.at(next);
 168     if ((n->Opcode() == Op_LoadX || n->Opcode() == Op_StoreX) &&
 169         !n->in(MemNode::Address)->is_AddP() &&
 170         _igvn->type(n->in(MemNode::Address))->isa_oopptr()) {
 171       // Load/Store at mark work address is at offset 0 so has no AddP which confuses EA
 172       Node* addp = new AddPNode(n->in(MemNode::Address), n->in(MemNode::Address), _igvn->MakeConX(0));
 173       _igvn->register_new_node_with_optimizer(addp);
 174       _igvn->replace_input_of(n, MemNode::Address, addp);
 175       ideal_nodes.push(addp);
 176       _nodes.at_put_grow(addp->_idx, nullptr, nullptr);
 177     }
 178     // Create PointsTo nodes and add them to Connection Graph. Called
 179     // only once per ideal node since ideal_nodes is Unique_Node list.
 180     add_node_to_connection_graph(n, &delayed_worklist);
 181     PointsToNode* ptn = ptnode_adr(n->_idx);
 182     if (ptn != nullptr && ptn != phantom_obj) {
 183       ptnodes_worklist.append(ptn);
 184       if (ptn->is_JavaObject()) {
 185         java_objects_worklist.append(ptn->as_JavaObject());
 186         if ((n->is_Allocate() || n->is_CallStaticJava()) &&
 187             (ptn->escape_state() < PointsToNode::GlobalEscape)) {
 188           // Only allocations and java static calls results are interesting.
 189           non_escaped_allocs_worklist.append(ptn->as_JavaObject());
 190         }
 191       } else if (ptn->is_Field() && ptn->as_Field()->is_oop()) {
 192         oop_fields_worklist.append(ptn->as_Field());
 193       }
 194     }
 195     // Collect some interesting nodes for further use.
 196     switch (n->Opcode()) {
 197       case Op_MergeMem:

 408     split_unique_types(alloc_worklist, arraycopy_worklist, mergemem_worklist, reducible_merges);
 409     if (C->failing()) {
 410       NOT_PRODUCT(escape_state_statistics(java_objects_worklist);)
 411       return false;
 412     }
 413     C->print_method(PHASE_AFTER_EA, 2);
 414 
 415 #ifdef ASSERT
 416   } else if (Verbose && (PrintEscapeAnalysis || PrintEliminateAllocations)) {
 417     tty->print("=== No allocations eliminated for ");
 418     C->method()->print_short_name();
 419     if (!EliminateAllocations) {
 420       tty->print(" since EliminateAllocations is off ===");
 421     } else if(!has_scalar_replaceable_candidates) {
 422       tty->print(" since there are no scalar replaceable candidates ===");
 423     }
 424     tty->cr();
 425 #endif
 426   }
 427 
 428   // 6. Expand flat accesses if the object does not escape. This adds nodes to
 429   // the graph, so it has to be after split_unique_types. This expands atomic
 430   // mismatched accesses (though encapsulated in LoadFlats and StoreFlats) into
 431   // non-mismatched accesses, so it is better before reduce allocation merges.
 432   if (has_non_escaping_obj) {
 433     optimize_flat_accesses(sfn_worklist);
 434   }
 435 
 436   // 7. Reduce allocation merges used as debug information. This is done after
 437   // split_unique_types because the methods used to create SafePointScalarObject
 438   // need to traverse the memory graph to find values for object fields. We also
 439   // set to null the scalarized inputs of reducible Phis so that the Allocate
 440   // that they point can be later scalar replaced.
 441   bool delay = _igvn->delay_transform();
 442   _igvn->set_delay_transform(true);
 443   for (uint i = 0; i < reducible_merges.size(); i++) {
 444     Node* n = reducible_merges.at(i);
 445     if (n->outcnt() > 0) {
 446       if (!reduce_phi_on_safepoints(n->as_Phi())) {
 447         NOT_PRODUCT(escape_state_statistics(java_objects_worklist);)
 448         C->record_failure(C2Compiler::retry_no_reduce_allocation_merges());
 449         return false;
 450       }
 451 
 452       // Now we set the scalar replaceable inputs of ophi to null, which is
 453       // the last piece that would prevent it from being scalar replaceable.
 454       reset_scalar_replaceable_entries(n->as_Phi());
 455     }
 456   }

1255 
1256     // The next two inputs are:
1257     //  (1) A copy of the original pointer to NSR objects.
1258     //  (2) A selector, used to decide if we need to rematerialize an object
1259     //      or use the pointer to a NSR object.
1260     // See more details of these fields in the declaration of SafePointScalarMergeNode
1261     sfpt->add_req(nsr_merge_pointer);
1262     sfpt->add_req(selector);
1263 
1264     for (uint i = 1; i < ophi->req(); i++) {
1265       Node* base = ophi->in(i);
1266       JavaObjectNode* ptn = unique_java_object(base);
1267 
1268       // If the base is not scalar replaceable we don't need to register information about
1269       // it at this time.
1270       if (ptn == nullptr || !ptn->scalar_replaceable()) {
1271         continue;
1272       }
1273 
1274       AllocateNode* alloc = ptn->ideal_node()->as_Allocate();
1275       Unique_Node_List value_worklist;
1276 #ifdef ASSERT
1277       const Type* res_type = alloc->result_cast()->bottom_type();
1278       if (res_type->is_inlinetypeptr() && !Compile::current()->has_circular_inline_type()) {
1279         PhiNode* phi = ophi->as_Phi();
1280         assert(!ophi->as_Phi()->can_push_inline_types_down(_igvn), "missed earlier scalarization opportunity");
1281       }
1282 #endif
1283       SafePointScalarObjectNode* sobj = mexp.create_scalarized_object_description(alloc, sfpt, &value_worklist);
1284       if (sobj == nullptr) {
1285         _compile->record_failure(C2Compiler::retry_no_reduce_allocation_merges());
1286         return false;
1287       }
1288 
1289       // Now make a pass over the debug information replacing any references
1290       // to the allocated object with "sobj"
1291       Node* ccpp = alloc->result_cast();
1292       sfpt->replace_edges_in_range(ccpp, sobj, debug_start, jvms->debug_end(), _igvn);
1293 
1294       // Register the scalarized object as a candidate for reallocation
1295       smerge->add_req(sobj);
1296 
1297       // Scalarize inline types that were added to the safepoint.
1298       // Don't allow linking a constant oop (if available) for flat array elements
1299       // because Deoptimization::reassign_flat_array_elements needs field values.
1300       const bool allow_oop = !merge_t->is_flat();
1301       for (uint j = 0; j < value_worklist.size(); ++j) {
1302         InlineTypeNode* vt = value_worklist.at(j)->as_InlineType();
1303         vt->make_scalar_in_safepoints(_igvn, allow_oop);
1304       }
1305     }
1306 
1307     // Replaces debug information references to "original_sfpt_parent" in "sfpt" with references to "smerge"
1308     sfpt->replace_edges_in_range(original_sfpt_parent, smerge, debug_start, jvms->debug_end(), _igvn);
1309 
1310     // The call to 'replace_edges_in_range' above might have removed the
1311     // reference to ophi that we need at _merge_pointer_idx. The line below make
1312     // sure the reference is maintained.
1313     sfpt->set_req(smerge->merge_pointer_idx(jvms), nsr_merge_pointer);
1314     _igvn->_worklist.push(sfpt);
1315   }
1316 
1317   return true;
1318 }
1319 
1320 void ConnectionGraph::reduce_phi(PhiNode* ophi, GrowableArray<Node *>  &alloc_worklist, GrowableArray<Node *>  &memnode_worklist) {
1321   bool delay = _igvn->delay_transform();
1322   _igvn->set_delay_transform(true);
1323   _igvn->hash_delete(ophi);
1324 

1483   return false;
1484 }
1485 
1486 // Returns true if at least one of the arguments to the call is an object
1487 // that does not escape globally.
1488 bool ConnectionGraph::has_arg_escape(CallJavaNode* call) {
1489   if (call->method() != nullptr) {
1490     uint max_idx = TypeFunc::Parms + call->method()->arg_size();
1491     for (uint idx = TypeFunc::Parms; idx < max_idx; idx++) {
1492       Node* p = call->in(idx);
1493       if (not_global_escape(p)) {
1494         return true;
1495       }
1496     }
1497   } else {
1498     const char* name = call->as_CallStaticJava()->_name;
1499     assert(name != nullptr, "no name");
1500     // no arg escapes through uncommon traps
1501     if (strcmp(name, "uncommon_trap") != 0) {
1502       // process_call_arguments() assumes that all arguments escape globally
1503       const TypeTuple* d = call->tf()->domain_sig();
1504       for (uint i = TypeFunc::Parms; i < d->cnt(); i++) {
1505         const Type* at = d->field_at(i);
1506         if (at->isa_oopptr() != nullptr) {
1507           return true;
1508         }
1509       }
1510     }
1511   }
1512   return false;
1513 }
1514 
1515 
1516 
1517 // Utility function for nodes that load an object
1518 void ConnectionGraph::add_objload_to_connection_graph(Node *n, Unique_Node_List *delayed_worklist) {
1519   // Using isa_ptr() instead of isa_oopptr() for LoadP and Phi because
1520   // ThreadLocal has RawPtr type.
1521   const Type* t = _igvn->type(n);
1522   if (t->make_ptr() != nullptr) {
1523     Node* adr = n->in(MemNode::Address);

1557       // first IGVN optimization when escape information is still available.
1558       record_for_optimizer(n);
1559     } else if (n->is_Allocate()) {
1560       add_call_node(n->as_Call());
1561       record_for_optimizer(n);
1562     } else {
1563       if (n->is_CallStaticJava()) {
1564         const char* name = n->as_CallStaticJava()->_name;
1565         if (name != nullptr && strcmp(name, "uncommon_trap") == 0) {
1566           return; // Skip uncommon traps
1567         }
1568       }
1569       // Don't mark as processed since call's arguments have to be processed.
1570       delayed_worklist->push(n);
1571       // Check if a call returns an object.
1572       if ((n->as_Call()->returns_pointer() &&
1573            n->as_Call()->proj_out_or_null(TypeFunc::Parms) != nullptr) ||
1574           (n->is_CallStaticJava() &&
1575            n->as_CallStaticJava()->is_boxing_method())) {
1576         add_call_node(n->as_Call());
1577       } else if (n->as_Call()->tf()->returns_inline_type_as_fields()) {
1578         bool returns_oop = false;
1579         for (DUIterator_Fast imax, i = n->fast_outs(imax); i < imax && !returns_oop; i++) {
1580           ProjNode* pn = n->fast_out(i)->as_Proj();
1581           if (pn->_con >= TypeFunc::Parms && pn->bottom_type()->isa_ptr()) {
1582             returns_oop = true;
1583           }
1584         }
1585         if (returns_oop) {
1586           add_call_node(n->as_Call());
1587         }
1588       }
1589     }
1590     return;
1591   }
1592   // Put this check here to process call arguments since some call nodes
1593   // point to phantom_obj.
1594   if (n_ptn == phantom_obj || n_ptn == null_obj) {
1595     return; // Skip predefined nodes.
1596   }
1597   switch (opcode) {
1598     case Op_AddP: {
1599       Node* base = get_addp_base(n);
1600       PointsToNode* ptn_base = ptnode_adr(base->_idx);
1601       // Field nodes are created for all field types. They are used in
1602       // adjust_scalar_replaceable_state() and split_unique_types().
1603       // Note, non-oop fields will have only base edges in Connection
1604       // Graph because such fields are not used for oop loads and stores.
1605       int offset = address_offset(n, igvn);
1606       add_field(n, PointsToNode::NoEscape, offset);
1607       if (ptn_base == nullptr) {
1608         delayed_worklist->push(n); // Process it later.
1609       } else {
1610         n_ptn = ptnode_adr(n_idx);
1611         add_base(n_ptn->as_Field(), ptn_base);
1612       }
1613       break;
1614     }
1615     case Op_CastX2P:
1616     case Op_CastI2N: {
1617       map_ideal_node(n, phantom_obj);
1618       break;
1619     }
1620     case Op_InlineType:
1621     case Op_CastPP:
1622     case Op_CheckCastPP:
1623     case Op_EncodeP:
1624     case Op_DecodeN:
1625     case Op_EncodePKlass:
1626     case Op_DecodeNKlass: {
1627       add_local_var_and_edge(n, PointsToNode::NoEscape, n->in(1), delayed_worklist);
1628       break;
1629     }
1630     case Op_CMoveP: {
1631       add_local_var(n, PointsToNode::NoEscape);
1632       // Do not add edges during first iteration because some could be
1633       // not defined yet.
1634       delayed_worklist->push(n);
1635       break;
1636     }
1637     case Op_ConP:
1638     case Op_ConN:
1639     case Op_ConNKlass: {
1640       // assume all oop constants globally escape except for null

1670       break;
1671     }
1672     case Op_PartialSubtypeCheck: {
1673       // Produces Null or notNull and is used in only in CmpP so
1674       // phantom_obj could be used.
1675       map_ideal_node(n, phantom_obj); // Result is unknown
1676       break;
1677     }
1678     case Op_Phi: {
1679       // Using isa_ptr() instead of isa_oopptr() for LoadP and Phi because
1680       // ThreadLocal has RawPtr type.
1681       const Type* t = n->as_Phi()->type();
1682       if (t->make_ptr() != nullptr) {
1683         add_local_var(n, PointsToNode::NoEscape);
1684         // Do not add edges during first iteration because some could be
1685         // not defined yet.
1686         delayed_worklist->push(n);
1687       }
1688       break;
1689     }
1690     case Op_LoadFlat:
1691       // Treat LoadFlat similar to an unknown call that receives nothing and produces its results
1692       map_ideal_node(n, phantom_obj);
1693       break;
1694     case Op_StoreFlat:
1695       // Treat StoreFlat similar to a call that escapes the stored flattened fields
1696       delayed_worklist->push(n);
1697       break;
1698     case Op_Proj: {
1699       // we are only interested in the oop result projection from a call
1700       if (n->as_Proj()->_con >= TypeFunc::Parms && n->in(0)->is_Call() &&
1701           (n->in(0)->as_Call()->returns_pointer() || n->bottom_type()->isa_ptr())) {
1702         assert((n->as_Proj()->_con == TypeFunc::Parms && n->in(0)->as_Call()->returns_pointer()) ||
1703                n->in(0)->as_Call()->tf()->returns_inline_type_as_fields(), "what kind of oop return is it?");
1704         add_local_var_and_edge(n, PointsToNode::NoEscape, n->in(0), delayed_worklist);
1705       } else if (n->as_Proj()->_con >= TypeFunc::Parms && n->in(0)->is_LoadFlat() && igvn->type(n)->isa_ptr()) {
1706         // Treat LoadFlat outputs similar to a call return value
1707         add_local_var_and_edge(n, PointsToNode::NoEscape, n->in(0), delayed_worklist);
1708       }
1709       break;
1710     }
1711     case Op_Rethrow: // Exception object escapes
1712     case Op_Return: {
1713       if (n->req() > TypeFunc::Parms &&
1714           igvn->type(n->in(TypeFunc::Parms))->isa_oopptr()) {
1715         // Treat Return value as LocalVar with GlobalEscape escape state.
1716         add_local_var_and_edge(n, PointsToNode::GlobalEscape, n->in(TypeFunc::Parms), delayed_worklist);
1717       }
1718       break;
1719     }
1720     case Op_CompareAndExchangeP:
1721     case Op_CompareAndExchangeN:
1722     case Op_GetAndSetP:
1723     case Op_GetAndSetN: {
1724       add_objload_to_connection_graph(n, delayed_worklist);
1725       // fall-through
1726     }

1772       break;
1773     }
1774     default:
1775       ; // Do nothing for nodes not related to EA.
1776   }
1777   return;
1778 }
1779 
1780 // Add final simple edges to graph.
1781 void ConnectionGraph::add_final_edges(Node *n) {
1782   PointsToNode* n_ptn = ptnode_adr(n->_idx);
1783 #ifdef ASSERT
1784   if (_verify && n_ptn->is_JavaObject())
1785     return; // This method does not change graph for JavaObject.
1786 #endif
1787 
1788   if (n->is_Call()) {
1789     process_call_arguments(n->as_Call());
1790     return;
1791   }
1792   assert(n->is_Store() || n->is_LoadStore() || n->is_StoreFlat() ||
1793          ((n_ptn != nullptr) && (n_ptn->ideal_node() != nullptr)),
1794          "node should be registered already");
1795   int opcode = n->Opcode();
1796   bool gc_handled = BarrierSet::barrier_set()->barrier_set_c2()->escape_add_final_edges(this, _igvn, n, opcode);
1797   if (gc_handled) {
1798     return; // Ignore node if already handled by GC.
1799   }
1800   switch (opcode) {
1801     case Op_AddP: {
1802       Node* base = get_addp_base(n);
1803       PointsToNode* ptn_base = ptnode_adr(base->_idx);
1804       assert(ptn_base != nullptr, "field's base should be registered");
1805       add_base(n_ptn->as_Field(), ptn_base);
1806       break;
1807     }
1808     case Op_InlineType:
1809     case Op_CastPP:
1810     case Op_CheckCastPP:
1811     case Op_EncodeP:
1812     case Op_DecodeN:
1813     case Op_EncodePKlass:
1814     case Op_DecodeNKlass: {
1815       add_local_var_and_edge(n, PointsToNode::NoEscape, n->in(1), nullptr);
1816       break;
1817     }
1818     case Op_CMoveP: {
1819       for (uint i = CMoveNode::IfFalse; i < n->req(); i++) {
1820         Node* in = n->in(i);
1821         if (in == nullptr) {
1822           continue;  // ignore null
1823         }
1824         Node* uncast_in = in->uncast();
1825         if (uncast_in->is_top() || uncast_in == n) {
1826           continue;  // ignore top or inputs which go back this node
1827         }
1828         PointsToNode* ptn = ptnode_adr(in->_idx);

1841     }
1842     case Op_Phi: {
1843       // Using isa_ptr() instead of isa_oopptr() for LoadP and Phi because
1844       // ThreadLocal has RawPtr type.
1845       assert(n->as_Phi()->type()->make_ptr() != nullptr, "Unexpected node type");
1846       for (uint i = 1; i < n->req(); i++) {
1847         Node* in = n->in(i);
1848         if (in == nullptr) {
1849           continue;  // ignore null
1850         }
1851         Node* uncast_in = in->uncast();
1852         if (uncast_in->is_top() || uncast_in == n) {
1853           continue;  // ignore top or inputs which go back this node
1854         }
1855         PointsToNode* ptn = ptnode_adr(in->_idx);
1856         assert(ptn != nullptr, "node should be registered");
1857         add_edge(n_ptn, ptn);
1858       }
1859       break;
1860     }
1861     case Op_StoreFlat: {
1862       // StoreFlat globally escapes its stored flattened fields
1863       InlineTypeNode* value = n->as_StoreFlat()->value();
1864       ciInlineKlass* vk = _igvn->type(value)->inline_klass();
1865       for (int i = 0; i < vk->nof_nonstatic_fields(); i++) {
1866         ciField* field = vk->nonstatic_field_at(i);
1867         if (field->type()->is_primitive_type()) {
1868           continue;
1869         }
1870 
1871         Node* field_value = value->field_value_by_offset(field->offset_in_bytes(), true);
1872         PointsToNode* field_value_ptn = ptnode_adr(field_value->_idx);
1873         set_escape_state(field_value_ptn, PointsToNode::GlobalEscape NOT_PRODUCT(COMMA "store into a flat field"));
1874       }
1875       break;
1876     }
1877     case Op_Proj: {
1878       if (n->in(0)->is_Call()) {
1879         // we are only interested in the oop result projection from a call
1880         assert((n->as_Proj()->_con == TypeFunc::Parms && n->in(0)->as_Call()->returns_pointer()) ||
1881               n->in(0)->as_Call()->tf()->returns_inline_type_as_fields(), "what kind of oop return is it?");
1882         add_local_var_and_edge(n, PointsToNode::NoEscape, n->in(0), nullptr);
1883       } else if (n->in(0)->is_LoadFlat()) {
1884         // Treat LoadFlat outputs similar to a call return value
1885         add_local_var_and_edge(n, PointsToNode::NoEscape, n->in(0), nullptr);
1886       }
1887       break;
1888     }
1889     case Op_Rethrow: // Exception object escapes
1890     case Op_Return: {
1891       assert(n->req() > TypeFunc::Parms && _igvn->type(n->in(TypeFunc::Parms))->isa_oopptr(),
1892              "Unexpected node type");
1893       // Treat Return value as LocalVar with GlobalEscape escape state.
1894       add_local_var_and_edge(n, PointsToNode::GlobalEscape, n->in(TypeFunc::Parms), nullptr);
1895       break;
1896     }
1897     case Op_CompareAndExchangeP:
1898     case Op_CompareAndExchangeN:
1899     case Op_GetAndSetP:
1900     case Op_GetAndSetN:{
1901       assert(_igvn->type(n)->make_ptr() != nullptr, "Unexpected node type");
1902       add_local_var_and_edge(n, PointsToNode::NoEscape, n->in(MemNode::Address), nullptr);
1903       // fall-through
1904     }
1905     case Op_CompareAndSwapP:
1906     case Op_CompareAndSwapN:

2041     PointsToNode* ptn = ptnode_adr(val->_idx);
2042     assert(ptn != nullptr, "node should be registered");
2043     set_escape_state(ptn, PointsToNode::GlobalEscape NOT_PRODUCT(COMMA "stored at raw address"));
2044     // Add edge to object for unsafe access with offset.
2045     PointsToNode* adr_ptn = ptnode_adr(adr->_idx);
2046     assert(adr_ptn != nullptr, "node should be registered");
2047     if (adr_ptn->is_Field()) {
2048       assert(adr_ptn->as_Field()->is_oop(), "should be oop field");
2049       add_edge(adr_ptn, ptn);
2050     }
2051     return true;
2052   }
2053 #ifdef ASSERT
2054   n->dump(1);
2055   assert(false, "not unsafe");
2056 #endif
2057   return false;
2058 }
2059 
2060 void ConnectionGraph::add_call_node(CallNode* call) {
2061   assert(call->returns_pointer() || call->tf()->returns_inline_type_as_fields(), "only for call which returns pointer");
2062   uint call_idx = call->_idx;
2063   if (call->is_Allocate()) {
2064     Node* k = call->in(AllocateNode::KlassNode);
2065     const TypeKlassPtr* kt = k->bottom_type()->isa_klassptr();
2066     assert(kt != nullptr, "TypeKlassPtr  required.");
2067     PointsToNode::EscapeState es = PointsToNode::NoEscape;
2068     bool scalar_replaceable = true;
2069     NOT_PRODUCT(const char* nsr_reason = "");
2070     if (call->is_AllocateArray()) {
2071       if (!kt->isa_aryklassptr()) { // StressReflectiveCode
2072         es = PointsToNode::GlobalEscape;
2073       } else {
2074         int length = call->in(AllocateNode::ALength)->find_int_con(-1);
2075         if (length < 0) {
2076           // Not scalar replaceable if the length is not constant.
2077           scalar_replaceable = false;
2078           NOT_PRODUCT(nsr_reason = "has a non-constant length");
2079         } else if (length > EliminateAllocationArraySizeLimit) {
2080           // Not scalar replaceable if the length is too big.
2081           scalar_replaceable = false;

2117     //
2118     //    - all oop arguments are escaping globally;
2119     //
2120     // 2. CallStaticJavaNode (execute bytecode analysis if possible):
2121     //
2122     //    - the same as CallDynamicJavaNode if can't do bytecode analysis;
2123     //
2124     //    - mapped to GlobalEscape JavaObject node if unknown oop is returned;
2125     //    - mapped to NoEscape JavaObject node if non-escaping object allocated
2126     //      during call is returned;
2127     //    - mapped to ArgEscape LocalVar node pointed to object arguments
2128     //      which are returned and does not escape during call;
2129     //
2130     //    - oop arguments escaping status is defined by bytecode analysis;
2131     //
2132     // For a static call, we know exactly what method is being called.
2133     // Use bytecode estimator to record whether the call's return value escapes.
2134     ciMethod* meth = call->as_CallJava()->method();
2135     if (meth == nullptr) {
2136       const char* name = call->as_CallStaticJava()->_name;
2137       assert(strncmp(name, "C2 Runtime multianewarray", 25) == 0 ||
2138              strncmp(name, "C2 Runtime load_unknown_inline", 30) == 0 ||
2139              strncmp(name, "store_inline_type_fields_to_buf", 31) == 0, "TODO: add failed case check");
2140       // Returns a newly allocated non-escaped object.
2141       add_java_object(call, PointsToNode::NoEscape);
2142       set_not_scalar_replaceable(ptnode_adr(call_idx) NOT_PRODUCT(COMMA "is result of multinewarray"));
2143     } else if (meth->is_boxing_method()) {
2144       // Returns boxing object
2145       PointsToNode::EscapeState es;
2146       vmIntrinsics::ID intr = meth->intrinsic_id();
2147       if (intr == vmIntrinsics::_floatValue || intr == vmIntrinsics::_doubleValue) {
2148         // It does not escape if object is always allocated.
2149         es = PointsToNode::NoEscape;
2150       } else {
2151         // It escapes globally if object could be loaded from cache.
2152         es = PointsToNode::GlobalEscape;
2153       }
2154       add_java_object(call, es);
2155       if (es == PointsToNode::GlobalEscape) {
2156         set_not_scalar_replaceable(ptnode_adr(call->_idx) NOT_PRODUCT(COMMA "object can be loaded from boxing cache"));
2157       }
2158     } else {
2159       BCEscapeAnalyzer* call_analyzer = meth->get_bcea();
2160       call_analyzer->copy_dependencies(_compile->dependencies());
2161       if (call_analyzer->is_return_allocated()) {
2162         // Returns a newly allocated non-escaped object, simply
2163         // update dependency information.
2164         // Mark it as NoEscape so that objects referenced by
2165         // it's fields will be marked as NoEscape at least.
2166         add_java_object(call, PointsToNode::NoEscape);
2167         set_not_scalar_replaceable(ptnode_adr(call_idx) NOT_PRODUCT(COMMA "is result of call"));
2168       } else {
2169         // Determine whether any arguments are returned.
2170         const TypeTuple* d = call->tf()->domain_cc();
2171         bool ret_arg = false;
2172         for (uint i = TypeFunc::Parms; i < d->cnt(); i++) {
2173           if (d->field_at(i)->isa_ptr() != nullptr &&
2174               call_analyzer->is_arg_returned(i - TypeFunc::Parms)) {
2175             ret_arg = true;
2176             break;
2177           }
2178         }
2179         if (ret_arg) {
2180           add_local_var(call, PointsToNode::ArgEscape);
2181         } else {
2182           // Returns unknown object.
2183           map_ideal_node(call, phantom_obj);
2184         }
2185       }
2186     }
2187   } else {
2188     // An other type of call, assume the worst case:
2189     // returned value is unknown and globally escapes.
2190     assert(call->Opcode() == Op_CallDynamicJava, "add failed case check");

2198 #ifdef ASSERT
2199     case Op_Allocate:
2200     case Op_AllocateArray:
2201     case Op_Lock:
2202     case Op_Unlock:
2203       assert(false, "should be done already");
2204       break;
2205 #endif
2206     case Op_ArrayCopy:
2207     case Op_CallLeafNoFP:
2208       // Most array copies are ArrayCopy nodes at this point but there
2209       // are still a few direct calls to the copy subroutines (See
2210       // PhaseStringOpts::copy_string())
2211       is_arraycopy = (call->Opcode() == Op_ArrayCopy) ||
2212         call->as_CallLeaf()->is_call_to_arraycopystub();
2213       // fall through
2214     case Op_CallLeafVector:
2215     case Op_CallLeaf: {
2216       // Stub calls, objects do not escape but they are not scale replaceable.
2217       // Adjust escape state for outgoing arguments.
2218       const TypeTuple * d = call->tf()->domain_sig();
2219       bool src_has_oops = false;
2220       for (uint i = TypeFunc::Parms; i < d->cnt(); i++) {
2221         const Type* at = d->field_at(i);
2222         Node *arg = call->in(i);
2223         if (arg == nullptr) {
2224           continue;
2225         }
2226         const Type *aat = _igvn->type(arg);
2227         if (arg->is_top() || !at->isa_ptr() || !aat->isa_ptr()) {
2228           continue;
2229         }
2230         if (arg->is_AddP()) {
2231           //
2232           // The inline_native_clone() case when the arraycopy stub is called
2233           // after the allocation before Initialize and CheckCastPP nodes.
2234           // Or normal arraycopy for object arrays case.
2235           //
2236           // Set AddP's base (Allocate) as not scalar replaceable since
2237           // pointer to the base (with offset) is passed as argument.
2238           //
2239           arg = get_addp_base(arg);
2240         }
2241         PointsToNode* arg_ptn = ptnode_adr(arg->_idx);
2242         assert(arg_ptn != nullptr, "should be registered");
2243         PointsToNode::EscapeState arg_esc = arg_ptn->escape_state();
2244         if (is_arraycopy || arg_esc < PointsToNode::ArgEscape) {
2245           assert(aat == Type::TOP || aat == TypePtr::NULL_PTR ||
2246                  aat->isa_ptr() != nullptr, "expecting an Ptr");
2247           bool arg_has_oops = aat->isa_oopptr() &&
2248                               (aat->isa_instptr() ||
2249                                (aat->isa_aryptr() && (aat->isa_aryptr()->elem() == Type::BOTTOM || aat->isa_aryptr()->elem()->make_oopptr() != nullptr)) ||
2250                                (aat->isa_aryptr() && aat->isa_aryptr()->elem() != nullptr &&
2251                                                                aat->isa_aryptr()->is_flat() &&
2252                                                                aat->isa_aryptr()->elem()->inline_klass()->contains_oops()));
2253           if (i == TypeFunc::Parms) {
2254             src_has_oops = arg_has_oops;
2255           }
2256           //
2257           // src or dst could be j.l.Object when other is basic type array:
2258           //
2259           //   arraycopy(char[],0,Object*,0,size);
2260           //   arraycopy(Object*,0,char[],0,size);
2261           //
2262           // Don't add edges in such cases.
2263           //
2264           bool arg_is_arraycopy_dest = src_has_oops && is_arraycopy &&
2265                                        arg_has_oops && (i > TypeFunc::Parms);
2266 #ifdef ASSERT
2267           if (!(is_arraycopy ||
2268                 BarrierSet::barrier_set()->barrier_set_c2()->is_gc_barrier_node(call) ||
2269                 (call->as_CallLeaf()->_name != nullptr &&
2270                  (strcmp(call->as_CallLeaf()->_name, "updateBytesCRC32") == 0 ||
2271                   strcmp(call->as_CallLeaf()->_name, "updateBytesCRC32C") == 0 ||
2272                   strcmp(call->as_CallLeaf()->_name, "updateBytesAdler32") == 0 ||

2296                   strcmp(call->as_CallLeaf()->_name, "dilithiumMontMulByConstant") == 0 ||
2297                   strcmp(call->as_CallLeaf()->_name, "dilithiumDecomposePoly") == 0 ||
2298                   strcmp(call->as_CallLeaf()->_name, "encodeBlock") == 0 ||
2299                   strcmp(call->as_CallLeaf()->_name, "decodeBlock") == 0 ||
2300                   strcmp(call->as_CallLeaf()->_name, "md5_implCompress") == 0 ||
2301                   strcmp(call->as_CallLeaf()->_name, "md5_implCompressMB") == 0 ||
2302                   strcmp(call->as_CallLeaf()->_name, "sha1_implCompress") == 0 ||
2303                   strcmp(call->as_CallLeaf()->_name, "sha1_implCompressMB") == 0 ||
2304                   strcmp(call->as_CallLeaf()->_name, "sha256_implCompress") == 0 ||
2305                   strcmp(call->as_CallLeaf()->_name, "sha256_implCompressMB") == 0 ||
2306                   strcmp(call->as_CallLeaf()->_name, "sha512_implCompress") == 0 ||
2307                   strcmp(call->as_CallLeaf()->_name, "sha512_implCompressMB") == 0 ||
2308                   strcmp(call->as_CallLeaf()->_name, "sha3_implCompress") == 0 ||
2309                   strcmp(call->as_CallLeaf()->_name, "double_keccak") == 0 ||
2310                   strcmp(call->as_CallLeaf()->_name, "sha3_implCompressMB") == 0 ||
2311                   strcmp(call->as_CallLeaf()->_name, "multiplyToLen") == 0 ||
2312                   strcmp(call->as_CallLeaf()->_name, "squareToLen") == 0 ||
2313                   strcmp(call->as_CallLeaf()->_name, "mulAdd") == 0 ||
2314                   strcmp(call->as_CallLeaf()->_name, "montgomery_multiply") == 0 ||
2315                   strcmp(call->as_CallLeaf()->_name, "montgomery_square") == 0 ||
2316                   strcmp(call->as_CallLeaf()->_name, "vectorizedMismatch") == 0 ||
2317                   strcmp(call->as_CallLeaf()->_name, "load_unknown_inline") == 0 ||
2318                   strcmp(call->as_CallLeaf()->_name, "store_unknown_inline") == 0 ||
2319                   strcmp(call->as_CallLeaf()->_name, "store_inline_type_fields_to_buf") == 0 ||
2320                   strcmp(call->as_CallLeaf()->_name, "bigIntegerRightShiftWorker") == 0 ||
2321                   strcmp(call->as_CallLeaf()->_name, "bigIntegerLeftShiftWorker") == 0 ||
2322                   strcmp(call->as_CallLeaf()->_name, "vectorizedMismatch") == 0 ||
2323                   strcmp(call->as_CallLeaf()->_name, "stringIndexOf") == 0 ||
2324                   strcmp(call->as_CallLeaf()->_name, "arraysort_stub") == 0 ||
2325                   strcmp(call->as_CallLeaf()->_name, "array_partition_stub") == 0 ||
2326                   strcmp(call->as_CallLeaf()->_name, "get_class_id_intrinsic") == 0 ||
2327                   strcmp(call->as_CallLeaf()->_name, "unsafe_setmemory") == 0)
2328                  ))) {
2329             call->dump();
2330             fatal("EA unexpected CallLeaf %s", call->as_CallLeaf()->_name);
2331           }
2332 #endif
2333           // Always process arraycopy's destination object since
2334           // we need to add all possible edges to references in
2335           // source object.
2336           if (arg_esc >= PointsToNode::ArgEscape &&
2337               !arg_is_arraycopy_dest) {
2338             continue;
2339           }

2362           }
2363         }
2364       }
2365       break;
2366     }
2367     case Op_CallStaticJava: {
2368       // For a static call, we know exactly what method is being called.
2369       // Use bytecode estimator to record the call's escape affects
2370 #ifdef ASSERT
2371       const char* name = call->as_CallStaticJava()->_name;
2372       assert((name == nullptr || strcmp(name, "uncommon_trap") != 0), "normal calls only");
2373 #endif
2374       ciMethod* meth = call->as_CallJava()->method();
2375       if ((meth != nullptr) && meth->is_boxing_method()) {
2376         break; // Boxing methods do not modify any oops.
2377       }
2378       BCEscapeAnalyzer* call_analyzer = (meth !=nullptr) ? meth->get_bcea() : nullptr;
2379       // fall-through if not a Java method or no analyzer information
2380       if (call_analyzer != nullptr) {
2381         PointsToNode* call_ptn = ptnode_adr(call->_idx);
2382         const TypeTuple* d = call->tf()->domain_cc();
2383         for (uint i = TypeFunc::Parms; i < d->cnt(); i++) {
2384           const Type* at = d->field_at(i);
2385           int k = i - TypeFunc::Parms;
2386           Node* arg = call->in(i);
2387           PointsToNode* arg_ptn = ptnode_adr(arg->_idx);
2388           if (at->isa_ptr() != nullptr &&
2389               call_analyzer->is_arg_returned(k)) {
2390             // The call returns arguments.
2391             if (call_ptn != nullptr) { // Is call's result used?
2392               assert(call_ptn->is_LocalVar(), "node should be registered");
2393               assert(arg_ptn != nullptr, "node should be registered");
2394               add_edge(call_ptn, arg_ptn);
2395             }
2396           }
2397           if (at->isa_oopptr() != nullptr &&
2398               arg_ptn->escape_state() < PointsToNode::GlobalEscape) {
2399             if (!call_analyzer->is_arg_stack(k)) {
2400               // The argument global escapes
2401               set_escape_state(arg_ptn, PointsToNode::GlobalEscape NOT_PRODUCT(COMMA trace_arg_escape_message(call)));
2402             } else {

2406                 set_fields_escape_state(arg_ptn, PointsToNode::GlobalEscape NOT_PRODUCT(COMMA trace_arg_escape_message(call)));
2407               }
2408             }
2409           }
2410         }
2411         if (call_ptn != nullptr && call_ptn->is_LocalVar()) {
2412           // The call returns arguments.
2413           assert(call_ptn->edge_count() > 0, "sanity");
2414           if (!call_analyzer->is_return_local()) {
2415             // Returns also unknown object.
2416             add_edge(call_ptn, phantom_obj);
2417           }
2418         }
2419         break;
2420       }
2421     }
2422     default: {
2423       // Fall-through here if not a Java method or no analyzer information
2424       // or some other type of call, assume the worst case: all arguments
2425       // globally escape.
2426       const TypeTuple* d = call->tf()->domain_cc();
2427       for (uint i = TypeFunc::Parms; i < d->cnt(); i++) {
2428         const Type* at = d->field_at(i);
2429         if (at->isa_oopptr() != nullptr) {
2430           Node* arg = call->in(i);
2431           if (arg->is_AddP()) {
2432             arg = get_addp_base(arg);
2433           }
2434           assert(ptnode_adr(arg->_idx) != nullptr, "should be defined already");
2435           set_escape_state(ptnode_adr(arg->_idx), PointsToNode::GlobalEscape NOT_PRODUCT(COMMA trace_arg_escape_message(call)));
2436         }
2437       }
2438     }
2439   }
2440 }
2441 
2442 
2443 // Finish Graph construction.
2444 bool ConnectionGraph::complete_connection_graph(
2445                          GrowableArray<PointsToNode*>&   ptnodes_worklist,
2446                          GrowableArray<JavaObjectNode*>& non_escaped_allocs_worklist,

2819     PointsToNode* base = i.get();
2820     if (base->is_JavaObject()) {
2821       // Skip Allocate's fields which will be processed later.
2822       if (base->ideal_node()->is_Allocate()) {
2823         return 0;
2824       }
2825       assert(base == null_obj, "only null ptr base expected here");
2826     }
2827   }
2828   if (add_edge(field, phantom_obj)) {
2829     // New edge was added
2830     new_edges++;
2831     add_field_uses_to_worklist(field);
2832   }
2833   return new_edges;
2834 }
2835 
2836 // Find fields initializing values for allocations.
2837 int ConnectionGraph::find_init_values_phantom(JavaObjectNode* pta) {
2838   assert(pta->escape_state() == PointsToNode::NoEscape, "Not escaped Allocate nodes only");
2839   PointsToNode* init_val = phantom_obj;
2840   Node* alloc = pta->ideal_node();
2841 
2842   // Do nothing for Allocate nodes since its fields values are
2843   // "known" unless they are initialized by arraycopy/clone.
2844   if (alloc->is_Allocate() && !pta->arraycopy_dst()) {
2845     if (alloc->as_Allocate()->in(AllocateNode::InitValue) != nullptr) {
2846       // Null-free inline type arrays are initialized with an init value instead of null
2847       init_val = ptnode_adr(alloc->as_Allocate()->in(AllocateNode::InitValue)->_idx);
2848       assert(init_val != nullptr, "init value should be registered");
2849     } else {
2850       return 0;
2851     }
2852   }
2853   // Non-escaped allocation returned from Java or runtime call has unknown values in fields.
2854   assert(pta->arraycopy_dst() || alloc->is_CallStaticJava() || init_val != phantom_obj, "sanity");
2855 #ifdef ASSERT
2856   if (alloc->is_CallStaticJava() && alloc->as_CallStaticJava()->method() == nullptr) {
2857     const char* name = alloc->as_CallStaticJava()->_name;
2858     assert(strncmp(name, "C2 Runtime multianewarray", 25) == 0 ||
2859            strncmp(name, "C2 Runtime load_unknown_inline", 30) == 0 ||
2860            strncmp(name, "store_inline_type_fields_to_buf", 31) == 0, "sanity");
2861   }
2862 #endif
2863   // Non-escaped allocation returned from Java or runtime call have unknown values in fields.
2864   int new_edges = 0;
2865   for (EdgeIterator i(pta); i.has_next(); i.next()) {
2866     PointsToNode* field = i.get();
2867     if (field->is_Field() && field->as_Field()->is_oop()) {
2868       if (add_edge(field, init_val)) {
2869         // New edge was added
2870         new_edges++;
2871         add_field_uses_to_worklist(field->as_Field());
2872       }
2873     }
2874   }
2875   return new_edges;
2876 }
2877 
2878 // Find fields initializing values for allocations.
2879 int ConnectionGraph::find_init_values_null(JavaObjectNode* pta, PhaseValues* phase) {
2880   assert(pta->escape_state() == PointsToNode::NoEscape, "Not escaped Allocate nodes only");
2881   Node* alloc = pta->ideal_node();
2882   // Do nothing for Call nodes since its fields values are unknown.
2883   if (!alloc->is_Allocate() || alloc->as_Allocate()->in(AllocateNode::InitValue) != nullptr) {
2884     return 0;
2885   }
2886   InitializeNode* ini = alloc->as_Allocate()->initialization();
2887   bool visited_bottom_offset = false;
2888   GrowableArray<int> offsets_worklist;
2889   int new_edges = 0;
2890 
2891   // Check if an oop field's initializing value is recorded and add
2892   // a corresponding null if field's value if it is not recorded.
2893   // Connection Graph does not record a default initialization by null
2894   // captured by Initialize node.
2895   //
2896   for (EdgeIterator i(pta); i.has_next(); i.next()) {
2897     PointsToNode* field = i.get(); // Field (AddP)
2898     if (!field->is_Field() || !field->as_Field()->is_oop()) {
2899       continue; // Not oop field
2900     }
2901     int offset = field->as_Field()->offset();
2902     if (offset == Type::OffsetBot) {
2903       if (!visited_bottom_offset) {

2949               } else {
2950                 if (!val->is_LocalVar() || (val->edge_count() == 0)) {
2951                   tty->print_cr("----------init store has invalid value -----");
2952                   store->dump();
2953                   val->dump();
2954                   assert(val->is_LocalVar() && (val->edge_count() > 0), "should be processed already");
2955                 }
2956                 for (EdgeIterator j(val); j.has_next(); j.next()) {
2957                   PointsToNode* obj = j.get();
2958                   if (obj->is_JavaObject()) {
2959                     if (!field->points_to(obj->as_JavaObject())) {
2960                       missed_obj = obj;
2961                       break;
2962                     }
2963                   }
2964                 }
2965               }
2966               if (missed_obj != nullptr) {
2967                 tty->print_cr("----------field---------------------------------");
2968                 field->dump();
2969                 tty->print_cr("----------missed reference to object------------");
2970                 missed_obj->dump();
2971                 tty->print_cr("----------object referenced by init store-------");
2972                 store->dump();
2973                 val->dump();
2974                 assert(!field->points_to(missed_obj->as_JavaObject()), "missed JavaObject reference");
2975               }
2976             }
2977 #endif
2978           } else {
2979             // There could be initializing stores which follow allocation.
2980             // For example, a volatile field store is not collected
2981             // by Initialize node.
2982             //
2983             // Need to check for dependent loads to separate such stores from
2984             // stores which follow loads. For now, add initial value null so
2985             // that compare pointers optimization works correctly.
2986           }
2987         }
2988         if (value == nullptr) {
2989           // A field's initializing value was not recorded. Add null.
2990           if (add_edge(field, null_obj)) {
2991             // New edge was added

3315         assert(field->edge_count() > 0, "sanity");
3316       }
3317     }
3318   }
3319 }
3320 #endif
3321 
3322 // Optimize ideal graph.
3323 void ConnectionGraph::optimize_ideal_graph(GrowableArray<Node*>& ptr_cmp_worklist,
3324                                            GrowableArray<MemBarStoreStoreNode*>& storestore_worklist) {
3325   Compile* C = _compile;
3326   PhaseIterGVN* igvn = _igvn;
3327   if (EliminateLocks) {
3328     // Mark locks before changing ideal graph.
3329     int cnt = C->macro_count();
3330     for (int i = 0; i < cnt; i++) {
3331       Node *n = C->macro_node(i);
3332       if (n->is_AbstractLock()) { // Lock and Unlock nodes
3333         AbstractLockNode* alock = n->as_AbstractLock();
3334         if (!alock->is_non_esc_obj()) {
3335           const Type* obj_type = igvn->type(alock->obj_node());
3336           if (can_eliminate_lock(alock) && !obj_type->is_inlinetypeptr()) {
3337             assert(!alock->is_eliminated() || alock->is_coarsened(), "sanity");
3338             // The lock could be marked eliminated by lock coarsening
3339             // code during first IGVN before EA. Replace coarsened flag
3340             // to eliminate all associated locks/unlocks.
3341 #ifdef ASSERT
3342             alock->log_lock_optimization(C, "eliminate_lock_set_non_esc3");
3343 #endif
3344             alock->set_non_esc_obj();
3345           }
3346         }
3347       }
3348     }
3349   }
3350 
3351   if (OptimizePtrCompare) {
3352     for (int i = 0; i < ptr_cmp_worklist.length(); i++) {
3353       Node *n = ptr_cmp_worklist.at(i);
3354       assert(n->Opcode() == Op_CmpN || n->Opcode() == Op_CmpP, "must be");
3355       const TypeInt* tcmp = optimize_ptr_compare(n->in(1), n->in(2));
3356       if (tcmp->singleton()) {

3358 #ifndef PRODUCT
3359         if (PrintOptimizePtrCompare) {
3360           tty->print_cr("++++ Replaced: %d %s(%d,%d) --> %s", n->_idx, (n->Opcode() == Op_CmpP ? "CmpP" : "CmpN"), n->in(1)->_idx, n->in(2)->_idx, (tcmp == TypeInt::CC_EQ ? "EQ" : "NotEQ"));
3361           if (Verbose) {
3362             n->dump(1);
3363           }
3364         }
3365 #endif
3366         igvn->replace_node(n, cmp);
3367       }
3368     }
3369   }
3370 
3371   // For MemBarStoreStore nodes added in library_call.cpp, check
3372   // escape status of associated AllocateNode and optimize out
3373   // MemBarStoreStore node if the allocated object never escapes.
3374   for (int i = 0; i < storestore_worklist.length(); i++) {
3375     Node* storestore = storestore_worklist.at(i);
3376     Node* alloc = storestore->in(MemBarNode::Precedent)->in(0);
3377     if (alloc->is_Allocate() && not_global_escape(alloc)) {
3378       if (alloc->in(AllocateNode::InlineType) != nullptr) {
3379         // Non-escaping inline type buffer allocations don't require a membar
3380         storestore->as_MemBar()->remove(_igvn);
3381       } else {
3382         MemBarNode* mb = MemBarNode::make(C, Op_MemBarCPUOrder, Compile::AliasIdxBot);
3383         mb->init_req(TypeFunc::Memory,  storestore->in(TypeFunc::Memory));
3384         mb->init_req(TypeFunc::Control, storestore->in(TypeFunc::Control));
3385         igvn->register_new_node_with_optimizer(mb);
3386         igvn->replace_node(storestore, mb);
3387       }
3388     }
3389   }
3390 }
3391 
3392 // Atomic flat accesses on non-escaping objects can be optimized to non-atomic accesses
3393 void ConnectionGraph::optimize_flat_accesses(GrowableArray<SafePointNode*>& sfn_worklist) {
3394   PhaseIterGVN& igvn = *_igvn;
3395   bool delay = igvn.delay_transform();
3396   igvn.set_delay_transform(true);
3397   igvn.C->for_each_flat_access([&](Node* n) {
3398     Node* base = n->is_LoadFlat() ? n->as_LoadFlat()->base() : n->as_StoreFlat()->base();
3399     if (!not_global_escape(base)) {
3400       return;
3401     }
3402 
3403     bool expanded;
3404     if (n->is_LoadFlat()) {
3405       expanded = n->as_LoadFlat()->expand_non_atomic(igvn);
3406     } else {
3407       expanded = n->as_StoreFlat()->expand_non_atomic(igvn);
3408     }
3409     if (expanded) {
3410       sfn_worklist.remove(n->as_SafePoint());
3411       igvn.C->remove_flat_access(n);
3412     }
3413   });
3414   igvn.set_delay_transform(delay);
3415 }
3416 
3417 // Optimize objects compare.
3418 const TypeInt* ConnectionGraph::optimize_ptr_compare(Node* left, Node* right) {
3419   const TypeInt* UNKNOWN = TypeInt::CC;    // [-1, 0,1]
3420   if (!OptimizePtrCompare) {
3421     return UNKNOWN;
3422   }
3423   const TypeInt* EQ = TypeInt::CC_EQ; // [0] == ZERO
3424   const TypeInt* NE = TypeInt::CC_GT; // [1] == ONE
3425 
3426   PointsToNode* ptn1 = ptnode_adr(left->_idx);
3427   PointsToNode* ptn2 = ptnode_adr(right->_idx);
3428   JavaObjectNode* jobj1 = unique_java_object(left);
3429   JavaObjectNode* jobj2 = unique_java_object(right);
3430 
3431   // The use of this method during allocation merge reduction may cause 'left'
3432   // or 'right' be something (e.g., a Phi) that isn't in the connection graph or
3433   // that doesn't reference an unique java object.
3434   if (ptn1 == nullptr || ptn2 == nullptr ||
3435       jobj1 == nullptr || jobj2 == nullptr) {
3436     return UNKNOWN;

3556   assert(!src->is_Field() && !dst->is_Field(), "only for JavaObject and LocalVar");
3557   assert((src != null_obj) && (dst != null_obj), "not for ConP null");
3558   PointsToNode* ptadr = _nodes.at(n->_idx);
3559   if (ptadr != nullptr) {
3560     assert(ptadr->is_Arraycopy() && ptadr->ideal_node() == n, "sanity");
3561     return;
3562   }
3563   Compile* C = _compile;
3564   ptadr = new (C->comp_arena()) ArraycopyNode(this, n, es);
3565   map_ideal_node(n, ptadr);
3566   // Add edge from arraycopy node to source object.
3567   (void)add_edge(ptadr, src);
3568   src->set_arraycopy_src();
3569   // Add edge from destination object to arraycopy node.
3570   (void)add_edge(dst, ptadr);
3571   dst->set_arraycopy_dst();
3572 }
3573 
3574 bool ConnectionGraph::is_oop_field(Node* n, int offset, bool* unsafe) {
3575   const Type* adr_type = n->as_AddP()->bottom_type();
3576   int field_offset = adr_type->isa_aryptr() ? adr_type->isa_aryptr()->field_offset().get() : Type::OffsetBot;
3577   BasicType bt = T_INT;
3578   if (offset == Type::OffsetBot && field_offset == Type::OffsetBot) {
3579     // Check only oop fields.
3580     if (!adr_type->isa_aryptr() ||
3581         adr_type->isa_aryptr()->elem() == Type::BOTTOM ||
3582         adr_type->isa_aryptr()->elem()->make_oopptr() != nullptr) {
3583       // OffsetBot is used to reference array's element. Ignore first AddP.
3584       if (find_second_addp(n, n->in(AddPNode::Base)) == nullptr) {
3585         bt = T_OBJECT;
3586       }
3587     }
3588   } else if (offset != oopDesc::klass_offset_in_bytes()) {
3589     if (adr_type->isa_instptr()) {
3590       ciField* field = _compile->alias_type(adr_type->is_ptr())->field();
3591       if (field != nullptr) {
3592         bt = field->layout_type();
3593       } else {
3594         // Check for unsafe oop field access
3595         if (n->has_out_with(Op_StoreP, Op_LoadP, Op_StoreN, Op_LoadN) ||
3596             n->has_out_with(Op_GetAndSetP, Op_GetAndSetN, Op_CompareAndExchangeP, Op_CompareAndExchangeN) ||
3597             n->has_out_with(Op_CompareAndSwapP, Op_CompareAndSwapN, Op_WeakCompareAndSwapP, Op_WeakCompareAndSwapN) ||
3598             BarrierSet::barrier_set()->barrier_set_c2()->escape_has_out_with_unsafe_object(n)) {
3599           bt = T_OBJECT;
3600           (*unsafe) = true;
3601         }
3602       }
3603     } else if (adr_type->isa_aryptr()) {
3604       if (offset == arrayOopDesc::length_offset_in_bytes()) {
3605         // Ignore array length load.
3606       } else if (find_second_addp(n, n->in(AddPNode::Base)) != nullptr) {
3607         // Ignore first AddP.
3608       } else {
3609         const Type* elemtype = adr_type->is_aryptr()->elem();
3610         if (adr_type->is_aryptr()->is_flat() && field_offset != Type::OffsetBot) {
3611           ciInlineKlass* vk = elemtype->inline_klass();
3612           field_offset += vk->payload_offset();
3613           ciField* field = vk->get_field_by_offset(field_offset, false);
3614           if (field != nullptr) {
3615             bt = field->layout_type();
3616           } else {
3617             assert(field_offset == vk->payload_offset() + vk->null_marker_offset_in_payload(), "no field or null marker of %s at offset %d", vk->name()->as_utf8(), field_offset);
3618             bt = T_BOOLEAN;
3619           }
3620         } else {
3621           bt = elemtype->array_element_basic_type();
3622         }
3623       }
3624     } else if (adr_type->isa_rawptr() || adr_type->isa_klassptr()) {
3625       // Allocation initialization, ThreadLocal field access, unsafe access
3626       if (n->has_out_with(Op_StoreP, Op_LoadP, Op_StoreN, Op_LoadN) ||
3627           n->has_out_with(Op_GetAndSetP, Op_GetAndSetN, Op_CompareAndExchangeP, Op_CompareAndExchangeN) ||
3628           n->has_out_with(Op_CompareAndSwapP, Op_CompareAndSwapN, Op_WeakCompareAndSwapP, Op_WeakCompareAndSwapN) ||
3629           BarrierSet::barrier_set()->barrier_set_c2()->escape_has_out_with_unsafe_object(n)) {
3630         bt = T_OBJECT;
3631       }
3632     }
3633   }
3634   // Note: T_NARROWOOP is not classed as a real reference type
3635   return (is_reference_type(bt) || bt == T_NARROWOOP);
3636 }
3637 
3638 // Returns unique pointed java object or null.
3639 JavaObjectNode* ConnectionGraph::unique_java_object(Node *n) const {
3640   // If the node was created after the escape computation we can't answer.
3641   uint idx = n->_idx;
3642   if (idx >= nodes_size()) {

3799             return true;
3800           }
3801         }
3802       }
3803     }
3804   }
3805   return false;
3806 }
3807 
3808 int ConnectionGraph::address_offset(Node* adr, PhaseValues* phase) {
3809   const Type *adr_type = phase->type(adr);
3810   if (adr->is_AddP() && adr_type->isa_oopptr() == nullptr && is_captured_store_address(adr)) {
3811     // We are computing a raw address for a store captured by an Initialize
3812     // compute an appropriate address type. AddP cases #3 and #5 (see below).
3813     int offs = (int)phase->find_intptr_t_con(adr->in(AddPNode::Offset), Type::OffsetBot);
3814     assert(offs != Type::OffsetBot ||
3815            adr->in(AddPNode::Address)->in(0)->is_AllocateArray(),
3816            "offset must be a constant or it is initialization of array");
3817     return offs;
3818   }
3819   return adr_type->is_ptr()->flat_offset();


3820 }
3821 
3822 Node* ConnectionGraph::get_addp_base(Node *addp) {
3823   assert(addp->is_AddP(), "must be AddP");
3824   //
3825   // AddP cases for Base and Address inputs:
3826   // case #1. Direct object's field reference:
3827   //     Allocate
3828   //       |
3829   //     Proj #5 ( oop result )
3830   //       |
3831   //     CheckCastPP (cast to instance type)
3832   //      | |
3833   //     AddP  ( base == address )
3834   //
3835   // case #2. Indirect object's field reference:
3836   //      Phi
3837   //       |
3838   //     CastPP (cast to instance type)
3839   //      | |

3953   }
3954   return nullptr;
3955 }
3956 
3957 //
3958 // Adjust the type and inputs of an AddP which computes the
3959 // address of a field of an instance
3960 //
3961 bool ConnectionGraph::split_AddP(Node *addp, Node *base) {
3962   PhaseGVN* igvn = _igvn;
3963   const TypeOopPtr *base_t = igvn->type(base)->isa_oopptr();
3964   assert(base_t != nullptr && base_t->is_known_instance(), "expecting instance oopptr");
3965   const TypeOopPtr *t = igvn->type(addp)->isa_oopptr();
3966   if (t == nullptr) {
3967     // We are computing a raw address for a store captured by an Initialize
3968     // compute an appropriate address type (cases #3 and #5).
3969     assert(igvn->type(addp) == TypeRawPtr::NOTNULL, "must be raw pointer");
3970     assert(addp->in(AddPNode::Address)->is_Proj(), "base of raw address must be result projection from allocation");
3971     intptr_t offs = (int)igvn->find_intptr_t_con(addp->in(AddPNode::Offset), Type::OffsetBot);
3972     assert(offs != Type::OffsetBot, "offset must be a constant");
3973     if (base_t->isa_aryptr() != nullptr) {
3974       // In the case of a flat inline type array, each field has its
3975       // own slice so we need to extract the field being accessed from
3976       // the address computation
3977       t = base_t->isa_aryptr()->add_field_offset_and_offset(offs)->is_oopptr();
3978     } else {
3979       t = base_t->add_offset(offs)->is_oopptr();
3980     }
3981   }
3982   int inst_id = base_t->instance_id();
3983   assert(!t->is_known_instance() || t->instance_id() == inst_id,
3984                              "old type must be non-instance or match new type");
3985 
3986   // The type 't' could be subclass of 'base_t'.
3987   // As result t->offset() could be large then base_t's size and it will
3988   // cause the failure in add_offset() with narrow oops since TypeOopPtr()
3989   // constructor verifies correctness of the offset.
3990   //
3991   // It could happened on subclass's branch (from the type profiling
3992   // inlining) which was not eliminated during parsing since the exactness
3993   // of the allocation type was not propagated to the subclass type check.
3994   //
3995   // Or the type 't' could be not related to 'base_t' at all.
3996   // It could happen when CHA type is different from MDO type on a dead path
3997   // (for example, from instanceof check) which is not collapsed during parsing.
3998   //
3999   // Do nothing for such AddP node and don't process its users since
4000   // this code branch will go away.
4001   //
4002   if (!t->is_known_instance() &&
4003       !base_t->maybe_java_subtype_of(t)) {
4004      return false; // bail out
4005   }
4006   const TypePtr* tinst = base_t->add_offset(t->offset());
4007   if (tinst->isa_aryptr() && t->isa_aryptr()) {
4008     // In the case of a flat inline type array, each field has its
4009     // own slice so we need to keep track of the field being accessed.
4010     tinst = tinst->is_aryptr()->with_field_offset(t->is_aryptr()->field_offset().get());
4011     // Keep array properties (not flat/null-free)
4012     tinst = tinst->is_aryptr()->update_properties(t->is_aryptr());
4013     if (tinst == nullptr) {
4014       return false; // Skip dead path with inconsistent properties
4015     }
4016   }
4017 
4018   // Do NOT remove the next line: ensure a new alias index is allocated
4019   // for the instance type. Note: C++ will not remove it since the call
4020   // has side effect.
4021   int alias_idx = _compile->get_alias_index(tinst);
4022   igvn->set_type(addp, tinst);
4023   // record the allocation in the node map
4024   set_map(addp, get_map(base->_idx));
4025   // Set addp's Base and Address to 'base'.
4026   Node *abase = addp->in(AddPNode::Base);
4027   Node *adr   = addp->in(AddPNode::Address);
4028   if (adr->is_Proj() && adr->in(0)->is_Allocate() &&
4029       adr->in(0)->_idx == (uint)inst_id) {
4030     // Skip AddP cases #3 and #5.
4031   } else {
4032     assert(!abase->is_top(), "sanity"); // AddP case #3
4033     if (abase != base) {
4034       igvn->hash_delete(addp);
4035       addp->set_req(AddPNode::Base, base);
4036       if (abase == adr) {
4037         addp->set_req(AddPNode::Address, base);

4703         ptnode_adr(n->_idx)->dump();
4704         assert(jobj != nullptr && jobj != phantom_obj, "escaped allocation");
4705 #endif
4706         _compile->record_failure(_invocation > 0 ? C2Compiler::retry_no_iterative_escape_analysis() : C2Compiler::retry_no_escape_analysis());
4707         return;
4708       } else {
4709         Node *val = get_map(jobj->idx());   // CheckCastPP node
4710         TypeNode *tn = n->as_Type();
4711         const TypeOopPtr* tinst = igvn->type(val)->isa_oopptr();
4712         assert(tinst != nullptr && tinst->is_known_instance() &&
4713                tinst->instance_id() == jobj->idx() , "instance type expected.");
4714 
4715         const Type *tn_type = igvn->type(tn);
4716         const TypeOopPtr *tn_t;
4717         if (tn_type->isa_narrowoop()) {
4718           tn_t = tn_type->make_ptr()->isa_oopptr();
4719         } else {
4720           tn_t = tn_type->isa_oopptr();
4721         }
4722         if (tn_t != nullptr && tinst->maybe_java_subtype_of(tn_t)) {
4723           if (tn_t->isa_aryptr()) {
4724             // Keep array properties (not flat/null-free)
4725             tinst = tinst->is_aryptr()->update_properties(tn_t->is_aryptr());
4726             if (tinst == nullptr) {
4727               continue; // Skip dead path with inconsistent properties
4728             }
4729           }
4730           if (tn_type->isa_narrowoop()) {
4731             tn_type = tinst->make_narrowoop();
4732           } else {
4733             tn_type = tinst;
4734           }
4735           igvn->hash_delete(tn);
4736           igvn->set_type(tn, tn_type);
4737           tn->set_type(tn_type);
4738           igvn->hash_insert(tn);
4739           record_for_optimizer(n);
4740         } else {
4741           assert(tn_type == TypePtr::NULL_PTR ||
4742                  (tn_t != nullptr && !tinst->maybe_java_subtype_of(tn_t)),
4743                  "unexpected type");
4744           continue; // Skip dead path with different type
4745         }
4746       }
4747     } else {
4748       DEBUG_ONLY(n->dump();)
4749       assert(false, "EA: unexpected node");
4750       continue;
4751     }
4752     // push allocation's users on appropriate worklist
4753     for (DUIterator_Fast imax, i = n->fast_outs(imax); i < imax; i++) {
4754       Node *use = n->fast_out(i);
4755       if (use->is_Mem() && use->in(MemNode::Address) == n) {
4756         // Load/store to instance's field
4757         memnode_worklist.append_if_missing(use);
4758       } else if (use->is_MemBar()) {
4759         if (use->in(TypeFunc::Memory) == n) { // Ignore precedent edge
4760           memnode_worklist.append_if_missing(use);
4761         }
4762       } else if (use->is_AddP() && use->outcnt() > 0) { // No dead nodes
4763         Node* addp2 = find_second_addp(use, n);
4764         if (addp2 != nullptr) {
4765           alloc_worklist.append_if_missing(addp2);
4766         }
4767         alloc_worklist.append_if_missing(use);
4768       } else if (use->is_Phi() ||
4769                  use->is_CheckCastPP() ||
4770                  use->is_EncodeNarrowPtr() ||
4771                  use->is_DecodeNarrowPtr() ||
4772                  (use->is_ConstraintCast() && use->Opcode() == Op_CastPP)) {
4773         alloc_worklist.append_if_missing(use);
4774 #ifdef ASSERT
4775       } else if (use->is_Mem()) {
4776         assert(use->in(MemNode::Address) != n, "EA: missing allocation reference path");
4777       } else if (use->is_MergeMem()) {
4778         assert(mergemem_worklist.contains(use->as_MergeMem()), "EA: missing MergeMem node in the worklist");
4779       } else if (use->is_SafePoint()) {
4780         // Look for MergeMem nodes for calls which reference unique allocation
4781         // (through CheckCastPP nodes) even for debug info.
4782         Node* m = use->in(TypeFunc::Memory);
4783         if (m->is_MergeMem()) {
4784           assert(mergemem_worklist.contains(m->as_MergeMem()), "EA: missing MergeMem node in the worklist");
4785         }
4786       } else if (use->Opcode() == Op_EncodeISOArray) {
4787         if (use->in(MemNode::Memory) == n || use->in(3) == n) {
4788           // EncodeISOArray overwrites destination array
4789           memnode_worklist.append_if_missing(use);
4790         }
4791       } else if (use->Opcode() == Op_Return) {
4792         // Allocation is referenced by field of returned inline type
4793         assert(_compile->tf()->returns_inline_type_as_fields(), "EA: unexpected reference by ReturnNode");
4794       } else {
4795         uint op = use->Opcode();
4796         if ((op == Op_StrCompressedCopy || op == Op_StrInflatedCopy) &&
4797             (use->in(MemNode::Memory) == n)) {
4798           // They overwrite memory edge corresponding to destination array,
4799           memnode_worklist.append_if_missing(use);
4800         } else if (!(op == Op_CmpP || op == Op_Conv2B ||
4801               op == Op_CastP2X ||
4802               op == Op_FastLock || op == Op_AryEq ||
4803               op == Op_StrComp || op == Op_CountPositives ||
4804               op == Op_StrCompressedCopy || op == Op_StrInflatedCopy ||
4805               op == Op_StrEquals || op == Op_VectorizedHashCode ||
4806               op == Op_StrIndexOf || op == Op_StrIndexOfChar ||
4807               op == Op_SubTypeCheck || op == Op_InlineType || op == Op_FlatArrayCheck ||
4808               op == Op_ReinterpretS2HF ||
4809               BarrierSet::barrier_set()->barrier_set_c2()->is_gc_barrier_node(use))) {
4810           n->dump();
4811           use->dump();
4812           assert(false, "EA: missing allocation reference path");
4813         }
4814 #endif
4815       }
4816     }
4817 
4818   }
4819 
4820 #ifdef ASSERT
4821   if (VerifyReduceAllocationMerges) {
4822     for (uint i = 0; i < reducible_merges.size(); i++) {
4823       Node* phi = reducible_merges.at(i);
4824 
4825       if (!reduced_merges.member(phi)) {
4826         phi->dump(2);
4827         phi->dump(-2);

4891       // we don't need to do anything, but the users must be pushed
4892       n = n->as_MemBar()->proj_out_or_null(TypeFunc::Memory);
4893       if (n == nullptr) {
4894         continue;
4895       }
4896     } else if (n->is_CallLeaf()) {
4897       // Runtime calls with narrow memory input (no MergeMem node)
4898       // get the memory projection
4899       n = n->as_Call()->proj_out_or_null(TypeFunc::Memory);
4900       if (n == nullptr) {
4901         continue;
4902       }
4903     } else if (n->Opcode() == Op_StrInflatedCopy) {
4904       // Check direct uses of StrInflatedCopy.
4905       // It is memory type Node - no special SCMemProj node.
4906     } else if (n->Opcode() == Op_StrCompressedCopy ||
4907                n->Opcode() == Op_EncodeISOArray) {
4908       // get the memory projection
4909       n = n->find_out_with(Op_SCMemProj);
4910       assert(n != nullptr && n->Opcode() == Op_SCMemProj, "memory projection required");
4911     } else if (n->is_CallLeaf() && n->as_CallLeaf()->_name != nullptr &&
4912                strcmp(n->as_CallLeaf()->_name, "store_unknown_inline") == 0) {
4913       n = n->as_CallLeaf()->proj_out(TypeFunc::Memory);
4914     } else {
4915 #ifdef ASSERT
4916       if (!n->is_Mem()) {
4917         n->dump();
4918       }
4919       assert(n->is_Mem(), "memory node required.");
4920 #endif
4921       Node *addr = n->in(MemNode::Address);
4922       const Type *addr_t = igvn->type(addr);
4923       if (addr_t == Type::TOP) {
4924         continue;
4925       }
4926       assert (addr_t->isa_ptr() != nullptr, "pointer type required.");
4927       int alias_idx = _compile->get_alias_index(addr_t->is_ptr());
4928       assert ((uint)alias_idx < new_index_end, "wrong alias index");
4929       Node *mem = find_inst_mem(n->in(MemNode::Memory), alias_idx, orig_phis);
4930       if (_compile->failing()) {
4931         return;
4932       }
4933       if (mem != n->in(MemNode::Memory)) {

4938       if (n->is_Load()) {
4939         continue;  // don't push users
4940       } else if (n->is_LoadStore()) {
4941         // get the memory projection
4942         n = n->find_out_with(Op_SCMemProj);
4943         assert(n != nullptr && n->Opcode() == Op_SCMemProj, "memory projection required");
4944       }
4945     }
4946     // push user on appropriate worklist
4947     for (DUIterator_Fast imax, i = n->fast_outs(imax); i < imax; i++) {
4948       Node *use = n->fast_out(i);
4949       if (use->is_Phi() || use->is_ClearArray()) {
4950         memnode_worklist.append_if_missing(use);
4951       } else if (use->is_Mem() && use->in(MemNode::Memory) == n) {
4952         memnode_worklist.append_if_missing(use);
4953       } else if (use->is_MemBar() || use->is_CallLeaf()) {
4954         if (use->in(TypeFunc::Memory) == n) { // Ignore precedent edge
4955           memnode_worklist.append_if_missing(use);
4956         }
4957 #ifdef ASSERT
4958       } else if (use->is_Mem()) {
4959         assert(use->in(MemNode::Memory) != n, "EA: missing memory path");
4960       } else if (use->is_MergeMem()) {
4961         assert(mergemem_worklist.contains(use->as_MergeMem()), "EA: missing MergeMem node in the worklist");
4962       } else if (use->Opcode() == Op_EncodeISOArray) {
4963         if (use->in(MemNode::Memory) == n || use->in(3) == n) {
4964           // EncodeISOArray overwrites destination array
4965           memnode_worklist.append_if_missing(use);
4966         }
4967       } else if (use->is_CallLeaf() && use->as_CallLeaf()->_name != nullptr &&
4968                  strcmp(use->as_CallLeaf()->_name, "store_unknown_inline") == 0) {
4969         // store_unknown_inline overwrites destination array
4970         memnode_worklist.append_if_missing(use);
4971       } else {
4972         uint op = use->Opcode();
4973         if ((use->in(MemNode::Memory) == n) &&
4974             (op == Op_StrCompressedCopy || op == Op_StrInflatedCopy)) {
4975           // They overwrite memory edge corresponding to destination array,
4976           memnode_worklist.append_if_missing(use);
4977         } else if (!(BarrierSet::barrier_set()->barrier_set_c2()->is_gc_barrier_node(use) ||
4978               op == Op_AryEq || op == Op_StrComp || op == Op_CountPositives ||
4979               op == Op_StrCompressedCopy || op == Op_StrInflatedCopy || op == Op_VectorizedHashCode ||
4980               op == Op_StrEquals || op == Op_StrIndexOf || op == Op_StrIndexOfChar || op == Op_FlatArrayCheck)) {
4981           n->dump();
4982           use->dump();
4983           assert(false, "EA: missing memory path");
4984         }
4985 #endif
4986       }
4987     }
4988   }
4989 
4990   //  Phase 3:  Process MergeMem nodes from mergemem_worklist.
4991   //            Walk each memory slice moving the first node encountered of each
4992   //            instance type to the input corresponding to its alias index.
4993   uint length = mergemem_worklist.length();
4994   for( uint next = 0; next < length; ++next ) {
4995     MergeMemNode* nmm = mergemem_worklist.at(next);
4996     assert(!visited.test_set(nmm->_idx), "should not be visited before");
4997     // Note: we don't want to use MergeMemStream here because we only want to
4998     // scan inputs which exist at the start, not ones we add during processing.
4999     // Note 2: MergeMem may already contains instance memory slices added
5000     // during find_inst_mem() call when memory nodes were processed above.

5061     if (_compile->live_nodes() >= _compile->max_node_limit() * 0.75) {
5062       if (_compile->do_reduce_allocation_merges()) {
5063         _compile->record_failure(C2Compiler::retry_no_reduce_allocation_merges());
5064       } else if (_invocation > 0) {
5065         _compile->record_failure(C2Compiler::retry_no_iterative_escape_analysis());
5066       } else {
5067         _compile->record_failure(C2Compiler::retry_no_escape_analysis());
5068       }
5069       return;
5070     }
5071 
5072     igvn->hash_insert(nmm);
5073     record_for_optimizer(nmm);
5074   }
5075 
5076   //  Phase 4:  Update the inputs of non-instance memory Phis and
5077   //            the Memory input of memnodes
5078   // First update the inputs of any non-instance Phi's from
5079   // which we split out an instance Phi.  Note we don't have
5080   // to recursively process Phi's encountered on the input memory
5081   // chains as is done in split_memory_phi() since they will
5082   // also be processed here.
5083   for (int j = 0; j < orig_phis.length(); j++) {
5084     PhiNode *phi = orig_phis.at(j);
5085     int alias_idx = _compile->get_alias_index(phi->adr_type());
5086     igvn->hash_delete(phi);
5087     for (uint i = 1; i < phi->req(); i++) {
5088       Node *mem = phi->in(i);
5089       Node *new_mem = find_inst_mem(mem, alias_idx, orig_phis);
5090       if (_compile->failing()) {
5091         return;
5092       }
5093       if (mem != new_mem) {
5094         phi->set_req(i, new_mem);
5095       }
5096     }
5097     igvn->hash_insert(phi);
5098     record_for_optimizer(phi);
5099   }
5100 
5101   // Update the memory inputs of MemNodes with the value we computed
< prev index next >