1 /*
   2  * Copyright (c) 1997, 2025, Oracle and/or its affiliates. All rights reserved.
   3  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
   4  *
   5  * This code is free software; you can redistribute it and/or modify it
   6  * under the terms of the GNU General Public License version 2 only, as
   7  * published by the Free Software Foundation.
   8  *
   9  * This code is distributed in the hope that it will be useful, but WITHOUT
  10  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  11  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  12  * version 2 for more details (a copy is included in the LICENSE file that
  13  * accompanied this code).
  14  *
  15  * You should have received a copy of the GNU General Public License version
  16  * 2 along with this work; if not, write to the Free Software Foundation,
  17  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
  18  *
  19  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
  20  * or visit www.oracle.com if you need additional information or have any
  21  * questions.
  22  *
  23  */
  24 
  25 #include "asm/macroAssembler.hpp"
  26 #include "asm/macroAssembler.inline.hpp"
  27 #include "ci/ciReplay.hpp"
  28 #include "classfile/javaClasses.hpp"
  29 #include "code/aotCodeCache.hpp"
  30 #include "code/exceptionHandlerTable.hpp"
  31 #include "code/nmethod.hpp"
  32 #include "compiler/compilationFailureInfo.hpp"
  33 #include "compiler/compilationMemoryStatistic.hpp"
  34 #include "compiler/compileBroker.hpp"
  35 #include "compiler/compileLog.hpp"
  36 #include "compiler/compiler_globals.hpp"
  37 #include "compiler/compilerDefinitions.hpp"
  38 #include "compiler/compilerOracle.hpp"
  39 #include "compiler/disassembler.hpp"
  40 #include "compiler/oopMap.hpp"
  41 #include "gc/shared/barrierSet.hpp"
  42 #include "gc/shared/c2/barrierSetC2.hpp"
  43 #include "jfr/jfrEvents.hpp"
  44 #include "jvm_io.h"
  45 #include "memory/allocation.hpp"
  46 #include "memory/arena.hpp"
  47 #include "memory/resourceArea.hpp"
  48 #include "opto/addnode.hpp"
  49 #include "opto/block.hpp"
  50 #include "opto/c2compiler.hpp"
  51 #include "opto/callGenerator.hpp"
  52 #include "opto/callnode.hpp"
  53 #include "opto/castnode.hpp"
  54 #include "opto/cfgnode.hpp"
  55 #include "opto/chaitin.hpp"
  56 #include "opto/compile.hpp"
  57 #include "opto/connode.hpp"
  58 #include "opto/convertnode.hpp"
  59 #include "opto/divnode.hpp"
  60 #include "opto/escape.hpp"
  61 #include "opto/idealGraphPrinter.hpp"
  62 #include "opto/inlinetypenode.hpp"
  63 #include "opto/locknode.hpp"
  64 #include "opto/loopnode.hpp"
  65 #include "opto/machnode.hpp"
  66 #include "opto/macro.hpp"
  67 #include "opto/matcher.hpp"
  68 #include "opto/mathexactnode.hpp"
  69 #include "opto/memnode.hpp"
  70 #include "opto/movenode.hpp"
  71 #include "opto/mulnode.hpp"
  72 #include "opto/narrowptrnode.hpp"
  73 #include "opto/node.hpp"
  74 #include "opto/opaquenode.hpp"
  75 #include "opto/opcodes.hpp"
  76 #include "opto/output.hpp"
  77 #include "opto/parse.hpp"
  78 #include "opto/phaseX.hpp"
  79 #include "opto/rootnode.hpp"
  80 #include "opto/runtime.hpp"
  81 #include "opto/stringopts.hpp"
  82 #include "opto/type.hpp"
  83 #include "opto/vector.hpp"
  84 #include "opto/vectornode.hpp"
  85 #include "runtime/globals_extension.hpp"
  86 #include "runtime/sharedRuntime.hpp"
  87 #include "runtime/signature.hpp"
  88 #include "runtime/stubRoutines.hpp"
  89 #include "runtime/timer.hpp"
  90 #include "utilities/align.hpp"
  91 #include "utilities/copy.hpp"
  92 #include "utilities/hashTable.hpp"
  93 #include "utilities/macros.hpp"
  94 
  95 // -------------------- Compile::mach_constant_base_node -----------------------
  96 // Constant table base node singleton.
  97 MachConstantBaseNode* Compile::mach_constant_base_node() {
  98   if (_mach_constant_base_node == nullptr) {
  99     _mach_constant_base_node = new MachConstantBaseNode();
 100     _mach_constant_base_node->add_req(C->root());
 101   }
 102   return _mach_constant_base_node;
 103 }
 104 
 105 
 106 /// Support for intrinsics.
 107 
 108 // Return the index at which m must be inserted (or already exists).
 109 // The sort order is by the address of the ciMethod, with is_virtual as minor key.
 110 class IntrinsicDescPair {
 111  private:
 112   ciMethod* _m;
 113   bool _is_virtual;
 114  public:
 115   IntrinsicDescPair(ciMethod* m, bool is_virtual) : _m(m), _is_virtual(is_virtual) {}
 116   static int compare(IntrinsicDescPair* const& key, CallGenerator* const& elt) {
 117     ciMethod* m= elt->method();
 118     ciMethod* key_m = key->_m;
 119     if (key_m < m)      return -1;
 120     else if (key_m > m) return 1;
 121     else {
 122       bool is_virtual = elt->is_virtual();
 123       bool key_virtual = key->_is_virtual;
 124       if (key_virtual < is_virtual)      return -1;
 125       else if (key_virtual > is_virtual) return 1;
 126       else                               return 0;
 127     }
 128   }
 129 };
 130 int Compile::intrinsic_insertion_index(ciMethod* m, bool is_virtual, bool& found) {
 131 #ifdef ASSERT
 132   for (int i = 1; i < _intrinsics.length(); i++) {
 133     CallGenerator* cg1 = _intrinsics.at(i-1);
 134     CallGenerator* cg2 = _intrinsics.at(i);
 135     assert(cg1->method() != cg2->method()
 136            ? cg1->method()     < cg2->method()
 137            : cg1->is_virtual() < cg2->is_virtual(),
 138            "compiler intrinsics list must stay sorted");
 139   }
 140 #endif
 141   IntrinsicDescPair pair(m, is_virtual);
 142   return _intrinsics.find_sorted<IntrinsicDescPair*, IntrinsicDescPair::compare>(&pair, found);
 143 }
 144 
 145 void Compile::register_intrinsic(CallGenerator* cg) {
 146   bool found = false;
 147   int index = intrinsic_insertion_index(cg->method(), cg->is_virtual(), found);
 148   assert(!found, "registering twice");
 149   _intrinsics.insert_before(index, cg);
 150   assert(find_intrinsic(cg->method(), cg->is_virtual()) == cg, "registration worked");
 151 }
 152 
 153 CallGenerator* Compile::find_intrinsic(ciMethod* m, bool is_virtual) {
 154   assert(m->is_loaded(), "don't try this on unloaded methods");
 155   if (_intrinsics.length() > 0) {
 156     bool found = false;
 157     int index = intrinsic_insertion_index(m, is_virtual, found);
 158      if (found) {
 159       return _intrinsics.at(index);
 160     }
 161   }
 162   // Lazily create intrinsics for intrinsic IDs well-known in the runtime.
 163   if (m->intrinsic_id() != vmIntrinsics::_none &&
 164       m->intrinsic_id() <= vmIntrinsics::LAST_COMPILER_INLINE) {
 165     CallGenerator* cg = make_vm_intrinsic(m, is_virtual);
 166     if (cg != nullptr) {
 167       // Save it for next time:
 168       register_intrinsic(cg);
 169       return cg;
 170     } else {
 171       gather_intrinsic_statistics(m->intrinsic_id(), is_virtual, _intrinsic_disabled);
 172     }
 173   }
 174   return nullptr;
 175 }
 176 
 177 // Compile::make_vm_intrinsic is defined in library_call.cpp.
 178 
 179 #ifndef PRODUCT
 180 // statistics gathering...
 181 
 182 juint  Compile::_intrinsic_hist_count[vmIntrinsics::number_of_intrinsics()] = {0};
 183 jubyte Compile::_intrinsic_hist_flags[vmIntrinsics::number_of_intrinsics()] = {0};
 184 
 185 inline int as_int(vmIntrinsics::ID id) {
 186   return vmIntrinsics::as_int(id);
 187 }
 188 
 189 bool Compile::gather_intrinsic_statistics(vmIntrinsics::ID id, bool is_virtual, int flags) {
 190   assert(id > vmIntrinsics::_none && id < vmIntrinsics::ID_LIMIT, "oob");
 191   int oflags = _intrinsic_hist_flags[as_int(id)];
 192   assert(flags != 0, "what happened?");
 193   if (is_virtual) {
 194     flags |= _intrinsic_virtual;
 195   }
 196   bool changed = (flags != oflags);
 197   if ((flags & _intrinsic_worked) != 0) {
 198     juint count = (_intrinsic_hist_count[as_int(id)] += 1);
 199     if (count == 1) {
 200       changed = true;           // first time
 201     }
 202     // increment the overall count also:
 203     _intrinsic_hist_count[as_int(vmIntrinsics::_none)] += 1;
 204   }
 205   if (changed) {
 206     if (((oflags ^ flags) & _intrinsic_virtual) != 0) {
 207       // Something changed about the intrinsic's virtuality.
 208       if ((flags & _intrinsic_virtual) != 0) {
 209         // This is the first use of this intrinsic as a virtual call.
 210         if (oflags != 0) {
 211           // We already saw it as a non-virtual, so note both cases.
 212           flags |= _intrinsic_both;
 213         }
 214       } else if ((oflags & _intrinsic_both) == 0) {
 215         // This is the first use of this intrinsic as a non-virtual
 216         flags |= _intrinsic_both;
 217       }
 218     }
 219     _intrinsic_hist_flags[as_int(id)] = (jubyte) (oflags | flags);
 220   }
 221   // update the overall flags also:
 222   _intrinsic_hist_flags[as_int(vmIntrinsics::_none)] |= (jubyte) flags;
 223   return changed;
 224 }
 225 
 226 static char* format_flags(int flags, char* buf) {
 227   buf[0] = 0;
 228   if ((flags & Compile::_intrinsic_worked) != 0)    strcat(buf, ",worked");
 229   if ((flags & Compile::_intrinsic_failed) != 0)    strcat(buf, ",failed");
 230   if ((flags & Compile::_intrinsic_disabled) != 0)  strcat(buf, ",disabled");
 231   if ((flags & Compile::_intrinsic_virtual) != 0)   strcat(buf, ",virtual");
 232   if ((flags & Compile::_intrinsic_both) != 0)      strcat(buf, ",nonvirtual");
 233   if (buf[0] == 0)  strcat(buf, ",");
 234   assert(buf[0] == ',', "must be");
 235   return &buf[1];
 236 }
 237 
 238 void Compile::print_intrinsic_statistics() {
 239   char flagsbuf[100];
 240   ttyLocker ttyl;
 241   if (xtty != nullptr)  xtty->head("statistics type='intrinsic'");
 242   tty->print_cr("Compiler intrinsic usage:");
 243   juint total = _intrinsic_hist_count[as_int(vmIntrinsics::_none)];
 244   if (total == 0)  total = 1;  // avoid div0 in case of no successes
 245   #define PRINT_STAT_LINE(name, c, f) \
 246     tty->print_cr("  %4d (%4.1f%%) %s (%s)", (int)(c), ((c) * 100.0) / total, name, f);
 247   for (auto id : EnumRange<vmIntrinsicID>{}) {
 248     int   flags = _intrinsic_hist_flags[as_int(id)];
 249     juint count = _intrinsic_hist_count[as_int(id)];
 250     if ((flags | count) != 0) {
 251       PRINT_STAT_LINE(vmIntrinsics::name_at(id), count, format_flags(flags, flagsbuf));
 252     }
 253   }
 254   PRINT_STAT_LINE("total", total, format_flags(_intrinsic_hist_flags[as_int(vmIntrinsics::_none)], flagsbuf));
 255   if (xtty != nullptr)  xtty->tail("statistics");
 256 }
 257 
 258 void Compile::print_statistics() {
 259   { ttyLocker ttyl;
 260     if (xtty != nullptr)  xtty->head("statistics type='opto'");
 261     Parse::print_statistics();
 262     PhaseStringOpts::print_statistics();
 263     PhaseCCP::print_statistics();
 264     PhaseRegAlloc::print_statistics();
 265     PhaseOutput::print_statistics();
 266     PhasePeephole::print_statistics();
 267     PhaseIdealLoop::print_statistics();
 268     ConnectionGraph::print_statistics();
 269     PhaseMacroExpand::print_statistics();
 270     if (xtty != nullptr)  xtty->tail("statistics");
 271   }
 272   if (_intrinsic_hist_flags[as_int(vmIntrinsics::_none)] != 0) {
 273     // put this under its own <statistics> element.
 274     print_intrinsic_statistics();
 275   }
 276 }
 277 #endif //PRODUCT
 278 
 279 void Compile::gvn_replace_by(Node* n, Node* nn) {
 280   for (DUIterator_Last imin, i = n->last_outs(imin); i >= imin; ) {
 281     Node* use = n->last_out(i);
 282     bool is_in_table = initial_gvn()->hash_delete(use);
 283     uint uses_found = 0;
 284     for (uint j = 0; j < use->len(); j++) {
 285       if (use->in(j) == n) {
 286         if (j < use->req())
 287           use->set_req(j, nn);
 288         else
 289           use->set_prec(j, nn);
 290         uses_found++;
 291       }
 292     }
 293     if (is_in_table) {
 294       // reinsert into table
 295       initial_gvn()->hash_find_insert(use);
 296     }
 297     record_for_igvn(use);
 298     PhaseIterGVN::add_users_of_use_to_worklist(nn, use, *_igvn_worklist);
 299     i -= uses_found;    // we deleted 1 or more copies of this edge
 300   }
 301 }
 302 
 303 
 304 // Identify all nodes that are reachable from below, useful.
 305 // Use breadth-first pass that records state in a Unique_Node_List,
 306 // recursive traversal is slower.
 307 void Compile::identify_useful_nodes(Unique_Node_List &useful) {
 308   int estimated_worklist_size = live_nodes();
 309   useful.map( estimated_worklist_size, nullptr );  // preallocate space
 310 
 311   // Initialize worklist
 312   if (root() != nullptr)  { useful.push(root()); }
 313   // If 'top' is cached, declare it useful to preserve cached node
 314   if (cached_top_node())  { useful.push(cached_top_node()); }
 315 
 316   // Push all useful nodes onto the list, breadthfirst
 317   for( uint next = 0; next < useful.size(); ++next ) {
 318     assert( next < unique(), "Unique useful nodes < total nodes");
 319     Node *n  = useful.at(next);
 320     uint max = n->len();
 321     for( uint i = 0; i < max; ++i ) {
 322       Node *m = n->in(i);
 323       if (not_a_node(m))  continue;
 324       useful.push(m);
 325     }
 326   }
 327 }
 328 
 329 // Update dead_node_list with any missing dead nodes using useful
 330 // list. Consider all non-useful nodes to be useless i.e., dead nodes.
 331 void Compile::update_dead_node_list(Unique_Node_List &useful) {
 332   uint max_idx = unique();
 333   VectorSet& useful_node_set = useful.member_set();
 334 
 335   for (uint node_idx = 0; node_idx < max_idx; node_idx++) {
 336     // If node with index node_idx is not in useful set,
 337     // mark it as dead in dead node list.
 338     if (!useful_node_set.test(node_idx)) {
 339       record_dead_node(node_idx);
 340     }
 341   }
 342 }
 343 
 344 void Compile::remove_useless_late_inlines(GrowableArray<CallGenerator*>* inlines, Unique_Node_List &useful) {
 345   int shift = 0;
 346   for (int i = 0; i < inlines->length(); i++) {
 347     CallGenerator* cg = inlines->at(i);
 348     if (useful.member(cg->call_node())) {
 349       if (shift > 0) {
 350         inlines->at_put(i - shift, cg);
 351       }
 352     } else {
 353       shift++; // skip over the dead element
 354     }
 355   }
 356   if (shift > 0) {
 357     inlines->trunc_to(inlines->length() - shift); // remove last elements from compacted array
 358   }
 359 }
 360 
 361 void Compile::remove_useless_late_inlines(GrowableArray<CallGenerator*>* inlines, Node* dead) {
 362   assert(dead != nullptr && dead->is_Call(), "sanity");
 363   int found = 0;
 364   for (int i = 0; i < inlines->length(); i++) {
 365     if (inlines->at(i)->call_node() == dead) {
 366       inlines->remove_at(i);
 367       found++;
 368       NOT_DEBUG( break; ) // elements are unique, so exit early
 369     }
 370   }
 371   assert(found <= 1, "not unique");
 372 }
 373 
 374 template<typename N, ENABLE_IF_SDEFN(std::is_base_of<Node, N>::value)>
 375 void Compile::remove_useless_nodes(GrowableArray<N*>& node_list, Unique_Node_List& useful) {
 376   for (int i = node_list.length() - 1; i >= 0; i--) {
 377     N* node = node_list.at(i);
 378     if (!useful.member(node)) {
 379       node_list.delete_at(i); // replaces i-th with last element which is known to be useful (already processed)
 380     }
 381   }
 382 }
 383 
 384 void Compile::remove_useless_node(Node* dead) {
 385   remove_modified_node(dead);
 386 
 387   // Constant node that has no out-edges and has only one in-edge from
 388   // root is usually dead. However, sometimes reshaping walk makes
 389   // it reachable by adding use edges. So, we will NOT count Con nodes
 390   // as dead to be conservative about the dead node count at any
 391   // given time.
 392   if (!dead->is_Con()) {
 393     record_dead_node(dead->_idx);
 394   }
 395   if (dead->is_macro()) {
 396     remove_macro_node(dead);
 397   }
 398   if (dead->is_expensive()) {
 399     remove_expensive_node(dead);
 400   }
 401   if (dead->is_OpaqueTemplateAssertionPredicate()) {
 402     remove_template_assertion_predicate_opaque(dead->as_OpaqueTemplateAssertionPredicate());
 403   }
 404   if (dead->is_ParsePredicate()) {
 405     remove_parse_predicate(dead->as_ParsePredicate());
 406   }
 407   if (dead->for_post_loop_opts_igvn()) {
 408     remove_from_post_loop_opts_igvn(dead);
 409   }
 410   if (dead->is_InlineType()) {
 411     remove_inline_type(dead);
 412   }
 413   if (dead->is_LoadFlat() || dead->is_StoreFlat()) {
 414     remove_flat_access(dead);
 415   }
 416   if (dead->for_merge_stores_igvn()) {
 417     remove_from_merge_stores_igvn(dead);
 418   }
 419   if (dead->is_Call()) {
 420     remove_useless_late_inlines(                &_late_inlines, dead);
 421     remove_useless_late_inlines(         &_string_late_inlines, dead);
 422     remove_useless_late_inlines(         &_boxing_late_inlines, dead);
 423     remove_useless_late_inlines(&_vector_reboxing_late_inlines, dead);
 424 
 425     if (dead->is_CallStaticJava()) {
 426       remove_unstable_if_trap(dead->as_CallStaticJava(), false);
 427     }
 428   }
 429   BarrierSetC2* bs = BarrierSet::barrier_set()->barrier_set_c2();
 430   bs->unregister_potential_barrier_node(dead);
 431 }
 432 
 433 // Disconnect all useless nodes by disconnecting those at the boundary.
 434 void Compile::disconnect_useless_nodes(Unique_Node_List& useful, Unique_Node_List& worklist, const Unique_Node_List* root_and_safepoints) {
 435   uint next = 0;
 436   while (next < useful.size()) {
 437     Node *n = useful.at(next++);
 438     if (n->is_SafePoint()) {
 439       // We're done with a parsing phase. Replaced nodes are not valid
 440       // beyond that point.
 441       n->as_SafePoint()->delete_replaced_nodes();
 442     }
 443     // Use raw traversal of out edges since this code removes out edges
 444     int max = n->outcnt();
 445     for (int j = 0; j < max; ++j) {
 446       Node* child = n->raw_out(j);
 447       if (!useful.member(child)) {
 448         assert(!child->is_top() || child != top(),
 449                "If top is cached in Compile object it is in useful list");
 450         // Only need to remove this out-edge to the useless node
 451         n->raw_del_out(j);
 452         --j;
 453         --max;
 454         if (child->is_data_proj_of_pure_function(n)) {
 455           worklist.push(n);
 456         }
 457       }
 458     }
 459     if (n->outcnt() == 1 && n->has_special_unique_user()) {
 460       assert(useful.member(n->unique_out()), "do not push a useless node");
 461       worklist.push(n->unique_out());
 462     }
 463     if (n->outcnt() == 0) {
 464       worklist.push(n);
 465     }
 466   }
 467 
 468   remove_useless_nodes(_macro_nodes,        useful); // remove useless macro nodes
 469   remove_useless_nodes(_parse_predicates,   useful); // remove useless Parse Predicate nodes
 470   // Remove useless Template Assertion Predicate opaque nodes
 471   remove_useless_nodes(_template_assertion_predicate_opaques, useful);
 472   remove_useless_nodes(_expensive_nodes,    useful); // remove useless expensive nodes
 473   remove_useless_nodes(_for_post_loop_igvn, useful); // remove useless node recorded for post loop opts IGVN pass
 474   remove_useless_nodes(_inline_type_nodes,  useful); // remove useless inline type nodes
 475   remove_useless_nodes(_flat_access_nodes, useful);  // remove useless flat access nodes
 476 #ifdef ASSERT
 477   if (_modified_nodes != nullptr) {
 478     _modified_nodes->remove_useless_nodes(useful.member_set());
 479   }
 480 #endif
 481   remove_useless_nodes(_for_merge_stores_igvn, useful); // remove useless node recorded for merge stores IGVN pass
 482   remove_useless_unstable_if_traps(useful);          // remove useless unstable_if traps
 483   remove_useless_coarsened_locks(useful);            // remove useless coarsened locks nodes
 484 #ifdef ASSERT
 485   if (_modified_nodes != nullptr) {
 486     _modified_nodes->remove_useless_nodes(useful.member_set());
 487   }
 488 #endif
 489 
 490   BarrierSetC2* bs = BarrierSet::barrier_set()->barrier_set_c2();
 491   bs->eliminate_useless_gc_barriers(useful, this);
 492   // clean up the late inline lists
 493   remove_useless_late_inlines(                &_late_inlines, useful);
 494   remove_useless_late_inlines(         &_string_late_inlines, useful);
 495   remove_useless_late_inlines(         &_boxing_late_inlines, useful);
 496   remove_useless_late_inlines(&_vector_reboxing_late_inlines, useful);
 497   DEBUG_ONLY(verify_graph_edges(true /*check for no_dead_code*/, root_and_safepoints);)
 498 }
 499 
 500 // ============================================================================
 501 //------------------------------CompileWrapper---------------------------------
 502 class CompileWrapper : public StackObj {
 503   Compile *const _compile;
 504  public:
 505   CompileWrapper(Compile* compile);
 506 
 507   ~CompileWrapper();
 508 };
 509 
 510 CompileWrapper::CompileWrapper(Compile* compile) : _compile(compile) {
 511   // the Compile* pointer is stored in the current ciEnv:
 512   ciEnv* env = compile->env();
 513   assert(env == ciEnv::current(), "must already be a ciEnv active");
 514   assert(env->compiler_data() == nullptr, "compile already active?");
 515   env->set_compiler_data(compile);
 516   assert(compile == Compile::current(), "sanity");
 517 
 518   compile->set_type_dict(nullptr);
 519   compile->set_clone_map(new Dict(cmpkey, hashkey, _compile->comp_arena()));
 520   compile->clone_map().set_clone_idx(0);
 521   compile->set_type_last_size(0);
 522   compile->set_last_tf(nullptr, nullptr);
 523   compile->set_indexSet_arena(nullptr);
 524   compile->set_indexSet_free_block_list(nullptr);
 525   compile->init_type_arena();
 526   Type::Initialize(compile);
 527   _compile->begin_method();
 528   _compile->clone_map().set_debug(_compile->has_method() && _compile->directive()->CloneMapDebugOption);
 529 }
 530 CompileWrapper::~CompileWrapper() {
 531   // simulate crash during compilation
 532   assert(CICrashAt < 0 || _compile->compile_id() != CICrashAt, "just as planned");
 533 
 534   _compile->end_method();
 535   _compile->env()->set_compiler_data(nullptr);
 536 }
 537 
 538 
 539 //----------------------------print_compile_messages---------------------------
 540 void Compile::print_compile_messages() {
 541 #ifndef PRODUCT
 542   // Check if recompiling
 543   if (!subsume_loads() && PrintOpto) {
 544     // Recompiling without allowing machine instructions to subsume loads
 545     tty->print_cr("*********************************************************");
 546     tty->print_cr("** Bailout: Recompile without subsuming loads          **");
 547     tty->print_cr("*********************************************************");
 548   }
 549   if ((do_escape_analysis() != DoEscapeAnalysis) && PrintOpto) {
 550     // Recompiling without escape analysis
 551     tty->print_cr("*********************************************************");
 552     tty->print_cr("** Bailout: Recompile without escape analysis          **");
 553     tty->print_cr("*********************************************************");
 554   }
 555   if (do_iterative_escape_analysis() != DoEscapeAnalysis && PrintOpto) {
 556     // Recompiling without iterative escape analysis
 557     tty->print_cr("*********************************************************");
 558     tty->print_cr("** Bailout: Recompile without iterative escape analysis**");
 559     tty->print_cr("*********************************************************");
 560   }
 561   if (do_reduce_allocation_merges() != ReduceAllocationMerges && PrintOpto) {
 562     // Recompiling without reducing allocation merges
 563     tty->print_cr("*********************************************************");
 564     tty->print_cr("** Bailout: Recompile without reduce allocation merges **");
 565     tty->print_cr("*********************************************************");
 566   }
 567   if ((eliminate_boxing() != EliminateAutoBox) && PrintOpto) {
 568     // Recompiling without boxing elimination
 569     tty->print_cr("*********************************************************");
 570     tty->print_cr("** Bailout: Recompile without boxing elimination       **");
 571     tty->print_cr("*********************************************************");
 572   }
 573   if ((do_locks_coarsening() != EliminateLocks) && PrintOpto) {
 574     // Recompiling without locks coarsening
 575     tty->print_cr("*********************************************************");
 576     tty->print_cr("** Bailout: Recompile without locks coarsening         **");
 577     tty->print_cr("*********************************************************");
 578   }
 579   if (env()->break_at_compile()) {
 580     // Open the debugger when compiling this method.
 581     tty->print("### Breaking when compiling: ");
 582     method()->print_short_name();
 583     tty->cr();
 584     BREAKPOINT;
 585   }
 586 
 587   if( PrintOpto ) {
 588     if (is_osr_compilation()) {
 589       tty->print("[OSR]%3d", _compile_id);
 590     } else {
 591       tty->print("%3d", _compile_id);
 592     }
 593   }
 594 #endif
 595 }
 596 
 597 #ifndef PRODUCT
 598 void Compile::print_phase(const char* phase_name) {
 599   tty->print_cr("%u.\t%s", ++_phase_counter, phase_name);
 600 }
 601 
 602 void Compile::print_ideal_ir(const char* phase_name) {
 603   // keep the following output all in one block
 604   // This output goes directly to the tty, not the compiler log.
 605   // To enable tools to match it up with the compilation activity,
 606   // be sure to tag this tty output with the compile ID.
 607 
 608   // Node dumping can cause a safepoint, which can break the tty lock.
 609   // Buffer all node dumps, so that all safepoints happen before we lock.
 610   ResourceMark rm;
 611   stringStream ss;
 612 
 613   if (_output == nullptr) {
 614     ss.print_cr("AFTER: %s", phase_name);
 615     // Print out all nodes in ascending order of index.
 616     root()->dump_bfs(MaxNodeLimit, nullptr, "+S$", &ss);
 617   } else {
 618     // Dump the node blockwise if we have a scheduling
 619     _output->print_scheduling(&ss);
 620   }
 621 
 622   // Check that the lock is not broken by a safepoint.
 623   NoSafepointVerifier nsv;
 624   ttyLocker ttyl;
 625   if (xtty != nullptr) {
 626     xtty->head("ideal compile_id='%d'%s compile_phase='%s'",
 627                compile_id(),
 628                is_osr_compilation() ? " compile_kind='osr'" : "",
 629                phase_name);
 630   }
 631 
 632   tty->print("%s", ss.as_string());
 633 
 634   if (xtty != nullptr) {
 635     xtty->tail("ideal");
 636   }
 637 }
 638 #endif
 639 
 640 // ============================================================================
 641 //------------------------------Compile standard-------------------------------
 642 
 643 // Compile a method.  entry_bci is -1 for normal compilations and indicates
 644 // the continuation bci for on stack replacement.
 645 
 646 
 647 Compile::Compile(ciEnv* ci_env, ciMethod* target, int osr_bci,
 648                  Options options, DirectiveSet* directive)
 649     : Phase(Compiler),
 650       _compile_id(ci_env->compile_id()),
 651       _options(options),
 652       _method(target),
 653       _entry_bci(osr_bci),
 654       _ilt(nullptr),
 655       _stub_function(nullptr),
 656       _stub_name(nullptr),
 657       _stub_id(StubId::NO_STUBID),
 658       _stub_entry_point(nullptr),
 659       _max_node_limit(MaxNodeLimit),
 660       _post_loop_opts_phase(false),
 661       _merge_stores_phase(false),
 662       _allow_macro_nodes(true),
 663       _inlining_progress(false),
 664       _inlining_incrementally(false),
 665       _do_cleanup(false),
 666       _has_reserved_stack_access(target->has_reserved_stack_access()),
 667       _has_circular_inline_type(false),
 668 #ifndef PRODUCT
 669       _igv_idx(0),
 670       _trace_opto_output(directive->TraceOptoOutputOption),
 671 #endif
 672       _clinit_barrier_on_entry(false),
 673       _stress_seed(0),
 674       _comp_arena(mtCompiler, Arena::Tag::tag_comp),
 675       _barrier_set_state(BarrierSet::barrier_set()->barrier_set_c2()->create_barrier_state(comp_arena())),
 676       _env(ci_env),
 677       _directive(directive),
 678       _log(ci_env->log()),
 679       _first_failure_details(nullptr),
 680       _intrinsics(comp_arena(), 0, 0, nullptr),
 681       _macro_nodes(comp_arena(), 8, 0, nullptr),
 682       _parse_predicates(comp_arena(), 8, 0, nullptr),
 683       _template_assertion_predicate_opaques(comp_arena(), 8, 0, nullptr),
 684       _expensive_nodes(comp_arena(), 8, 0, nullptr),
 685       _for_post_loop_igvn(comp_arena(), 8, 0, nullptr),
 686       _inline_type_nodes (comp_arena(), 8, 0, nullptr),
 687       _flat_access_nodes(comp_arena(), 8, 0, nullptr),
 688       _for_merge_stores_igvn(comp_arena(), 8, 0, nullptr),
 689       _unstable_if_traps(comp_arena(), 8, 0, nullptr),
 690       _coarsened_locks(comp_arena(), 8, 0, nullptr),
 691       _congraph(nullptr),
 692       NOT_PRODUCT(_igv_printer(nullptr) COMMA)
 693           _unique(0),
 694       _dead_node_count(0),
 695       _dead_node_list(comp_arena()),
 696       _node_arena_one(mtCompiler, Arena::Tag::tag_node),
 697       _node_arena_two(mtCompiler, Arena::Tag::tag_node),
 698       _node_arena(&_node_arena_one),
 699       _mach_constant_base_node(nullptr),
 700       _Compile_types(mtCompiler, Arena::Tag::tag_type),
 701       _initial_gvn(nullptr),
 702       _igvn_worklist(nullptr),
 703       _types(nullptr),
 704       _node_hash(nullptr),
 705       _late_inlines(comp_arena(), 2, 0, nullptr),
 706       _string_late_inlines(comp_arena(), 2, 0, nullptr),
 707       _boxing_late_inlines(comp_arena(), 2, 0, nullptr),
 708       _vector_reboxing_late_inlines(comp_arena(), 2, 0, nullptr),
 709       _late_inlines_pos(0),
 710       _number_of_mh_late_inlines(0),
 711       _oom(false),
 712       _replay_inline_data(nullptr),
 713       _inline_printer(this),
 714       _java_calls(0),
 715       _inner_loops(0),
 716       _FIRST_STACK_mask(comp_arena()),
 717       _interpreter_frame_size(0),
 718       _regmask_arena(mtCompiler, Arena::Tag::tag_regmask),
 719       _output(nullptr)
 720 #ifndef PRODUCT
 721       ,
 722       _in_dump_cnt(0)
 723 #endif
 724 {
 725   C = this;
 726   CompileWrapper cw(this);
 727 
 728   TraceTime t1("Total compilation time", &_t_totalCompilation, CITime, CITimeVerbose);
 729   TraceTime t2(nullptr, &_t_methodCompilation, CITime, false);
 730 
 731 #if defined(SUPPORT_ASSEMBLY) || defined(SUPPORT_ABSTRACT_ASSEMBLY)
 732   bool print_opto_assembly = directive->PrintOptoAssemblyOption;
 733   // We can always print a disassembly, either abstract (hex dump) or
 734   // with the help of a suitable hsdis library. Thus, we should not
 735   // couple print_assembly and print_opto_assembly controls.
 736   // But: always print opto and regular assembly on compile command 'print'.
 737   bool print_assembly = directive->PrintAssemblyOption;
 738   set_print_assembly(print_opto_assembly || print_assembly);
 739 #else
 740   set_print_assembly(false); // must initialize.
 741 #endif
 742 
 743 #ifndef PRODUCT
 744   set_parsed_irreducible_loop(false);
 745 #endif
 746 
 747   if (directive->ReplayInlineOption) {
 748     _replay_inline_data = ciReplay::load_inline_data(method(), entry_bci(), ci_env->comp_level());
 749   }
 750   set_print_inlining(directive->PrintInliningOption || PrintOptoInlining);
 751   set_print_intrinsics(directive->PrintIntrinsicsOption);
 752   set_has_irreducible_loop(true); // conservative until build_loop_tree() reset it
 753 
 754   if (ProfileTraps) {
 755     // Make sure the method being compiled gets its own MDO,
 756     // so we can at least track the decompile_count().
 757     method()->ensure_method_data();
 758   }
 759 
 760   if (StressLCM || StressGCM || StressIGVN || StressCCP ||
 761       StressIncrementalInlining || StressMacroExpansion ||
 762       StressMacroElimination || StressUnstableIfTraps ||
 763       StressBailout || StressLoopPeeling) {
 764     initialize_stress_seed(directive);
 765   }
 766 
 767   Init(/*do_aliasing=*/ true);
 768 
 769   print_compile_messages();
 770 
 771   _ilt = InlineTree::build_inline_tree_root();
 772 
 773   // Even if NO memory addresses are used, MergeMem nodes must have at least 1 slice
 774   assert(num_alias_types() >= AliasIdxRaw, "");
 775 
 776 #define MINIMUM_NODE_HASH  1023
 777 
 778   // GVN that will be run immediately on new nodes
 779   uint estimated_size = method()->code_size()*4+64;
 780   estimated_size = (estimated_size < MINIMUM_NODE_HASH ? MINIMUM_NODE_HASH : estimated_size);
 781   _igvn_worklist = new (comp_arena()) Unique_Node_List(comp_arena());
 782   _types = new (comp_arena()) Type_Array(comp_arena());
 783   _node_hash = new (comp_arena()) NodeHash(comp_arena(), estimated_size);
 784   PhaseGVN gvn;
 785   set_initial_gvn(&gvn);
 786 
 787   { // Scope for timing the parser
 788     TracePhase tp(_t_parser);
 789 
 790     // Put top into the hash table ASAP.
 791     initial_gvn()->transform(top());
 792 
 793     // Set up tf(), start(), and find a CallGenerator.
 794     CallGenerator* cg = nullptr;
 795     if (is_osr_compilation()) {
 796       init_tf(TypeFunc::make(method(), /* is_osr_compilation = */ true));
 797       StartNode* s = new StartOSRNode(root(), tf()->domain_sig());
 798       initial_gvn()->set_type_bottom(s);
 799       verify_start(s);
 800       cg = CallGenerator::for_osr(method(), entry_bci());
 801     } else {
 802       // Normal case.
 803       init_tf(TypeFunc::make(method()));
 804       StartNode* s = new StartNode(root(), tf()->domain_cc());
 805       initial_gvn()->set_type_bottom(s);
 806       verify_start(s);
 807       float past_uses = method()->interpreter_invocation_count();
 808       float expected_uses = past_uses;
 809       cg = CallGenerator::for_inline(method(), expected_uses);
 810     }
 811     if (failing())  return;
 812     if (cg == nullptr) {
 813       const char* reason = InlineTree::check_can_parse(method());
 814       assert(reason != nullptr, "expect reason for parse failure");
 815       stringStream ss;
 816       ss.print("cannot parse method: %s", reason);
 817       record_method_not_compilable(ss.as_string());
 818       return;
 819     }
 820 
 821     gvn.set_type(root(), root()->bottom_type());
 822 
 823     JVMState* jvms = build_start_state(start(), tf());
 824     if ((jvms = cg->generate(jvms)) == nullptr) {
 825       assert(failure_reason() != nullptr, "expect reason for parse failure");
 826       stringStream ss;
 827       ss.print("method parse failed: %s", failure_reason());
 828       record_method_not_compilable(ss.as_string() DEBUG_ONLY(COMMA true));
 829       return;
 830     }
 831     GraphKit kit(jvms);
 832 
 833     if (!kit.stopped()) {
 834       // Accept return values, and transfer control we know not where.
 835       // This is done by a special, unique ReturnNode bound to root.
 836       return_values(kit.jvms());
 837     }
 838 
 839     if (kit.has_exceptions()) {
 840       // Any exceptions that escape from this call must be rethrown
 841       // to whatever caller is dynamically above us on the stack.
 842       // This is done by a special, unique RethrowNode bound to root.
 843       rethrow_exceptions(kit.transfer_exceptions_into_jvms());
 844     }
 845 
 846     assert(IncrementalInline || (_late_inlines.length() == 0 && !has_mh_late_inlines()), "incremental inlining is off");
 847 
 848     if (_late_inlines.length() == 0 && !has_mh_late_inlines() && !failing() && has_stringbuilder()) {
 849       inline_string_calls(true);
 850     }
 851 
 852     if (failing())  return;
 853 
 854     // Remove clutter produced by parsing.
 855     if (!failing()) {
 856       ResourceMark rm;
 857       PhaseRemoveUseless pru(initial_gvn(), *igvn_worklist());
 858     }
 859   }
 860 
 861   // Note:  Large methods are capped off in do_one_bytecode().
 862   if (failing())  return;
 863 
 864   // After parsing, node notes are no longer automagic.
 865   // They must be propagated by register_new_node_with_optimizer(),
 866   // clone(), or the like.
 867   set_default_node_notes(nullptr);
 868 
 869 #ifndef PRODUCT
 870   if (should_print_igv(1)) {
 871     _igv_printer->print_inlining();
 872   }
 873 #endif
 874 
 875   if (failing())  return;
 876   NOT_PRODUCT( verify_graph_edges(); )
 877 
 878   // Now optimize
 879   Optimize();
 880   if (failing())  return;
 881   NOT_PRODUCT( verify_graph_edges(); )
 882 
 883 #ifndef PRODUCT
 884   if (should_print_ideal()) {
 885     print_ideal_ir("print_ideal");
 886   }
 887 #endif
 888 
 889 #ifdef ASSERT
 890   BarrierSetC2* bs = BarrierSet::barrier_set()->barrier_set_c2();
 891   bs->verify_gc_barriers(this, BarrierSetC2::BeforeCodeGen);
 892 #endif
 893 
 894   // Dump compilation data to replay it.
 895   if (directive->DumpReplayOption) {
 896     env()->dump_replay_data(_compile_id);
 897   }
 898   if (directive->DumpInlineOption && (ilt() != nullptr)) {
 899     env()->dump_inline_data(_compile_id);
 900   }
 901 
 902   // Now that we know the size of all the monitors we can add a fixed slot
 903   // for the original deopt pc.
 904   int next_slot = fixed_slots() + (sizeof(address) / VMRegImpl::stack_slot_size);
 905   if (needs_stack_repair()) {
 906     // One extra slot for the special stack increment value
 907     next_slot += 2;
 908   }
 909   // TODO 8284443 Only reserve extra slot if needed
 910   if (InlineTypeReturnedAsFields) {
 911     // One extra slot to hold the null marker for a nullable
 912     // inline type return if we run out of registers.
 913     next_slot += 2;
 914   }
 915   set_fixed_slots(next_slot);
 916 
 917   // Compute when to use implicit null checks. Used by matching trap based
 918   // nodes and NullCheck optimization.
 919   set_allowed_deopt_reasons();
 920 
 921   // Now generate code
 922   Code_Gen();
 923 }
 924 
 925 //------------------------------Compile----------------------------------------
 926 // Compile a runtime stub
 927 Compile::Compile(ciEnv* ci_env,
 928                  TypeFunc_generator generator,
 929                  address stub_function,
 930                  const char* stub_name,
 931                  StubId stub_id,
 932                  int is_fancy_jump,
 933                  bool pass_tls,
 934                  bool return_pc,
 935                  DirectiveSet* directive)
 936     : Phase(Compiler),
 937       _compile_id(0),
 938       _options(Options::for_runtime_stub()),
 939       _method(nullptr),
 940       _entry_bci(InvocationEntryBci),
 941       _stub_function(stub_function),
 942       _stub_name(stub_name),
 943       _stub_id(stub_id),
 944       _stub_entry_point(nullptr),
 945       _max_node_limit(MaxNodeLimit),
 946       _post_loop_opts_phase(false),
 947       _merge_stores_phase(false),
 948       _allow_macro_nodes(true),
 949       _inlining_progress(false),
 950       _inlining_incrementally(false),
 951       _has_reserved_stack_access(false),
 952       _has_circular_inline_type(false),
 953 #ifndef PRODUCT
 954       _igv_idx(0),
 955       _trace_opto_output(directive->TraceOptoOutputOption),
 956 #endif
 957       _clinit_barrier_on_entry(false),
 958       _stress_seed(0),
 959       _comp_arena(mtCompiler, Arena::Tag::tag_comp),
 960       _barrier_set_state(BarrierSet::barrier_set()->barrier_set_c2()->create_barrier_state(comp_arena())),
 961       _env(ci_env),
 962       _directive(directive),
 963       _log(ci_env->log()),
 964       _first_failure_details(nullptr),
 965       _for_post_loop_igvn(comp_arena(), 8, 0, nullptr),
 966       _for_merge_stores_igvn(comp_arena(), 8, 0, nullptr),
 967       _congraph(nullptr),
 968       NOT_PRODUCT(_igv_printer(nullptr) COMMA)
 969           _unique(0),
 970       _dead_node_count(0),
 971       _dead_node_list(comp_arena()),
 972       _node_arena_one(mtCompiler, Arena::Tag::tag_node),
 973       _node_arena_two(mtCompiler, Arena::Tag::tag_node),
 974       _node_arena(&_node_arena_one),
 975       _mach_constant_base_node(nullptr),
 976       _Compile_types(mtCompiler, Arena::Tag::tag_type),
 977       _initial_gvn(nullptr),
 978       _igvn_worklist(nullptr),
 979       _types(nullptr),
 980       _node_hash(nullptr),
 981       _number_of_mh_late_inlines(0),
 982       _oom(false),
 983       _replay_inline_data(nullptr),
 984       _inline_printer(this),
 985       _java_calls(0),
 986       _inner_loops(0),
 987       _FIRST_STACK_mask(comp_arena()),
 988       _interpreter_frame_size(0),
 989       _regmask_arena(mtCompiler, Arena::Tag::tag_regmask),
 990       _output(nullptr),
 991 #ifndef PRODUCT
 992       _in_dump_cnt(0),
 993 #endif
 994       _allowed_reasons(0) {
 995   C = this;
 996 
 997   // try to reuse an existing stub
 998   {
 999     BlobId blob_id = StubInfo::blob(_stub_id);
1000     CodeBlob* blob = AOTCodeCache::load_code_blob(AOTCodeEntry::C2Blob, blob_id);
1001     if (blob != nullptr) {
1002       RuntimeStub* rs = blob->as_runtime_stub();
1003       _stub_entry_point = rs->entry_point();
1004       return;
1005     }
1006   }
1007 
1008   TraceTime t1(nullptr, &_t_totalCompilation, CITime, false);
1009   TraceTime t2(nullptr, &_t_stubCompilation, CITime, false);
1010 
1011 #ifndef PRODUCT
1012   set_print_assembly(PrintFrameConverterAssembly);
1013   set_parsed_irreducible_loop(false);
1014 #else
1015   set_print_assembly(false); // Must initialize.
1016 #endif
1017   set_has_irreducible_loop(false); // no loops
1018 
1019   CompileWrapper cw(this);
1020   Init(/*do_aliasing=*/ false);
1021   init_tf((*generator)());
1022 
1023   _igvn_worklist = new (comp_arena()) Unique_Node_List(comp_arena());
1024   _types = new (comp_arena()) Type_Array(comp_arena());
1025   _node_hash = new (comp_arena()) NodeHash(comp_arena(), 255);
1026 
1027   if (StressLCM || StressGCM || StressBailout) {
1028     initialize_stress_seed(directive);
1029   }
1030 
1031   {
1032     PhaseGVN gvn;
1033     set_initial_gvn(&gvn);    // not significant, but GraphKit guys use it pervasively
1034     gvn.transform(top());
1035 
1036     GraphKit kit;
1037     kit.gen_stub(stub_function, stub_name, is_fancy_jump, pass_tls, return_pc);
1038   }
1039 
1040   NOT_PRODUCT( verify_graph_edges(); )
1041 
1042   Code_Gen();
1043 }
1044 
1045 Compile::~Compile() {
1046   delete _first_failure_details;
1047 };
1048 
1049 //------------------------------Init-------------------------------------------
1050 // Prepare for a single compilation
1051 void Compile::Init(bool aliasing) {
1052   _do_aliasing = aliasing;
1053   _unique  = 0;
1054   _regalloc = nullptr;
1055 
1056   _tf      = nullptr;  // filled in later
1057   _top     = nullptr;  // cached later
1058   _matcher = nullptr;  // filled in later
1059   _cfg     = nullptr;  // filled in later
1060 
1061   _node_note_array = nullptr;
1062   _default_node_notes = nullptr;
1063   DEBUG_ONLY( _modified_nodes = nullptr; ) // Used in Optimize()
1064 
1065   _immutable_memory = nullptr; // filled in at first inquiry
1066 
1067 #ifdef ASSERT
1068   _phase_optimize_finished = false;
1069   _phase_verify_ideal_loop = false;
1070   _exception_backedge = false;
1071   _type_verify = nullptr;
1072 #endif
1073 
1074   // Globally visible Nodes
1075   // First set TOP to null to give safe behavior during creation of RootNode
1076   set_cached_top_node(nullptr);
1077   set_root(new RootNode());
1078   // Now that you have a Root to point to, create the real TOP
1079   set_cached_top_node( new ConNode(Type::TOP) );
1080   set_recent_alloc(nullptr, nullptr);
1081 
1082   // Create Debug Information Recorder to record scopes, oopmaps, etc.
1083   env()->set_oop_recorder(new OopRecorder(env()->arena()));
1084   env()->set_debug_info(new DebugInformationRecorder(env()->oop_recorder()));
1085   env()->set_dependencies(new Dependencies(env()));
1086 
1087   _fixed_slots = 0;
1088   set_has_split_ifs(false);
1089   set_has_loops(false); // first approximation
1090   set_has_stringbuilder(false);
1091   set_has_boxed_value(false);
1092   _trap_can_recompile = false;  // no traps emitted yet
1093   _major_progress = true; // start out assuming good things will happen
1094   set_has_unsafe_access(false);
1095   set_max_vector_size(0);
1096   set_clear_upper_avx(false);  //false as default for clear upper bits of ymm registers
1097   Copy::zero_to_bytes(_trap_hist, sizeof(_trap_hist));
1098   set_decompile_count(0);
1099 
1100 #ifndef PRODUCT
1101   _phase_counter = 0;
1102   Copy::zero_to_bytes(_igv_phase_iter, sizeof(_igv_phase_iter));
1103 #endif
1104 
1105   set_do_freq_based_layout(_directive->BlockLayoutByFrequencyOption);
1106   _loop_opts_cnt = LoopOptsCount;
1107   _has_flat_accesses = false;
1108   _flat_accesses_share_alias = true;
1109   _scalarize_in_safepoints = false;
1110 
1111   set_do_inlining(Inline);
1112   set_max_inline_size(MaxInlineSize);
1113   set_freq_inline_size(FreqInlineSize);
1114   set_do_scheduling(OptoScheduling);
1115 
1116   set_do_vector_loop(false);
1117   set_has_monitors(false);
1118   set_has_scoped_access(false);
1119 
1120   if (AllowVectorizeOnDemand) {
1121     if (has_method() && _directive->VectorizeOption) {
1122       set_do_vector_loop(true);
1123       NOT_PRODUCT(if (do_vector_loop() && Verbose) {tty->print("Compile::Init: do vectorized loops (SIMD like) for method %s\n",  method()->name()->as_quoted_ascii());})
1124     } else if (has_method() && method()->name() != nullptr &&
1125                method()->intrinsic_id() == vmIntrinsics::_forEachRemaining) {
1126       set_do_vector_loop(true);
1127     }
1128   }
1129   set_use_cmove(UseCMoveUnconditionally /* || do_vector_loop()*/); //TODO: consider do_vector_loop() mandate use_cmove unconditionally
1130   NOT_PRODUCT(if (use_cmove() && Verbose && has_method()) {tty->print("Compile::Init: use CMove without profitability tests for method %s\n",  method()->name()->as_quoted_ascii());})
1131 
1132   _max_node_limit = _directive->MaxNodeLimitOption;
1133 
1134   if (VM_Version::supports_fast_class_init_checks() && has_method() && !is_osr_compilation() && method()->needs_clinit_barrier()) {
1135     set_clinit_barrier_on_entry(true);
1136   }
1137   if (debug_info()->recording_non_safepoints()) {
1138     set_node_note_array(new(comp_arena()) GrowableArray<Node_Notes*>
1139                         (comp_arena(), 8, 0, nullptr));
1140     set_default_node_notes(Node_Notes::make(this));
1141   }
1142 
1143   const int grow_ats = 16;
1144   _max_alias_types = grow_ats;
1145   _alias_types   = NEW_ARENA_ARRAY(comp_arena(), AliasType*, grow_ats);
1146   AliasType* ats = NEW_ARENA_ARRAY(comp_arena(), AliasType,  grow_ats);
1147   Copy::zero_to_bytes(ats, sizeof(AliasType)*grow_ats);
1148   {
1149     for (int i = 0; i < grow_ats; i++)  _alias_types[i] = &ats[i];
1150   }
1151   // Initialize the first few types.
1152   _alias_types[AliasIdxTop]->Init(AliasIdxTop, nullptr);
1153   _alias_types[AliasIdxBot]->Init(AliasIdxBot, TypePtr::BOTTOM);
1154   _alias_types[AliasIdxRaw]->Init(AliasIdxRaw, TypeRawPtr::BOTTOM);
1155   _num_alias_types = AliasIdxRaw+1;
1156   // Zero out the alias type cache.
1157   Copy::zero_to_bytes(_alias_cache, sizeof(_alias_cache));
1158   // A null adr_type hits in the cache right away.  Preload the right answer.
1159   probe_alias_cache(nullptr)->_index = AliasIdxTop;
1160 }
1161 
1162 #ifdef ASSERT
1163 // Verify that the current StartNode is valid.
1164 void Compile::verify_start(StartNode* s) const {
1165   assert(failing_internal() || s == start(), "should be StartNode");
1166 }
1167 #endif
1168 
1169 /**
1170  * Return the 'StartNode'. We must not have a pending failure, since the ideal graph
1171  * can be in an inconsistent state, i.e., we can get segmentation faults when traversing
1172  * the ideal graph.
1173  */
1174 StartNode* Compile::start() const {
1175   assert (!failing_internal() || C->failure_is_artificial(), "Must not have pending failure. Reason is: %s", failure_reason());
1176   for (DUIterator_Fast imax, i = root()->fast_outs(imax); i < imax; i++) {
1177     Node* start = root()->fast_out(i);
1178     if (start->is_Start()) {
1179       return start->as_Start();
1180     }
1181   }
1182   fatal("Did not find Start node!");
1183   return nullptr;
1184 }
1185 
1186 //-------------------------------immutable_memory-------------------------------------
1187 // Access immutable memory
1188 Node* Compile::immutable_memory() {
1189   if (_immutable_memory != nullptr) {
1190     return _immutable_memory;
1191   }
1192   StartNode* s = start();
1193   for (DUIterator_Fast imax, i = s->fast_outs(imax); true; i++) {
1194     Node *p = s->fast_out(i);
1195     if (p != s && p->as_Proj()->_con == TypeFunc::Memory) {
1196       _immutable_memory = p;
1197       return _immutable_memory;
1198     }
1199   }
1200   ShouldNotReachHere();
1201   return nullptr;
1202 }
1203 
1204 //----------------------set_cached_top_node------------------------------------
1205 // Install the cached top node, and make sure Node::is_top works correctly.
1206 void Compile::set_cached_top_node(Node* tn) {
1207   if (tn != nullptr)  verify_top(tn);
1208   Node* old_top = _top;
1209   _top = tn;
1210   // Calling Node::setup_is_top allows the nodes the chance to adjust
1211   // their _out arrays.
1212   if (_top != nullptr)     _top->setup_is_top();
1213   if (old_top != nullptr)  old_top->setup_is_top();
1214   assert(_top == nullptr || top()->is_top(), "");
1215 }
1216 
1217 #ifdef ASSERT
1218 uint Compile::count_live_nodes_by_graph_walk() {
1219   Unique_Node_List useful(comp_arena());
1220   // Get useful node list by walking the graph.
1221   identify_useful_nodes(useful);
1222   return useful.size();
1223 }
1224 
1225 void Compile::print_missing_nodes() {
1226 
1227   // Return if CompileLog is null and PrintIdealNodeCount is false.
1228   if ((_log == nullptr) && (! PrintIdealNodeCount)) {
1229     return;
1230   }
1231 
1232   // This is an expensive function. It is executed only when the user
1233   // specifies VerifyIdealNodeCount option or otherwise knows the
1234   // additional work that needs to be done to identify reachable nodes
1235   // by walking the flow graph and find the missing ones using
1236   // _dead_node_list.
1237 
1238   Unique_Node_List useful(comp_arena());
1239   // Get useful node list by walking the graph.
1240   identify_useful_nodes(useful);
1241 
1242   uint l_nodes = C->live_nodes();
1243   uint l_nodes_by_walk = useful.size();
1244 
1245   if (l_nodes != l_nodes_by_walk) {
1246     if (_log != nullptr) {
1247       _log->begin_head("mismatched_nodes count='%d'", abs((int) (l_nodes - l_nodes_by_walk)));
1248       _log->stamp();
1249       _log->end_head();
1250     }
1251     VectorSet& useful_member_set = useful.member_set();
1252     int last_idx = l_nodes_by_walk;
1253     for (int i = 0; i < last_idx; i++) {
1254       if (useful_member_set.test(i)) {
1255         if (_dead_node_list.test(i)) {
1256           if (_log != nullptr) {
1257             _log->elem("mismatched_node_info node_idx='%d' type='both live and dead'", i);
1258           }
1259           if (PrintIdealNodeCount) {
1260             // Print the log message to tty
1261               tty->print_cr("mismatched_node idx='%d' both live and dead'", i);
1262               useful.at(i)->dump();
1263           }
1264         }
1265       }
1266       else if (! _dead_node_list.test(i)) {
1267         if (_log != nullptr) {
1268           _log->elem("mismatched_node_info node_idx='%d' type='neither live nor dead'", i);
1269         }
1270         if (PrintIdealNodeCount) {
1271           // Print the log message to tty
1272           tty->print_cr("mismatched_node idx='%d' type='neither live nor dead'", i);
1273         }
1274       }
1275     }
1276     if (_log != nullptr) {
1277       _log->tail("mismatched_nodes");
1278     }
1279   }
1280 }
1281 void Compile::record_modified_node(Node* n) {
1282   if (_modified_nodes != nullptr && !_inlining_incrementally && !n->is_Con()) {
1283     _modified_nodes->push(n);
1284   }
1285 }
1286 
1287 void Compile::remove_modified_node(Node* n) {
1288   if (_modified_nodes != nullptr) {
1289     _modified_nodes->remove(n);
1290   }
1291 }
1292 #endif
1293 
1294 #ifndef PRODUCT
1295 void Compile::verify_top(Node* tn) const {
1296   if (tn != nullptr) {
1297     assert(tn->is_Con(), "top node must be a constant");
1298     assert(((ConNode*)tn)->type() == Type::TOP, "top node must have correct type");
1299     assert(tn->in(0) != nullptr, "must have live top node");
1300   }
1301 }
1302 #endif
1303 
1304 
1305 ///-------------------Managing Per-Node Debug & Profile Info-------------------
1306 
1307 void Compile::grow_node_notes(GrowableArray<Node_Notes*>* arr, int grow_by) {
1308   guarantee(arr != nullptr, "");
1309   int num_blocks = arr->length();
1310   if (grow_by < num_blocks)  grow_by = num_blocks;
1311   int num_notes = grow_by * _node_notes_block_size;
1312   Node_Notes* notes = NEW_ARENA_ARRAY(node_arena(), Node_Notes, num_notes);
1313   Copy::zero_to_bytes(notes, num_notes * sizeof(Node_Notes));
1314   while (num_notes > 0) {
1315     arr->append(notes);
1316     notes     += _node_notes_block_size;
1317     num_notes -= _node_notes_block_size;
1318   }
1319   assert(num_notes == 0, "exact multiple, please");
1320 }
1321 
1322 bool Compile::copy_node_notes_to(Node* dest, Node* source) {
1323   if (source == nullptr || dest == nullptr)  return false;
1324 
1325   if (dest->is_Con())
1326     return false;               // Do not push debug info onto constants.
1327 
1328 #ifdef ASSERT
1329   // Leave a bread crumb trail pointing to the original node:
1330   if (dest != nullptr && dest != source && dest->debug_orig() == nullptr) {
1331     dest->set_debug_orig(source);
1332   }
1333 #endif
1334 
1335   if (node_note_array() == nullptr)
1336     return false;               // Not collecting any notes now.
1337 
1338   // This is a copy onto a pre-existing node, which may already have notes.
1339   // If both nodes have notes, do not overwrite any pre-existing notes.
1340   Node_Notes* source_notes = node_notes_at(source->_idx);
1341   if (source_notes == nullptr || source_notes->is_clear())  return false;
1342   Node_Notes* dest_notes   = node_notes_at(dest->_idx);
1343   if (dest_notes == nullptr || dest_notes->is_clear()) {
1344     return set_node_notes_at(dest->_idx, source_notes);
1345   }
1346 
1347   Node_Notes merged_notes = (*source_notes);
1348   // The order of operations here ensures that dest notes will win...
1349   merged_notes.update_from(dest_notes);
1350   return set_node_notes_at(dest->_idx, &merged_notes);
1351 }
1352 
1353 
1354 //--------------------------allow_range_check_smearing-------------------------
1355 // Gating condition for coalescing similar range checks.
1356 // Sometimes we try 'speculatively' replacing a series of a range checks by a
1357 // single covering check that is at least as strong as any of them.
1358 // If the optimization succeeds, the simplified (strengthened) range check
1359 // will always succeed.  If it fails, we will deopt, and then give up
1360 // on the optimization.
1361 bool Compile::allow_range_check_smearing() const {
1362   // If this method has already thrown a range-check,
1363   // assume it was because we already tried range smearing
1364   // and it failed.
1365   uint already_trapped = trap_count(Deoptimization::Reason_range_check);
1366   return !already_trapped;
1367 }
1368 
1369 
1370 //------------------------------flatten_alias_type-----------------------------
1371 const TypePtr *Compile::flatten_alias_type( const TypePtr *tj ) const {
1372   assert(do_aliasing(), "Aliasing should be enabled");
1373   int offset = tj->offset();
1374   TypePtr::PTR ptr = tj->ptr();
1375 
1376   // Known instance (scalarizable allocation) alias only with itself.
1377   bool is_known_inst = tj->isa_oopptr() != nullptr &&
1378                        tj->is_oopptr()->is_known_instance();
1379 
1380   // Process weird unsafe references.
1381   if (offset == Type::OffsetBot && (tj->isa_instptr() /*|| tj->isa_klassptr()*/)) {
1382     assert(InlineUnsafeOps || StressReflectiveCode, "indeterminate pointers come only from unsafe ops");
1383     assert(!is_known_inst, "scalarizable allocation should not have unsafe references");
1384     tj = TypeOopPtr::BOTTOM;
1385     ptr = tj->ptr();
1386     offset = tj->offset();
1387   }
1388 
1389   // Array pointers need some flattening
1390   const TypeAryPtr* ta = tj->isa_aryptr();
1391   if (ta && ta->is_stable()) {
1392     // Erase stability property for alias analysis.
1393     tj = ta = ta->cast_to_stable(false);
1394   }
1395   if (ta && ta->is_not_flat()) {
1396     // Erase not flat property for alias analysis.
1397     tj = ta = ta->cast_to_not_flat(false);
1398   }
1399   if (ta && ta->is_not_null_free()) {
1400     // Erase not null free property for alias analysis.
1401     tj = ta = ta->cast_to_not_null_free(false);
1402   }
1403 
1404   if( ta && is_known_inst ) {
1405     if ( offset != Type::OffsetBot &&
1406          offset > arrayOopDesc::length_offset_in_bytes() ) {
1407       offset = Type::OffsetBot; // Flatten constant access into array body only
1408       tj = ta = ta->
1409               remove_speculative()->
1410               cast_to_ptr_type(ptr)->
1411               with_offset(offset);
1412     }
1413   } else if (ta) {
1414     // For arrays indexed by constant indices, we flatten the alias
1415     // space to include all of the array body.  Only the header, klass
1416     // and array length can be accessed un-aliased.
1417     // For flat inline type array, each field has its own slice so
1418     // we must include the field offset.
1419     if( offset != Type::OffsetBot ) {
1420       if( ta->const_oop() ) { // MethodData* or Method*
1421         offset = Type::OffsetBot;   // Flatten constant access into array body
1422         tj = ta = ta->
1423                 remove_speculative()->
1424                 cast_to_ptr_type(ptr)->
1425                 cast_to_exactness(false)->
1426                 with_offset(offset);
1427       } else if( offset == arrayOopDesc::length_offset_in_bytes() ) {
1428         // range is OK as-is.
1429         tj = ta = TypeAryPtr::RANGE;
1430       } else if( offset == oopDesc::klass_offset_in_bytes() ) {
1431         tj = TypeInstPtr::KLASS; // all klass loads look alike
1432         ta = TypeAryPtr::RANGE; // generic ignored junk
1433         ptr = TypePtr::BotPTR;
1434       } else if( offset == oopDesc::mark_offset_in_bytes() ) {
1435         tj = TypeInstPtr::MARK;
1436         ta = TypeAryPtr::RANGE; // generic ignored junk
1437         ptr = TypePtr::BotPTR;
1438       } else {                  // Random constant offset into array body
1439         offset = Type::OffsetBot;   // Flatten constant access into array body
1440         tj = ta = ta->
1441                 remove_speculative()->
1442                 cast_to_ptr_type(ptr)->
1443                 cast_to_exactness(false)->
1444                 with_offset(offset);
1445       }
1446     }
1447     // Arrays of fixed size alias with arrays of unknown size.
1448     if (ta->size() != TypeInt::POS) {
1449       const TypeAry *tary = TypeAry::make(ta->elem(), TypeInt::POS);
1450       tj = ta = ta->
1451               remove_speculative()->
1452               cast_to_ptr_type(ptr)->
1453               with_ary(tary)->
1454               cast_to_exactness(false);
1455     }
1456     // Arrays of known objects become arrays of unknown objects.
1457     if (ta->elem()->isa_narrowoop() && ta->elem() != TypeNarrowOop::BOTTOM) {
1458       const TypeAry *tary = TypeAry::make(TypeNarrowOop::BOTTOM, ta->size());
1459       tj = ta = TypeAryPtr::make(ptr,ta->const_oop(),tary,nullptr,false,Type::Offset(offset), ta->field_offset());
1460     }
1461     if (ta->elem()->isa_oopptr() && ta->elem() != TypeInstPtr::BOTTOM) {
1462       const TypeAry *tary = TypeAry::make(TypeInstPtr::BOTTOM, ta->size());
1463       tj = ta = TypeAryPtr::make(ptr,ta->const_oop(),tary,nullptr,false,Type::Offset(offset), ta->field_offset());
1464     }
1465     // Initially all flattened array accesses share a single slice
1466     if (ta->is_flat() && ta->elem() != TypeInstPtr::BOTTOM && _flat_accesses_share_alias) {
1467       const TypeAry* tary = TypeAry::make(TypeInstPtr::BOTTOM, ta->size(), /* stable= */ false, /* flat= */ true);
1468       tj = ta = TypeAryPtr::make(ptr,ta->const_oop(),tary,nullptr,false,Type::Offset(offset), Type::Offset(Type::OffsetBot));
1469     }
1470     // Arrays of bytes and of booleans both use 'bastore' and 'baload' so
1471     // cannot be distinguished by bytecode alone.
1472     if (ta->elem() == TypeInt::BOOL) {
1473       const TypeAry *tary = TypeAry::make(TypeInt::BYTE, ta->size());
1474       ciKlass* aklass = ciTypeArrayKlass::make(T_BYTE);
1475       tj = ta = TypeAryPtr::make(ptr,ta->const_oop(),tary,aklass,false,Type::Offset(offset), ta->field_offset());
1476     }
1477     // During the 2nd round of IterGVN, NotNull castings are removed.
1478     // Make sure the Bottom and NotNull variants alias the same.
1479     // Also, make sure exact and non-exact variants alias the same.
1480     if (ptr == TypePtr::NotNull || ta->klass_is_exact() || ta->speculative() != nullptr) {
1481       tj = ta = ta->
1482               remove_speculative()->
1483               cast_to_ptr_type(TypePtr::BotPTR)->
1484               cast_to_exactness(false)->
1485               with_offset(offset);
1486     }
1487   }
1488 
1489   // Oop pointers need some flattening
1490   const TypeInstPtr *to = tj->isa_instptr();
1491   if (to && to != TypeOopPtr::BOTTOM) {
1492     ciInstanceKlass* ik = to->instance_klass();
1493     if( ptr == TypePtr::Constant ) {
1494       if (ik != ciEnv::current()->Class_klass() ||
1495           offset < ik->layout_helper_size_in_bytes()) {
1496         // No constant oop pointers (such as Strings); they alias with
1497         // unknown strings.
1498         assert(!is_known_inst, "not scalarizable allocation");
1499         tj = to = to->
1500                 cast_to_instance_id(TypeOopPtr::InstanceBot)->
1501                 remove_speculative()->
1502                 cast_to_ptr_type(TypePtr::BotPTR)->
1503                 cast_to_exactness(false);
1504       }
1505     } else if( is_known_inst ) {
1506       tj = to; // Keep NotNull and klass_is_exact for instance type
1507     } else if( ptr == TypePtr::NotNull || to->klass_is_exact() ) {
1508       // During the 2nd round of IterGVN, NotNull castings are removed.
1509       // Make sure the Bottom and NotNull variants alias the same.
1510       // Also, make sure exact and non-exact variants alias the same.
1511       tj = to = to->
1512               remove_speculative()->
1513               cast_to_instance_id(TypeOopPtr::InstanceBot)->
1514               cast_to_ptr_type(TypePtr::BotPTR)->
1515               cast_to_exactness(false);
1516     }
1517     if (to->speculative() != nullptr) {
1518       tj = to = to->remove_speculative();
1519     }
1520     // Canonicalize the holder of this field
1521     if (offset >= 0 && offset < instanceOopDesc::base_offset_in_bytes()) {
1522       // First handle header references such as a LoadKlassNode, even if the
1523       // object's klass is unloaded at compile time (4965979).
1524       if (!is_known_inst) { // Do it only for non-instance types
1525         tj = to = TypeInstPtr::make(TypePtr::BotPTR, env()->Object_klass(), false, nullptr, Type::Offset(offset));
1526       }
1527     } else if (offset < 0 || offset >= ik->layout_helper_size_in_bytes()) {
1528       // Static fields are in the space above the normal instance
1529       // fields in the java.lang.Class instance.
1530       if (ik != ciEnv::current()->Class_klass()) {
1531         to = nullptr;
1532         tj = TypeOopPtr::BOTTOM;
1533         offset = tj->offset();
1534       }
1535     } else {
1536       ciInstanceKlass *canonical_holder = ik->get_canonical_holder(offset);
1537       assert(offset < canonical_holder->layout_helper_size_in_bytes(), "");
1538       assert(tj->offset() == offset, "no change to offset expected");
1539       bool xk = to->klass_is_exact();
1540       int instance_id = to->instance_id();
1541 
1542       // If the input type's class is the holder: if exact, the type only includes interfaces implemented by the holder
1543       // but if not exact, it may include extra interfaces: build new type from the holder class to make sure only
1544       // its interfaces are included.
1545       if (xk && ik->equals(canonical_holder)) {
1546         assert(tj == TypeInstPtr::make(to->ptr(), canonical_holder, is_known_inst, nullptr, Type::Offset(offset), instance_id), "exact type should be canonical type");
1547       } else {
1548         assert(xk || !is_known_inst, "Known instance should be exact type");
1549         tj = to = TypeInstPtr::make(to->ptr(), canonical_holder, is_known_inst, nullptr, Type::Offset(offset), instance_id);
1550       }
1551     }
1552   }
1553 
1554   // Klass pointers to object array klasses need some flattening
1555   const TypeKlassPtr *tk = tj->isa_klassptr();
1556   if( tk ) {
1557     // If we are referencing a field within a Klass, we need
1558     // to assume the worst case of an Object.  Both exact and
1559     // inexact types must flatten to the same alias class so
1560     // use NotNull as the PTR.
1561     if ( offset == Type::OffsetBot || (offset >= 0 && (size_t)offset < sizeof(Klass)) ) {
1562       tj = tk = TypeInstKlassPtr::make(TypePtr::NotNull,
1563                                        env()->Object_klass(),
1564                                        Type::Offset(offset));
1565     }
1566 
1567     if (tk->isa_aryklassptr() && tk->is_aryklassptr()->elem()->isa_klassptr()) {
1568       ciKlass* k = ciObjArrayKlass::make(env()->Object_klass());
1569       if (!k || !k->is_loaded()) {                  // Only fails for some -Xcomp runs
1570         tj = tk = TypeInstKlassPtr::make(TypePtr::NotNull, env()->Object_klass(), Type::Offset(offset));
1571       } else {
1572         tj = tk = TypeAryKlassPtr::make(TypePtr::NotNull, tk->is_aryklassptr()->elem(), k, Type::Offset(offset), tk->is_not_flat(), tk->is_not_null_free(), tk->is_flat(), tk->is_null_free(), tk->is_atomic(), tk->is_aryklassptr()->is_vm_type());
1573       }
1574     }
1575     // Check for precise loads from the primary supertype array and force them
1576     // to the supertype cache alias index.  Check for generic array loads from
1577     // the primary supertype array and also force them to the supertype cache
1578     // alias index.  Since the same load can reach both, we need to merge
1579     // these 2 disparate memories into the same alias class.  Since the
1580     // primary supertype array is read-only, there's no chance of confusion
1581     // where we bypass an array load and an array store.
1582     int primary_supers_offset = in_bytes(Klass::primary_supers_offset());
1583     if (offset == Type::OffsetBot ||
1584         (offset >= primary_supers_offset &&
1585          offset < (int)(primary_supers_offset + Klass::primary_super_limit() * wordSize)) ||
1586         offset == (int)in_bytes(Klass::secondary_super_cache_offset())) {
1587       offset = in_bytes(Klass::secondary_super_cache_offset());
1588       tj = tk = tk->with_offset(offset);
1589     }
1590   }
1591 
1592   // Flatten all Raw pointers together.
1593   if (tj->base() == Type::RawPtr)
1594     tj = TypeRawPtr::BOTTOM;
1595 
1596   if (tj->base() == Type::AnyPtr)
1597     tj = TypePtr::BOTTOM;      // An error, which the caller must check for.
1598 
1599   offset = tj->offset();
1600   assert( offset != Type::OffsetTop, "Offset has fallen from constant" );
1601 
1602   assert( (offset != Type::OffsetBot && tj->base() != Type::AryPtr) ||
1603           (offset == Type::OffsetBot && tj->base() == Type::AryPtr) ||
1604           (offset == Type::OffsetBot && tj == TypeOopPtr::BOTTOM) ||
1605           (offset == Type::OffsetBot && tj == TypePtr::BOTTOM) ||
1606           (offset == oopDesc::mark_offset_in_bytes() && tj->base() == Type::AryPtr) ||
1607           (offset == oopDesc::klass_offset_in_bytes() && tj->base() == Type::AryPtr) ||
1608           (offset == arrayOopDesc::length_offset_in_bytes() && tj->base() == Type::AryPtr),
1609           "For oops, klasses, raw offset must be constant; for arrays the offset is never known" );
1610   assert( tj->ptr() != TypePtr::TopPTR &&
1611           tj->ptr() != TypePtr::AnyNull &&
1612           tj->ptr() != TypePtr::Null, "No imprecise addresses" );
1613 //    assert( tj->ptr() != TypePtr::Constant ||
1614 //            tj->base() == Type::RawPtr ||
1615 //            tj->base() == Type::KlassPtr, "No constant oop addresses" );
1616 
1617   return tj;
1618 }
1619 
1620 void Compile::AliasType::Init(int i, const TypePtr* at) {
1621   assert(AliasIdxTop <= i && i < Compile::current()->_max_alias_types, "Invalid alias index");
1622   _index = i;
1623   _adr_type = at;
1624   _field = nullptr;
1625   _element = nullptr;
1626   _is_rewritable = true; // default
1627   const TypeOopPtr *atoop = (at != nullptr) ? at->isa_oopptr() : nullptr;
1628   if (atoop != nullptr && atoop->is_known_instance()) {
1629     const TypeOopPtr *gt = atoop->cast_to_instance_id(TypeOopPtr::InstanceBot);
1630     _general_index = Compile::current()->get_alias_index(gt);
1631   } else {
1632     _general_index = 0;
1633   }
1634 }
1635 
1636 BasicType Compile::AliasType::basic_type() const {
1637   if (element() != nullptr) {
1638     const Type* element = adr_type()->is_aryptr()->elem();
1639     return element->isa_narrowoop() ? T_OBJECT : element->array_element_basic_type();
1640   } if (field() != nullptr) {
1641     return field()->layout_type();
1642   } else {
1643     return T_ILLEGAL; // unknown
1644   }
1645 }
1646 
1647 //---------------------------------print_on------------------------------------
1648 #ifndef PRODUCT
1649 void Compile::AliasType::print_on(outputStream* st) {
1650   if (index() < 10)
1651         st->print("@ <%d> ", index());
1652   else  st->print("@ <%d>",  index());
1653   st->print(is_rewritable() ? "   " : " RO");
1654   int offset = adr_type()->offset();
1655   if (offset == Type::OffsetBot)
1656         st->print(" +any");
1657   else  st->print(" +%-3d", offset);
1658   st->print(" in ");
1659   adr_type()->dump_on(st);
1660   const TypeOopPtr* tjp = adr_type()->isa_oopptr();
1661   if (field() != nullptr && tjp) {
1662     if (tjp->is_instptr()->instance_klass()  != field()->holder() ||
1663         tjp->offset() != field()->offset_in_bytes()) {
1664       st->print(" != ");
1665       field()->print();
1666       st->print(" ***");
1667     }
1668   }
1669 }
1670 
1671 void print_alias_types() {
1672   Compile* C = Compile::current();
1673   tty->print_cr("--- Alias types, AliasIdxBot .. %d", C->num_alias_types()-1);
1674   for (int idx = Compile::AliasIdxBot; idx < C->num_alias_types(); idx++) {
1675     C->alias_type(idx)->print_on(tty);
1676     tty->cr();
1677   }
1678 }
1679 #endif
1680 
1681 
1682 //----------------------------probe_alias_cache--------------------------------
1683 Compile::AliasCacheEntry* Compile::probe_alias_cache(const TypePtr* adr_type) {
1684   intptr_t key = (intptr_t) adr_type;
1685   key ^= key >> logAliasCacheSize;
1686   return &_alias_cache[key & right_n_bits(logAliasCacheSize)];
1687 }
1688 
1689 
1690 //-----------------------------grow_alias_types--------------------------------
1691 void Compile::grow_alias_types() {
1692   const int old_ats  = _max_alias_types; // how many before?
1693   const int new_ats  = old_ats;          // how many more?
1694   const int grow_ats = old_ats+new_ats;  // how many now?
1695   _max_alias_types = grow_ats;
1696   _alias_types =  REALLOC_ARENA_ARRAY(comp_arena(), AliasType*, _alias_types, old_ats, grow_ats);
1697   AliasType* ats =    NEW_ARENA_ARRAY(comp_arena(), AliasType, new_ats);
1698   Copy::zero_to_bytes(ats, sizeof(AliasType)*new_ats);
1699   for (int i = 0; i < new_ats; i++)  _alias_types[old_ats+i] = &ats[i];
1700 }
1701 
1702 
1703 //--------------------------------find_alias_type------------------------------
1704 Compile::AliasType* Compile::find_alias_type(const TypePtr* adr_type, bool no_create, ciField* original_field, bool uncached) {
1705   if (!do_aliasing()) {
1706     return alias_type(AliasIdxBot);
1707   }
1708 
1709   AliasCacheEntry* ace = nullptr;
1710   if (!uncached) {
1711     ace = probe_alias_cache(adr_type);
1712     if (ace->_adr_type == adr_type) {
1713       return alias_type(ace->_index);
1714     }
1715   }
1716 
1717   // Handle special cases.
1718   if (adr_type == nullptr)          return alias_type(AliasIdxTop);
1719   if (adr_type == TypePtr::BOTTOM)  return alias_type(AliasIdxBot);
1720 
1721   // Do it the slow way.
1722   const TypePtr* flat = flatten_alias_type(adr_type);
1723 
1724 #ifdef ASSERT
1725   {
1726     ResourceMark rm;
1727     assert(flat == flatten_alias_type(flat), "not idempotent: adr_type = %s; flat = %s => %s",
1728            Type::str(adr_type), Type::str(flat), Type::str(flatten_alias_type(flat)));
1729     assert(flat != TypePtr::BOTTOM, "cannot alias-analyze an untyped ptr: adr_type = %s",
1730            Type::str(adr_type));
1731     if (flat->isa_oopptr() && !flat->isa_klassptr()) {
1732       const TypeOopPtr* foop = flat->is_oopptr();
1733       // Scalarizable allocations have exact klass always.
1734       bool exact = !foop->klass_is_exact() || foop->is_known_instance();
1735       const TypePtr* xoop = foop->cast_to_exactness(exact)->is_ptr();
1736       assert(foop == flatten_alias_type(xoop), "exactness must not affect alias type: foop = %s; xoop = %s",
1737              Type::str(foop), Type::str(xoop));
1738     }
1739   }
1740 #endif
1741 
1742   int idx = AliasIdxTop;
1743   for (int i = 0; i < num_alias_types(); i++) {
1744     if (alias_type(i)->adr_type() == flat) {
1745       idx = i;
1746       break;
1747     }
1748   }
1749 
1750   if (idx == AliasIdxTop) {
1751     if (no_create)  return nullptr;
1752     // Grow the array if necessary.
1753     if (_num_alias_types == _max_alias_types)  grow_alias_types();
1754     // Add a new alias type.
1755     idx = _num_alias_types++;
1756     _alias_types[idx]->Init(idx, flat);
1757     if (flat == TypeInstPtr::KLASS)  alias_type(idx)->set_rewritable(false);
1758     if (flat == TypeAryPtr::RANGE)   alias_type(idx)->set_rewritable(false);
1759     if (flat->isa_instptr()) {
1760       if (flat->offset() == java_lang_Class::klass_offset()
1761           && flat->is_instptr()->instance_klass() == env()->Class_klass())
1762         alias_type(idx)->set_rewritable(false);
1763     }
1764     ciField* field = nullptr;
1765     if (flat->isa_aryptr()) {
1766 #ifdef ASSERT
1767       const int header_size_min  = arrayOopDesc::base_offset_in_bytes(T_BYTE);
1768       // (T_BYTE has the weakest alignment and size restrictions...)
1769       assert(flat->offset() < header_size_min, "array body reference must be OffsetBot");
1770 #endif
1771       const Type* elemtype = flat->is_aryptr()->elem();
1772       if (flat->offset() == TypePtr::OffsetBot) {
1773         alias_type(idx)->set_element(elemtype);
1774       }
1775       int field_offset = flat->is_aryptr()->field_offset().get();
1776       if (flat->is_flat() &&
1777           field_offset != Type::OffsetBot) {
1778         ciInlineKlass* vk = elemtype->inline_klass();
1779         field_offset += vk->payload_offset();
1780         field = vk->get_field_by_offset(field_offset, false);
1781       }
1782     }
1783     if (flat->isa_klassptr()) {
1784       if (UseCompactObjectHeaders) {
1785         if (flat->offset() == in_bytes(Klass::prototype_header_offset()))
1786           alias_type(idx)->set_rewritable(false);
1787       }
1788       if (flat->offset() == in_bytes(Klass::super_check_offset_offset()))
1789         alias_type(idx)->set_rewritable(false);
1790       if (flat->offset() == in_bytes(Klass::access_flags_offset()))
1791         alias_type(idx)->set_rewritable(false);
1792       if (flat->offset() == in_bytes(Klass::misc_flags_offset()))
1793         alias_type(idx)->set_rewritable(false);
1794       if (flat->offset() == in_bytes(Klass::java_mirror_offset()))
1795         alias_type(idx)->set_rewritable(false);
1796       if (flat->offset() == in_bytes(Klass::layout_helper_offset()))
1797         alias_type(idx)->set_rewritable(false);
1798       if (flat->offset() == in_bytes(Klass::secondary_super_cache_offset()))
1799         alias_type(idx)->set_rewritable(false);
1800     }
1801     // %%% (We would like to finalize JavaThread::threadObj_offset(),
1802     // but the base pointer type is not distinctive enough to identify
1803     // references into JavaThread.)
1804 
1805     // Check for final fields.
1806     const TypeInstPtr* tinst = flat->isa_instptr();
1807     if (tinst && tinst->offset() >= instanceOopDesc::base_offset_in_bytes()) {
1808       if (tinst->const_oop() != nullptr &&
1809           tinst->instance_klass() == ciEnv::current()->Class_klass() &&
1810           tinst->offset() >= (tinst->instance_klass()->layout_helper_size_in_bytes())) {
1811         // static field
1812         ciInstanceKlass* k = tinst->const_oop()->as_instance()->java_lang_Class_klass()->as_instance_klass();
1813         field = k->get_field_by_offset(tinst->offset(), true);
1814       } else if (tinst->is_inlinetypeptr()) {
1815         // Inline type field
1816         ciInlineKlass* vk = tinst->inline_klass();
1817         field = vk->get_field_by_offset(tinst->offset(), false);
1818       } else {
1819         ciInstanceKlass *k = tinst->instance_klass();
1820         field = k->get_field_by_offset(tinst->offset(), false);
1821       }
1822     }
1823     assert(field == nullptr ||
1824            original_field == nullptr ||
1825            (field->holder() == original_field->holder() &&
1826             field->offset_in_bytes() == original_field->offset_in_bytes() &&
1827             field->is_static() == original_field->is_static()), "wrong field?");
1828     // Set field() and is_rewritable() attributes.
1829     if (field != nullptr) {
1830       alias_type(idx)->set_field(field);
1831       if (flat->isa_aryptr()) {
1832         // Fields of flat arrays are rewritable although they are declared final
1833         assert(flat->is_flat(), "must be a flat array");
1834         alias_type(idx)->set_rewritable(true);
1835       }
1836     }
1837   }
1838 
1839   // Fill the cache for next time.
1840   if (!uncached) {
1841     ace->_adr_type = adr_type;
1842     ace->_index    = idx;
1843     assert(alias_type(adr_type) == alias_type(idx),  "type must be installed");
1844 
1845     // Might as well try to fill the cache for the flattened version, too.
1846     AliasCacheEntry* face = probe_alias_cache(flat);
1847     if (face->_adr_type == nullptr) {
1848       face->_adr_type = flat;
1849       face->_index    = idx;
1850       assert(alias_type(flat) == alias_type(idx), "flat type must work too");
1851     }
1852   }
1853 
1854   return alias_type(idx);
1855 }
1856 
1857 
1858 Compile::AliasType* Compile::alias_type(ciField* field) {
1859   const TypeOopPtr* t;
1860   if (field->is_static())
1861     t = TypeInstPtr::make(field->holder()->java_mirror());
1862   else
1863     t = TypeOopPtr::make_from_klass_raw(field->holder());
1864   AliasType* atp = alias_type(t->add_offset(field->offset_in_bytes()), field);
1865   assert((field->is_final() || field->is_stable()) == !atp->is_rewritable(), "must get the rewritable bits correct");
1866   return atp;
1867 }
1868 
1869 
1870 //------------------------------have_alias_type--------------------------------
1871 bool Compile::have_alias_type(const TypePtr* adr_type) {
1872   AliasCacheEntry* ace = probe_alias_cache(adr_type);
1873   if (ace->_adr_type == adr_type) {
1874     return true;
1875   }
1876 
1877   // Handle special cases.
1878   if (adr_type == nullptr)             return true;
1879   if (adr_type == TypePtr::BOTTOM)  return true;
1880 
1881   return find_alias_type(adr_type, true, nullptr) != nullptr;
1882 }
1883 
1884 //-----------------------------must_alias--------------------------------------
1885 // True if all values of the given address type are in the given alias category.
1886 bool Compile::must_alias(const TypePtr* adr_type, int alias_idx) {
1887   if (alias_idx == AliasIdxBot)         return true;  // the universal category
1888   if (adr_type == nullptr)              return true;  // null serves as TypePtr::TOP
1889   if (alias_idx == AliasIdxTop)         return false; // the empty category
1890   if (adr_type->base() == Type::AnyPtr) return false; // TypePtr::BOTTOM or its twins
1891 
1892   // the only remaining possible overlap is identity
1893   int adr_idx = get_alias_index(adr_type);
1894   assert(adr_idx != AliasIdxBot && adr_idx != AliasIdxTop, "");
1895   assert(adr_idx == alias_idx ||
1896          (alias_type(alias_idx)->adr_type() != TypeOopPtr::BOTTOM
1897           && adr_type                       != TypeOopPtr::BOTTOM),
1898          "should not be testing for overlap with an unsafe pointer");
1899   return adr_idx == alias_idx;
1900 }
1901 
1902 //------------------------------can_alias--------------------------------------
1903 // True if any values of the given address type are in the given alias category.
1904 bool Compile::can_alias(const TypePtr* adr_type, int alias_idx) {
1905   if (alias_idx == AliasIdxTop)         return false; // the empty category
1906   if (adr_type == nullptr)              return false; // null serves as TypePtr::TOP
1907   // Known instance doesn't alias with bottom memory
1908   if (alias_idx == AliasIdxBot)         return !adr_type->is_known_instance();                   // the universal category
1909   if (adr_type->base() == Type::AnyPtr) return !C->get_adr_type(alias_idx)->is_known_instance(); // TypePtr::BOTTOM or its twins
1910 
1911   // the only remaining possible overlap is identity
1912   int adr_idx = get_alias_index(adr_type);
1913   assert(adr_idx != AliasIdxBot && adr_idx != AliasIdxTop, "");
1914   return adr_idx == alias_idx;
1915 }
1916 
1917 // Mark all ParsePredicateNodes as useless. They will later be removed from the graph in IGVN together with their
1918 // uncommon traps if no Runtime Predicates were created from the Parse Predicates.
1919 void Compile::mark_parse_predicate_nodes_useless(PhaseIterGVN& igvn) {
1920   if (parse_predicate_count() == 0) {
1921     return;
1922   }
1923   for (int i = 0; i < parse_predicate_count(); i++) {
1924     ParsePredicateNode* parse_predicate = _parse_predicates.at(i);
1925     parse_predicate->mark_useless(igvn);
1926   }
1927   _parse_predicates.clear();
1928 }
1929 
1930 void Compile::record_for_post_loop_opts_igvn(Node* n) {
1931   if (!n->for_post_loop_opts_igvn()) {
1932     assert(!_for_post_loop_igvn.contains(n), "duplicate");
1933     n->add_flag(Node::NodeFlags::Flag_for_post_loop_opts_igvn);
1934     _for_post_loop_igvn.append(n);
1935   }
1936 }
1937 
1938 void Compile::remove_from_post_loop_opts_igvn(Node* n) {
1939   n->remove_flag(Node::NodeFlags::Flag_for_post_loop_opts_igvn);
1940   _for_post_loop_igvn.remove(n);
1941 }
1942 
1943 void Compile::process_for_post_loop_opts_igvn(PhaseIterGVN& igvn) {
1944   // Verify that all previous optimizations produced a valid graph
1945   // at least to this point, even if no loop optimizations were done.
1946   PhaseIdealLoop::verify(igvn);
1947 
1948   if (has_loops() || _loop_opts_cnt > 0) {
1949     print_method(PHASE_AFTER_LOOP_OPTS, 2);
1950   }
1951   C->set_post_loop_opts_phase(); // no more loop opts allowed
1952 
1953   assert(!C->major_progress(), "not cleared");
1954 
1955   if (_for_post_loop_igvn.length() > 0) {
1956     while (_for_post_loop_igvn.length() > 0) {
1957       Node* n = _for_post_loop_igvn.pop();
1958       n->remove_flag(Node::NodeFlags::Flag_for_post_loop_opts_igvn);
1959       igvn._worklist.push(n);
1960     }
1961     igvn.optimize();
1962     if (failing()) return;
1963     assert(_for_post_loop_igvn.length() == 0, "no more delayed nodes allowed");
1964     assert(C->parse_predicate_count() == 0, "all parse predicates should have been removed now");
1965 
1966     // Sometimes IGVN sets major progress (e.g., when processing loop nodes).
1967     if (C->major_progress()) {
1968       C->clear_major_progress(); // ensure that major progress is now clear
1969     }
1970   }
1971 }
1972 
1973 void Compile::add_inline_type(Node* n) {
1974   assert(n->is_InlineType(), "unexpected node");
1975   _inline_type_nodes.push(n);
1976 }
1977 
1978 void Compile::remove_inline_type(Node* n) {
1979   assert(n->is_InlineType(), "unexpected node");
1980   if (_inline_type_nodes.contains(n)) {
1981     _inline_type_nodes.remove(n);
1982   }
1983 }
1984 
1985 // Does the return value keep otherwise useless inline type allocations alive?
1986 static bool return_val_keeps_allocations_alive(Node* ret_val) {
1987   ResourceMark rm;
1988   Unique_Node_List wq;
1989   wq.push(ret_val);
1990   bool some_allocations = false;
1991   for (uint i = 0; i < wq.size(); i++) {
1992     Node* n = wq.at(i);
1993     if (n->outcnt() > 1) {
1994       // Some other use for the allocation
1995       return false;
1996     } else if (n->is_InlineType()) {
1997       wq.push(n->in(1));
1998     } else if (n->is_Phi()) {
1999       for (uint j = 1; j < n->req(); j++) {
2000         wq.push(n->in(j));
2001       }
2002     } else if (n->is_CheckCastPP() &&
2003                n->in(1)->is_Proj() &&
2004                n->in(1)->in(0)->is_Allocate()) {
2005       some_allocations = true;
2006     } else if (n->is_CheckCastPP()) {
2007       wq.push(n->in(1));
2008     }
2009   }
2010   return some_allocations;
2011 }
2012 
2013 void Compile::process_inline_types(PhaseIterGVN &igvn, bool remove) {
2014   // Make sure that the return value does not keep an otherwise unused allocation alive
2015   if (tf()->returns_inline_type_as_fields()) {
2016     Node* ret = nullptr;
2017     for (uint i = 1; i < root()->req(); i++) {
2018       Node* in = root()->in(i);
2019       if (in->Opcode() == Op_Return) {
2020         assert(ret == nullptr, "only one return");
2021         ret = in;
2022       }
2023     }
2024     if (ret != nullptr) {
2025       Node* ret_val = ret->in(TypeFunc::Parms);
2026       if (igvn.type(ret_val)->isa_oopptr() &&
2027           return_val_keeps_allocations_alive(ret_val)) {
2028         igvn.replace_input_of(ret, TypeFunc::Parms, InlineTypeNode::tagged_klass(igvn.type(ret_val)->inline_klass(), igvn));
2029         assert(ret_val->outcnt() == 0, "should be dead now");
2030         igvn.remove_dead_node(ret_val);
2031       }
2032     }
2033   }
2034   if (_inline_type_nodes.length() == 0) {
2035     return;
2036   }
2037   // Scalarize inline types in safepoint debug info.
2038   // Delay this until all inlining is over to avoid getting inconsistent debug info.
2039   set_scalarize_in_safepoints(true);
2040   for (int i = _inline_type_nodes.length()-1; i >= 0; i--) {
2041     InlineTypeNode* vt = _inline_type_nodes.at(i)->as_InlineType();
2042     vt->make_scalar_in_safepoints(&igvn);
2043     igvn.record_for_igvn(vt);
2044   }
2045   if (remove) {
2046     // Remove inline type nodes by replacing them with their oop input
2047     while (_inline_type_nodes.length() > 0) {
2048       InlineTypeNode* vt = _inline_type_nodes.pop()->as_InlineType();
2049       if (vt->outcnt() == 0) {
2050         igvn.remove_dead_node(vt);
2051         continue;
2052       }
2053       for (DUIterator i = vt->outs(); vt->has_out(i); i++) {
2054         DEBUG_ONLY(bool must_be_buffered = false);
2055         Node* u = vt->out(i);
2056         // Check if any users are blackholes. If so, rewrite them to use either the
2057         // allocated buffer, or individual components, instead of the inline type node
2058         // that goes away.
2059         if (u->is_Blackhole()) {
2060           BlackholeNode* bh = u->as_Blackhole();
2061 
2062           // Unlink the old input
2063           int idx = bh->find_edge(vt);
2064           assert(idx != -1, "The edge should be there");
2065           bh->del_req(idx);
2066           --i;
2067 
2068           if (vt->is_allocated(&igvn)) {
2069             // Already has the allocated instance, blackhole that
2070             bh->add_req(vt->get_oop());
2071           } else {
2072             // Not allocated yet, blackhole the components
2073             for (uint c = 0; c < vt->field_count(); c++) {
2074               bh->add_req(vt->field_value(c));
2075             }
2076           }
2077 
2078           // Node modified, record for IGVN
2079           igvn.record_for_igvn(bh);
2080         }
2081 #ifdef ASSERT
2082         // Verify that inline type is buffered when replacing by oop
2083         else if (u->is_InlineType()) {
2084           // InlineType uses don't need buffering because they are about to be replaced as well
2085         } else if (u->is_Phi()) {
2086           // TODO 8302217 Remove this once InlineTypeNodes are reliably pushed through
2087         } else {
2088           must_be_buffered = true;
2089         }
2090         if (must_be_buffered && !vt->is_allocated(&igvn)) {
2091           vt->dump(0);
2092           u->dump(0);
2093           assert(false, "Should have been buffered");
2094         }
2095 #endif
2096       }
2097       igvn.replace_node(vt, vt->get_oop());
2098     }
2099   }
2100   igvn.optimize();
2101 }
2102 
2103 void Compile::add_flat_access(Node* n) {
2104   assert(n != nullptr && (n->Opcode() == Op_LoadFlat || n->Opcode() == Op_StoreFlat), "unexpected node %s", n == nullptr ? "nullptr" : n->Name());
2105   assert(!_flat_access_nodes.contains(n), "duplicate insertion");
2106   _flat_access_nodes.push(n);
2107 }
2108 
2109 void Compile::remove_flat_access(Node* n) {
2110   assert(n != nullptr && (n->Opcode() == Op_LoadFlat || n->Opcode() == Op_StoreFlat), "unexpected node %s", n == nullptr ? "nullptr" : n->Name());
2111   _flat_access_nodes.remove_if_existing(n);
2112 }
2113 
2114 void Compile::process_flat_accesses(PhaseIterGVN& igvn) {
2115   assert(igvn._worklist.size() == 0, "should be empty");
2116   igvn.set_delay_transform(true);
2117   for (int i = _flat_access_nodes.length() - 1; i >= 0; i--) {
2118     Node* n = _flat_access_nodes.at(i);
2119     assert(n != nullptr, "unexpected nullptr");
2120     if (n->is_LoadFlat()) {
2121       n->as_LoadFlat()->expand_atomic(igvn);
2122     } else {
2123       n->as_StoreFlat()->expand_atomic(igvn);
2124     }
2125   }
2126   _flat_access_nodes.clear_and_deallocate();
2127   igvn.set_delay_transform(false);
2128   igvn.optimize();
2129 }
2130 
2131 void Compile::adjust_flat_array_access_aliases(PhaseIterGVN& igvn) {
2132   if (!_has_flat_accesses) {
2133     return;
2134   }
2135   // Initially, all flat array accesses share the same slice to
2136   // keep dependencies with Object[] array accesses (that could be
2137   // to a flat array) correct. We're done with parsing so we
2138   // now know all flat array accesses in this compile
2139   // unit. Let's move flat array accesses to their own slice,
2140   // one per element field. This should help memory access
2141   // optimizations.
2142   ResourceMark rm;
2143   Unique_Node_List wq;
2144   wq.push(root());
2145 
2146   Node_List mergememnodes;
2147   Node_List memnodes;
2148 
2149   // Alias index currently shared by all flat memory accesses
2150   int index = get_alias_index(TypeAryPtr::INLINES);
2151 
2152   // Find MergeMem nodes and flat array accesses
2153   for (uint i = 0; i < wq.size(); i++) {
2154     Node* n = wq.at(i);
2155     if (n->is_Mem()) {
2156       const TypePtr* adr_type = nullptr;
2157       adr_type = get_adr_type(get_alias_index(n->adr_type()));
2158       if (adr_type == TypeAryPtr::INLINES) {
2159         memnodes.push(n);
2160       }
2161     } else if (n->is_MergeMem()) {
2162       MergeMemNode* mm = n->as_MergeMem();
2163       if (mm->memory_at(index) != mm->base_memory()) {
2164         mergememnodes.push(n);
2165       }
2166     }
2167     for (uint j = 0; j < n->req(); j++) {
2168       Node* m = n->in(j);
2169       if (m != nullptr) {
2170         wq.push(m);
2171       }
2172     }
2173   }
2174 
2175   if (memnodes.size() > 0) {
2176     _flat_accesses_share_alias = false;
2177 
2178     // We are going to change the slice for the flat array
2179     // accesses so we need to clear the cache entries that refer to
2180     // them.
2181     for (uint i = 0; i < AliasCacheSize; i++) {
2182       AliasCacheEntry* ace = &_alias_cache[i];
2183       if (ace->_adr_type != nullptr &&
2184           ace->_adr_type->is_flat()) {
2185         ace->_adr_type = nullptr;
2186         ace->_index = (i != 0) ? 0 : AliasIdxTop; // Make sure the nullptr adr_type resolves to AliasIdxTop
2187       }
2188     }
2189 
2190     // Find what aliases we are going to add
2191     int start_alias = num_alias_types()-1;
2192     int stop_alias = 0;
2193 
2194     for (uint i = 0; i < memnodes.size(); i++) {
2195       Node* m = memnodes.at(i);
2196       const TypePtr* adr_type = nullptr;
2197       adr_type = m->adr_type();
2198 #ifdef ASSERT
2199       m->as_Mem()->set_adr_type(adr_type);
2200 #endif
2201       int idx = get_alias_index(adr_type);
2202       start_alias = MIN2(start_alias, idx);
2203       stop_alias = MAX2(stop_alias, idx);
2204     }
2205 
2206     assert(stop_alias >= start_alias, "should have expanded aliases");
2207 
2208     Node_Stack stack(0);
2209 #ifdef ASSERT
2210     VectorSet seen(Thread::current()->resource_area());
2211 #endif
2212     // Now let's fix the memory graph so each flat array access
2213     // is moved to the right slice. Start from the MergeMem nodes.
2214     uint last = unique();
2215     for (uint i = 0; i < mergememnodes.size(); i++) {
2216       MergeMemNode* current = mergememnodes.at(i)->as_MergeMem();
2217       Node* n = current->memory_at(index);
2218       MergeMemNode* mm = nullptr;
2219       do {
2220         // Follow memory edges through memory accesses, phis and
2221         // narrow membars and push nodes on the stack. Once we hit
2222         // bottom memory, we pop element off the stack one at a
2223         // time, in reverse order, and move them to the right slice
2224         // by changing their memory edges.
2225         if ((n->is_Phi() && n->adr_type() != TypePtr::BOTTOM) || n->is_Mem() || n->adr_type() == TypeAryPtr::INLINES) {
2226           assert(!seen.test_set(n->_idx), "");
2227           // Uses (a load for instance) will need to be moved to the
2228           // right slice as well and will get a new memory state
2229           // that we don't know yet. The use could also be the
2230           // backedge of a loop. We put a place holder node between
2231           // the memory node and its uses. We replace that place
2232           // holder with the correct memory state once we know it,
2233           // i.e. when nodes are popped off the stack. Using the
2234           // place holder make the logic work in the presence of
2235           // loops.
2236           if (n->outcnt() > 1) {
2237             Node* place_holder = nullptr;
2238             assert(!n->has_out_with(Op_Node), "");
2239             for (DUIterator k = n->outs(); n->has_out(k); k++) {
2240               Node* u = n->out(k);
2241               if (u != current && u->_idx < last) {
2242                 bool success = false;
2243                 for (uint l = 0; l < u->req(); l++) {
2244                   if (!stack.is_empty() && u == stack.node() && l == stack.index()) {
2245                     continue;
2246                   }
2247                   Node* in = u->in(l);
2248                   if (in == n) {
2249                     if (place_holder == nullptr) {
2250                       place_holder = new Node(1);
2251                       place_holder->init_req(0, n);
2252                     }
2253                     igvn.replace_input_of(u, l, place_holder);
2254                     success = true;
2255                   }
2256                 }
2257                 if (success) {
2258                   --k;
2259                 }
2260               }
2261             }
2262           }
2263           if (n->is_Phi()) {
2264             stack.push(n, 1);
2265             n = n->in(1);
2266           } else if (n->is_Mem()) {
2267             stack.push(n, n->req());
2268             n = n->in(MemNode::Memory);
2269           } else {
2270             assert(n->is_Proj() && n->in(0)->Opcode() == Op_MemBarCPUOrder, "");
2271             stack.push(n, n->req());
2272             n = n->in(0)->in(TypeFunc::Memory);
2273           }
2274         } else {
2275           assert(n->adr_type() == TypePtr::BOTTOM || (n->Opcode() == Op_Node && n->_idx >= last) || (n->is_Proj() && n->in(0)->is_Initialize()), "");
2276           // Build a new MergeMem node to carry the new memory state
2277           // as we build it. IGVN should fold extraneous MergeMem
2278           // nodes.
2279           mm = MergeMemNode::make(n);
2280           igvn.register_new_node_with_optimizer(mm);
2281           while (stack.size() > 0) {
2282             Node* m = stack.node();
2283             uint idx = stack.index();
2284             if (m->is_Mem()) {
2285               // Move memory node to its new slice
2286               const TypePtr* adr_type = m->adr_type();
2287               int alias = get_alias_index(adr_type);
2288               Node* prev = mm->memory_at(alias);
2289               igvn.replace_input_of(m, MemNode::Memory, prev);
2290               mm->set_memory_at(alias, m);
2291             } else if (m->is_Phi()) {
2292               // We need as many new phis as there are new aliases
2293               igvn.replace_input_of(m, idx, mm);
2294               if (idx == m->req()-1) {
2295                 Node* r = m->in(0);
2296                 for (uint j = (uint)start_alias; j <= (uint)stop_alias; j++) {
2297                   const TypePtr* adr_type = get_adr_type(j);
2298                   if (!adr_type->isa_aryptr() || !adr_type->is_flat() || j == (uint)index) {
2299                     continue;
2300                   }
2301                   Node* phi = new PhiNode(r, Type::MEMORY, get_adr_type(j));
2302                   igvn.register_new_node_with_optimizer(phi);
2303                   for (uint k = 1; k < m->req(); k++) {
2304                     phi->init_req(k, m->in(k)->as_MergeMem()->memory_at(j));
2305                   }
2306                   mm->set_memory_at(j, phi);
2307                 }
2308                 Node* base_phi = new PhiNode(r, Type::MEMORY, TypePtr::BOTTOM);
2309                 igvn.register_new_node_with_optimizer(base_phi);
2310                 for (uint k = 1; k < m->req(); k++) {
2311                   base_phi->init_req(k, m->in(k)->as_MergeMem()->base_memory());
2312                 }
2313                 mm->set_base_memory(base_phi);
2314               }
2315             } else {
2316               // This is a MemBarCPUOrder node from
2317               // Parse::array_load()/Parse::array_store(), in the
2318               // branch that handles flat arrays hidden under
2319               // an Object[] array. We also need one new membar per
2320               // new alias to keep the unknown access that the
2321               // membars protect properly ordered with accesses to
2322               // known flat array.
2323               assert(m->is_Proj(), "projection expected");
2324               Node* ctrl = m->in(0)->in(TypeFunc::Control);
2325               igvn.replace_input_of(m->in(0), TypeFunc::Control, top());
2326               for (uint j = (uint)start_alias; j <= (uint)stop_alias; j++) {
2327                 const TypePtr* adr_type = get_adr_type(j);
2328                 if (!adr_type->isa_aryptr() || !adr_type->is_flat() || j == (uint)index) {
2329                   continue;
2330                 }
2331                 MemBarNode* mb = new MemBarCPUOrderNode(this, j, nullptr);
2332                 igvn.register_new_node_with_optimizer(mb);
2333                 Node* mem = mm->memory_at(j);
2334                 mb->init_req(TypeFunc::Control, ctrl);
2335                 mb->init_req(TypeFunc::Memory, mem);
2336                 ctrl = new ProjNode(mb, TypeFunc::Control);
2337                 igvn.register_new_node_with_optimizer(ctrl);
2338                 mem = new ProjNode(mb, TypeFunc::Memory);
2339                 igvn.register_new_node_with_optimizer(mem);
2340                 mm->set_memory_at(j, mem);
2341               }
2342               igvn.replace_node(m->in(0)->as_Multi()->proj_out(TypeFunc::Control), ctrl);
2343             }
2344             if (idx < m->req()-1) {
2345               idx += 1;
2346               stack.set_index(idx);
2347               n = m->in(idx);
2348               break;
2349             }
2350             // Take care of place holder nodes
2351             if (m->has_out_with(Op_Node)) {
2352               Node* place_holder = m->find_out_with(Op_Node);
2353               if (place_holder != nullptr) {
2354                 Node* mm_clone = mm->clone();
2355                 igvn.register_new_node_with_optimizer(mm_clone);
2356                 Node* hook = new Node(1);
2357                 hook->init_req(0, mm);
2358                 igvn.replace_node(place_holder, mm_clone);
2359                 hook->destruct(&igvn);
2360               }
2361               assert(!m->has_out_with(Op_Node), "place holder should be gone now");
2362             }
2363             stack.pop();
2364           }
2365         }
2366       } while(stack.size() > 0);
2367       // Fix the memory state at the MergeMem we started from
2368       igvn.rehash_node_delayed(current);
2369       for (uint j = (uint)start_alias; j <= (uint)stop_alias; j++) {
2370         const TypePtr* adr_type = get_adr_type(j);
2371         if (!adr_type->isa_aryptr() || !adr_type->is_flat()) {
2372           continue;
2373         }
2374         current->set_memory_at(j, mm);
2375       }
2376       current->set_memory_at(index, current->base_memory());
2377     }
2378     igvn.optimize();
2379   }
2380   print_method(PHASE_SPLIT_INLINES_ARRAY, 2);
2381 #ifdef ASSERT
2382   if (!_flat_accesses_share_alias) {
2383     wq.clear();
2384     wq.push(root());
2385     for (uint i = 0; i < wq.size(); i++) {
2386       Node* n = wq.at(i);
2387       assert(n->adr_type() != TypeAryPtr::INLINES, "should have been removed from the graph");
2388       for (uint j = 0; j < n->req(); j++) {
2389         Node* m = n->in(j);
2390         if (m != nullptr) {
2391           wq.push(m);
2392         }
2393       }
2394     }
2395   }
2396 #endif
2397 }
2398 
2399 void Compile::record_for_merge_stores_igvn(Node* n) {
2400   if (!n->for_merge_stores_igvn()) {
2401     assert(!_for_merge_stores_igvn.contains(n), "duplicate");
2402     n->add_flag(Node::NodeFlags::Flag_for_merge_stores_igvn);
2403     _for_merge_stores_igvn.append(n);
2404   }
2405 }
2406 
2407 void Compile::remove_from_merge_stores_igvn(Node* n) {
2408   n->remove_flag(Node::NodeFlags::Flag_for_merge_stores_igvn);
2409   _for_merge_stores_igvn.remove(n);
2410 }
2411 
2412 // We need to wait with merging stores until RangeCheck smearing has removed the RangeChecks during
2413 // the post loops IGVN phase. If we do it earlier, then there may still be some RangeChecks between
2414 // the stores, and we merge the wrong sequence of stores.
2415 // Example:
2416 //   StoreI RangeCheck StoreI StoreI RangeCheck StoreI
2417 // Apply MergeStores:
2418 //   StoreI RangeCheck [   StoreL  ] RangeCheck StoreI
2419 // Remove more RangeChecks:
2420 //   StoreI            [   StoreL  ]            StoreI
2421 // But now it would have been better to do this instead:
2422 //   [         StoreL       ] [       StoreL         ]
2423 //
2424 // Note: we allow stores to merge in this dedicated IGVN round, and any later IGVN round,
2425 //       since we never unset _merge_stores_phase.
2426 void Compile::process_for_merge_stores_igvn(PhaseIterGVN& igvn) {
2427   C->set_merge_stores_phase();
2428 
2429   if (_for_merge_stores_igvn.length() > 0) {
2430     while (_for_merge_stores_igvn.length() > 0) {
2431       Node* n = _for_merge_stores_igvn.pop();
2432       n->remove_flag(Node::NodeFlags::Flag_for_merge_stores_igvn);
2433       igvn._worklist.push(n);
2434     }
2435     igvn.optimize();
2436     if (failing()) return;
2437     assert(_for_merge_stores_igvn.length() == 0, "no more delayed nodes allowed");
2438     print_method(PHASE_AFTER_MERGE_STORES, 3);
2439   }
2440 }
2441 
2442 void Compile::record_unstable_if_trap(UnstableIfTrap* trap) {
2443   if (OptimizeUnstableIf) {
2444     _unstable_if_traps.append(trap);
2445   }
2446 }
2447 
2448 void Compile::remove_useless_unstable_if_traps(Unique_Node_List& useful) {
2449   for (int i = _unstable_if_traps.length() - 1; i >= 0; i--) {
2450     UnstableIfTrap* trap = _unstable_if_traps.at(i);
2451     Node* n = trap->uncommon_trap();
2452     if (!useful.member(n)) {
2453       _unstable_if_traps.delete_at(i); // replaces i-th with last element which is known to be useful (already processed)
2454     }
2455   }
2456 }
2457 
2458 // Remove the unstable if trap associated with 'unc' from candidates. It is either dead
2459 // or fold-compares case. Return true if succeed or not found.
2460 //
2461 // In rare cases, the found trap has been processed. It is too late to delete it. Return
2462 // false and ask fold-compares to yield.
2463 //
2464 // 'fold-compares' may use the uncommon_trap of the dominating IfNode to cover the fused
2465 // IfNode. This breaks the unstable_if trap invariant: control takes the unstable path
2466 // when deoptimization does happen.
2467 bool Compile::remove_unstable_if_trap(CallStaticJavaNode* unc, bool yield) {
2468   for (int i = 0; i < _unstable_if_traps.length(); ++i) {
2469     UnstableIfTrap* trap = _unstable_if_traps.at(i);
2470     if (trap->uncommon_trap() == unc) {
2471       if (yield && trap->modified()) {
2472         return false;
2473       }
2474       _unstable_if_traps.delete_at(i);
2475       break;
2476     }
2477   }
2478   return true;
2479 }
2480 
2481 // Re-calculate unstable_if traps with the liveness of next_bci, which points to the unlikely path.
2482 // It needs to be done after igvn because fold-compares may fuse uncommon_traps and before renumbering.
2483 void Compile::process_for_unstable_if_traps(PhaseIterGVN& igvn) {
2484   for (int i = _unstable_if_traps.length() - 1; i >= 0; --i) {
2485     UnstableIfTrap* trap = _unstable_if_traps.at(i);
2486     CallStaticJavaNode* unc = trap->uncommon_trap();
2487     int next_bci = trap->next_bci();
2488     bool modified = trap->modified();
2489 
2490     if (next_bci != -1 && !modified) {
2491       assert(!_dead_node_list.test(unc->_idx), "changing a dead node!");
2492       JVMState* jvms = unc->jvms();
2493       ciMethod* method = jvms->method();
2494       ciBytecodeStream iter(method);
2495 
2496       iter.force_bci(jvms->bci());
2497       assert(next_bci == iter.next_bci() || next_bci == iter.get_dest(), "wrong next_bci at unstable_if");
2498       Bytecodes::Code c = iter.cur_bc();
2499       Node* lhs = nullptr;
2500       Node* rhs = nullptr;
2501       if (c == Bytecodes::_if_acmpeq || c == Bytecodes::_if_acmpne) {
2502         lhs = unc->peek_operand(0);
2503         rhs = unc->peek_operand(1);
2504       } else if (c == Bytecodes::_ifnull || c == Bytecodes::_ifnonnull) {
2505         lhs = unc->peek_operand(0);
2506       }
2507 
2508       ResourceMark rm;
2509       const MethodLivenessResult& live_locals = method->liveness_at_bci(next_bci);
2510       assert(live_locals.is_valid(), "broken liveness info");
2511       int len = (int)live_locals.size();
2512 
2513       for (int i = 0; i < len; i++) {
2514         Node* local = unc->local(jvms, i);
2515         // kill local using the liveness of next_bci.
2516         // give up when the local looks like an operand to secure reexecution.
2517         if (!live_locals.at(i) && !local->is_top() && local != lhs && local != rhs) {
2518           uint idx = jvms->locoff() + i;
2519 #ifdef ASSERT
2520           if (PrintOpto && Verbose) {
2521             tty->print("[unstable_if] kill local#%d: ", idx);
2522             local->dump();
2523             tty->cr();
2524           }
2525 #endif
2526           igvn.replace_input_of(unc, idx, top());
2527           modified = true;
2528         }
2529       }
2530     }
2531 
2532     // keep the modified trap for late query
2533     if (modified) {
2534       trap->set_modified();
2535     } else {
2536       _unstable_if_traps.delete_at(i);
2537     }
2538   }
2539   igvn.optimize();
2540 }
2541 
2542 // StringOpts and late inlining of string methods
2543 void Compile::inline_string_calls(bool parse_time) {
2544   {
2545     // remove useless nodes to make the usage analysis simpler
2546     ResourceMark rm;
2547     PhaseRemoveUseless pru(initial_gvn(), *igvn_worklist());
2548   }
2549 
2550   {
2551     ResourceMark rm;
2552     print_method(PHASE_BEFORE_STRINGOPTS, 3);
2553     PhaseStringOpts pso(initial_gvn());
2554     print_method(PHASE_AFTER_STRINGOPTS, 3);
2555   }
2556 
2557   // now inline anything that we skipped the first time around
2558   if (!parse_time) {
2559     _late_inlines_pos = _late_inlines.length();
2560   }
2561 
2562   while (_string_late_inlines.length() > 0) {
2563     CallGenerator* cg = _string_late_inlines.pop();
2564     cg->do_late_inline();
2565     if (failing())  return;
2566   }
2567   _string_late_inlines.trunc_to(0);
2568 }
2569 
2570 // Late inlining of boxing methods
2571 void Compile::inline_boxing_calls(PhaseIterGVN& igvn) {
2572   if (_boxing_late_inlines.length() > 0) {
2573     assert(has_boxed_value(), "inconsistent");
2574 
2575     set_inlining_incrementally(true);
2576 
2577     igvn_worklist()->ensure_empty(); // should be done with igvn
2578 
2579     _late_inlines_pos = _late_inlines.length();
2580 
2581     while (_boxing_late_inlines.length() > 0) {
2582       CallGenerator* cg = _boxing_late_inlines.pop();
2583       cg->do_late_inline();
2584       if (failing())  return;
2585     }
2586     _boxing_late_inlines.trunc_to(0);
2587 
2588     inline_incrementally_cleanup(igvn);
2589 
2590     set_inlining_incrementally(false);
2591   }
2592 }
2593 
2594 bool Compile::inline_incrementally_one() {
2595   assert(IncrementalInline, "incremental inlining should be on");
2596 
2597   TracePhase tp(_t_incrInline_inline);
2598 
2599   set_inlining_progress(false);
2600   set_do_cleanup(false);
2601 
2602   for (int i = 0; i < _late_inlines.length(); i++) {
2603     _late_inlines_pos = i+1;
2604     CallGenerator* cg = _late_inlines.at(i);
2605     bool is_scheduled_for_igvn_before = C->igvn_worklist()->member(cg->call_node());
2606     bool does_dispatch = cg->is_virtual_late_inline() || cg->is_mh_late_inline();
2607     if (inlining_incrementally() || does_dispatch) { // a call can be either inlined or strength-reduced to a direct call
2608       if (should_stress_inlining()) {
2609         // randomly add repeated inline attempt if stress-inlining
2610         cg->call_node()->set_generator(cg);
2611         C->igvn_worklist()->push(cg->call_node());
2612         continue;
2613       }
2614       cg->do_late_inline();
2615       assert(_late_inlines.at(i) == cg, "no insertions before current position allowed");
2616       if (failing()) {
2617         return false;
2618       } else if (inlining_progress()) {
2619         _late_inlines_pos = i+1; // restore the position in case new elements were inserted
2620         print_method(PHASE_INCREMENTAL_INLINE_STEP, 3, cg->call_node());
2621         break; // process one call site at a time
2622       } else {
2623         bool is_scheduled_for_igvn_after = C->igvn_worklist()->member(cg->call_node());
2624         if (!is_scheduled_for_igvn_before && is_scheduled_for_igvn_after) {
2625           // Avoid potential infinite loop if node already in the IGVN list
2626           assert(false, "scheduled for IGVN during inlining attempt");
2627         } else {
2628           // Ensure call node has not disappeared from IGVN worklist during a failed inlining attempt
2629           assert(!is_scheduled_for_igvn_before || is_scheduled_for_igvn_after, "call node removed from IGVN list during inlining pass");
2630           cg->call_node()->set_generator(cg);
2631         }
2632       }
2633     } else {
2634       // Ignore late inline direct calls when inlining is not allowed.
2635       // They are left in the late inline list when node budget is exhausted until the list is fully drained.
2636     }
2637   }
2638   // Remove processed elements.
2639   _late_inlines.remove_till(_late_inlines_pos);
2640   _late_inlines_pos = 0;
2641 
2642   assert(inlining_progress() || _late_inlines.length() == 0, "no progress");
2643 
2644   bool needs_cleanup = do_cleanup() || over_inlining_cutoff();
2645 
2646   set_inlining_progress(false);
2647   set_do_cleanup(false);
2648 
2649   bool force_cleanup = directive()->IncrementalInlineForceCleanupOption;
2650   return (_late_inlines.length() > 0) && !needs_cleanup && !force_cleanup;
2651 }
2652 
2653 void Compile::inline_incrementally_cleanup(PhaseIterGVN& igvn) {
2654   {
2655     TracePhase tp(_t_incrInline_pru);
2656     ResourceMark rm;
2657     PhaseRemoveUseless pru(initial_gvn(), *igvn_worklist());
2658   }
2659   {
2660     TracePhase tp(_t_incrInline_igvn);
2661     igvn.reset();
2662     igvn.optimize();
2663     if (failing()) return;
2664   }
2665   print_method(PHASE_INCREMENTAL_INLINE_CLEANUP, 3);
2666 }
2667 
2668 // Perform incremental inlining until bound on number of live nodes is reached
2669 void Compile::inline_incrementally(PhaseIterGVN& igvn) {
2670   TracePhase tp(_t_incrInline);
2671 
2672   set_inlining_incrementally(true);
2673   uint low_live_nodes = 0;
2674 
2675   while (_late_inlines.length() > 0) {
2676     if (live_nodes() > (uint)LiveNodeCountInliningCutoff) {
2677       if (low_live_nodes < (uint)LiveNodeCountInliningCutoff * 8 / 10) {
2678         TracePhase tp(_t_incrInline_ideal);
2679         // PhaseIdealLoop is expensive so we only try it once we are
2680         // out of live nodes and we only try it again if the previous
2681         // helped got the number of nodes down significantly
2682         PhaseIdealLoop::optimize(igvn, LoopOptsNone);
2683         if (failing())  return;
2684         low_live_nodes = live_nodes();
2685         _major_progress = true;
2686       }
2687 
2688       if (live_nodes() > (uint)LiveNodeCountInliningCutoff) {
2689         bool do_print_inlining = print_inlining() || print_intrinsics();
2690         if (do_print_inlining || log() != nullptr) {
2691           // Print inlining message for candidates that we couldn't inline for lack of space.
2692           for (int i = 0; i < _late_inlines.length(); i++) {
2693             CallGenerator* cg = _late_inlines.at(i);
2694             const char* msg = "live nodes > LiveNodeCountInliningCutoff";
2695             if (do_print_inlining) {
2696               inline_printer()->record(cg->method(), cg->call_node()->jvms(), InliningResult::FAILURE, msg);
2697             }
2698             log_late_inline_failure(cg, msg);
2699           }
2700         }
2701         break; // finish
2702       }
2703     }
2704 
2705     igvn_worklist()->ensure_empty(); // should be done with igvn
2706 
2707     while (inline_incrementally_one()) {
2708       assert(!failing_internal() || failure_is_artificial(), "inconsistent");
2709     }
2710     if (failing())  return;
2711 
2712     inline_incrementally_cleanup(igvn);
2713 
2714     print_method(PHASE_INCREMENTAL_INLINE_STEP, 3);
2715 
2716     if (failing())  return;
2717 
2718     if (_late_inlines.length() == 0) {
2719       break; // no more progress
2720     }
2721   }
2722 
2723   igvn_worklist()->ensure_empty(); // should be done with igvn
2724 
2725   if (_string_late_inlines.length() > 0) {
2726     assert(has_stringbuilder(), "inconsistent");
2727 
2728     inline_string_calls(false);
2729 
2730     if (failing())  return;
2731 
2732     inline_incrementally_cleanup(igvn);
2733   }
2734 
2735   set_inlining_incrementally(false);
2736 }
2737 
2738 void Compile::process_late_inline_calls_no_inline(PhaseIterGVN& igvn) {
2739   // "inlining_incrementally() == false" is used to signal that no inlining is allowed
2740   // (see LateInlineVirtualCallGenerator::do_late_inline_check() for details).
2741   // Tracking and verification of modified nodes is disabled by setting "_modified_nodes == nullptr"
2742   // as if "inlining_incrementally() == true" were set.
2743   assert(inlining_incrementally() == false, "not allowed");
2744 #ifdef ASSERT
2745   Unique_Node_List* modified_nodes = _modified_nodes;
2746   _modified_nodes = nullptr;
2747 #endif
2748   assert(_late_inlines.length() > 0, "sanity");
2749 
2750   while (_late_inlines.length() > 0) {
2751     igvn_worklist()->ensure_empty(); // should be done with igvn
2752 
2753     while (inline_incrementally_one()) {
2754       assert(!failing_internal() || failure_is_artificial(), "inconsistent");
2755     }
2756     if (failing())  return;
2757 
2758     inline_incrementally_cleanup(igvn);
2759   }
2760   DEBUG_ONLY( _modified_nodes = modified_nodes; )
2761 }
2762 
2763 bool Compile::optimize_loops(PhaseIterGVN& igvn, LoopOptsMode mode) {
2764   if (_loop_opts_cnt > 0) {
2765     while (major_progress() && (_loop_opts_cnt > 0)) {
2766       TracePhase tp(_t_idealLoop);
2767       PhaseIdealLoop::optimize(igvn, mode);
2768       _loop_opts_cnt--;
2769       if (failing())  return false;
2770       if (major_progress()) print_method(PHASE_PHASEIDEALLOOP_ITERATIONS, 2);
2771     }
2772   }
2773   return true;
2774 }
2775 
2776 // Remove edges from "root" to each SafePoint at a backward branch.
2777 // They were inserted during parsing (see add_safepoint()) to make
2778 // infinite loops without calls or exceptions visible to root, i.e.,
2779 // useful.
2780 void Compile::remove_root_to_sfpts_edges(PhaseIterGVN& igvn) {
2781   Node *r = root();
2782   if (r != nullptr) {
2783     for (uint i = r->req(); i < r->len(); ++i) {
2784       Node *n = r->in(i);
2785       if (n != nullptr && n->is_SafePoint()) {
2786         r->rm_prec(i);
2787         if (n->outcnt() == 0) {
2788           igvn.remove_dead_node(n);
2789         }
2790         --i;
2791       }
2792     }
2793     // Parsing may have added top inputs to the root node (Path
2794     // leading to the Halt node proven dead). Make sure we get a
2795     // chance to clean them up.
2796     igvn._worklist.push(r);
2797     igvn.optimize();
2798   }
2799 }
2800 
2801 //------------------------------Optimize---------------------------------------
2802 // Given a graph, optimize it.
2803 void Compile::Optimize() {
2804   TracePhase tp(_t_optimizer);
2805 
2806 #ifndef PRODUCT
2807   if (env()->break_at_compile()) {
2808     BREAKPOINT;
2809   }
2810 
2811 #endif
2812 
2813   BarrierSetC2* bs = BarrierSet::barrier_set()->barrier_set_c2();
2814 #ifdef ASSERT
2815   bs->verify_gc_barriers(this, BarrierSetC2::BeforeOptimize);
2816 #endif
2817 
2818   ResourceMark rm;
2819 
2820   NOT_PRODUCT( verify_graph_edges(); )
2821 
2822   print_method(PHASE_AFTER_PARSING, 1);
2823 
2824  {
2825   // Iterative Global Value Numbering, including ideal transforms
2826   PhaseIterGVN igvn;
2827 #ifdef ASSERT
2828   _modified_nodes = new (comp_arena()) Unique_Node_List(comp_arena());
2829 #endif
2830   {
2831     TracePhase tp(_t_iterGVN);
2832     igvn.optimize();
2833   }
2834 
2835   if (failing())  return;
2836 
2837   print_method(PHASE_ITER_GVN1, 2);
2838 
2839   process_for_unstable_if_traps(igvn);
2840 
2841   if (failing())  return;
2842 
2843   inline_incrementally(igvn);
2844 
2845   print_method(PHASE_INCREMENTAL_INLINE, 2);
2846 
2847   if (failing())  return;
2848 
2849   if (eliminate_boxing()) {
2850     // Inline valueOf() methods now.
2851     inline_boxing_calls(igvn);
2852 
2853     if (failing())  return;
2854 
2855     if (AlwaysIncrementalInline || StressIncrementalInlining) {
2856       inline_incrementally(igvn);
2857     }
2858 
2859     print_method(PHASE_INCREMENTAL_BOXING_INLINE, 2);
2860 
2861     if (failing())  return;
2862   }
2863 
2864   // Remove the speculative part of types and clean up the graph from
2865   // the extra CastPP nodes whose only purpose is to carry them. Do
2866   // that early so that optimizations are not disrupted by the extra
2867   // CastPP nodes.
2868   remove_speculative_types(igvn);
2869 
2870   if (failing())  return;
2871 
2872   // No more new expensive nodes will be added to the list from here
2873   // so keep only the actual candidates for optimizations.
2874   cleanup_expensive_nodes(igvn);
2875 
2876   if (failing())  return;
2877 
2878   assert(EnableVectorSupport || !has_vbox_nodes(), "sanity");
2879   if (EnableVectorSupport && has_vbox_nodes()) {
2880     TracePhase tp(_t_vector);
2881     PhaseVector pv(igvn);
2882     pv.optimize_vector_boxes();
2883     if (failing())  return;
2884     print_method(PHASE_ITER_GVN_AFTER_VECTOR, 2);
2885   }
2886   assert(!has_vbox_nodes(), "sanity");
2887 
2888   if (!failing() && RenumberLiveNodes && live_nodes() + NodeLimitFudgeFactor < unique()) {
2889     Compile::TracePhase tp(_t_renumberLive);
2890     igvn_worklist()->ensure_empty(); // should be done with igvn
2891     {
2892       ResourceMark rm;
2893       PhaseRenumberLive prl(initial_gvn(), *igvn_worklist());
2894     }
2895     igvn.reset();
2896     igvn.optimize();
2897     if (failing()) return;
2898   }
2899 
2900   // Now that all inlining is over and no PhaseRemoveUseless will run, cut edge from root to loop
2901   // safepoints
2902   remove_root_to_sfpts_edges(igvn);
2903 
2904   // Process inline type nodes now that all inlining is over
2905   process_inline_types(igvn);
2906 
2907   adjust_flat_array_access_aliases(igvn);
2908 
2909   if (failing())  return;
2910 
2911   if (C->macro_count() > 0) {
2912     // Eliminate some macro nodes before EA to reduce analysis pressure
2913     PhaseMacroExpand mexp(igvn);
2914     mexp.eliminate_macro_nodes(/* eliminate_locks= */ false);
2915     if (failing()) {
2916       return;
2917     }
2918     igvn.set_delay_transform(false);
2919     print_method(PHASE_ITER_GVN_AFTER_ELIMINATION, 2);
2920   }
2921 
2922   if (has_loops()) {
2923     print_method(PHASE_BEFORE_LOOP_OPTS, 2);
2924   }
2925 
2926   // Perform escape analysis
2927   if (do_escape_analysis() && ConnectionGraph::has_candidates(this)) {
2928     if (has_loops()) {
2929       // Cleanup graph (remove dead nodes).
2930       TracePhase tp(_t_idealLoop);
2931       PhaseIdealLoop::optimize(igvn, LoopOptsMaxUnroll);
2932       if (failing()) {
2933         return;
2934       }
2935       print_method(PHASE_PHASEIDEAL_BEFORE_EA, 2);
2936       if (C->macro_count() > 0) {
2937         // Eliminate some macro nodes before EA to reduce analysis pressure
2938         PhaseMacroExpand mexp(igvn);
2939         mexp.eliminate_macro_nodes(/* eliminate_locks= */ false);
2940         if (failing()) {
2941           return;
2942         }
2943         igvn.set_delay_transform(false);
2944         print_method(PHASE_ITER_GVN_AFTER_ELIMINATION, 2);
2945       }
2946     }
2947 
2948     bool progress;
2949     do {
2950       ConnectionGraph::do_analysis(this, &igvn);
2951 
2952       if (failing())  return;
2953 
2954       int mcount = macro_count(); // Record number of allocations and locks before IGVN
2955 
2956       // Optimize out fields loads from scalar replaceable allocations.
2957       igvn.optimize();
2958       print_method(PHASE_ITER_GVN_AFTER_EA, 2);
2959 
2960       if (failing()) return;
2961 
2962       if (congraph() != nullptr && macro_count() > 0) {
2963         TracePhase tp(_t_macroEliminate);
2964         PhaseMacroExpand mexp(igvn);
2965         mexp.eliminate_macro_nodes();
2966         if (failing()) {
2967           return;
2968         }
2969         print_method(PHASE_AFTER_MACRO_ELIMINATION, 2);
2970 
2971         igvn.set_delay_transform(false);
2972         print_method(PHASE_ITER_GVN_AFTER_ELIMINATION, 2);
2973       }
2974 
2975       ConnectionGraph::verify_ram_nodes(this, root());
2976       if (failing())  return;
2977 
2978       progress = do_iterative_escape_analysis() &&
2979                  (macro_count() < mcount) &&
2980                  ConnectionGraph::has_candidates(this);
2981       // Try again if candidates exist and made progress
2982       // by removing some allocations and/or locks.
2983     } while (progress);
2984   }
2985 
2986   process_flat_accesses(igvn);
2987   if (failing()) {
2988     return;
2989   }
2990 
2991   // Loop transforms on the ideal graph.  Range Check Elimination,
2992   // peeling, unrolling, etc.
2993 
2994   // Set loop opts counter
2995   if((_loop_opts_cnt > 0) && (has_loops() || has_split_ifs())) {
2996     {
2997       TracePhase tp(_t_idealLoop);
2998       PhaseIdealLoop::optimize(igvn, LoopOptsDefault);
2999       _loop_opts_cnt--;
3000       if (major_progress()) print_method(PHASE_PHASEIDEALLOOP1, 2);
3001       if (failing())  return;
3002     }
3003     // Loop opts pass if partial peeling occurred in previous pass
3004     if(PartialPeelLoop && major_progress() && (_loop_opts_cnt > 0)) {
3005       TracePhase tp(_t_idealLoop);
3006       PhaseIdealLoop::optimize(igvn, LoopOptsSkipSplitIf);
3007       _loop_opts_cnt--;
3008       if (major_progress()) print_method(PHASE_PHASEIDEALLOOP2, 2);
3009       if (failing())  return;
3010     }
3011     // Loop opts pass for loop-unrolling before CCP
3012     if(major_progress() && (_loop_opts_cnt > 0)) {
3013       TracePhase tp(_t_idealLoop);
3014       PhaseIdealLoop::optimize(igvn, LoopOptsSkipSplitIf);
3015       _loop_opts_cnt--;
3016       if (major_progress()) print_method(PHASE_PHASEIDEALLOOP3, 2);
3017     }
3018     if (!failing()) {
3019       // Verify that last round of loop opts produced a valid graph
3020       PhaseIdealLoop::verify(igvn);
3021     }
3022   }
3023   if (failing())  return;
3024 
3025   // Conditional Constant Propagation;
3026   print_method(PHASE_BEFORE_CCP1, 2);
3027   PhaseCCP ccp( &igvn );
3028   assert( true, "Break here to ccp.dump_nodes_and_types(_root,999,1)");
3029   {
3030     TracePhase tp(_t_ccp);
3031     ccp.do_transform();
3032   }
3033   print_method(PHASE_CCP1, 2);
3034 
3035   assert( true, "Break here to ccp.dump_old2new_map()");
3036 
3037   // Iterative Global Value Numbering, including ideal transforms
3038   {
3039     TracePhase tp(_t_iterGVN2);
3040     igvn.reset_from_igvn(&ccp);
3041     igvn.optimize();
3042   }
3043   print_method(PHASE_ITER_GVN2, 2);
3044 
3045   if (failing())  return;
3046 
3047   // Loop transforms on the ideal graph.  Range Check Elimination,
3048   // peeling, unrolling, etc.
3049   if (!optimize_loops(igvn, LoopOptsDefault)) {
3050     return;
3051   }
3052 
3053   if (failing())  return;
3054 
3055   C->clear_major_progress(); // ensure that major progress is now clear
3056 
3057   process_for_post_loop_opts_igvn(igvn);
3058 
3059   process_for_merge_stores_igvn(igvn);
3060 
3061   if (failing())  return;
3062 
3063 #ifdef ASSERT
3064   bs->verify_gc_barriers(this, BarrierSetC2::BeforeMacroExpand);
3065 #endif
3066 
3067   assert(_late_inlines.length() == 0 || IncrementalInlineMH || IncrementalInlineVirtual, "not empty");
3068 
3069   if (_late_inlines.length() > 0) {
3070     // More opportunities to optimize virtual and MH calls.
3071     // Though it's maybe too late to perform inlining, strength-reducing them to direct calls is still an option.
3072     process_late_inline_calls_no_inline(igvn);
3073   }
3074 
3075   {
3076     TracePhase tp(_t_macroExpand);
3077     PhaseMacroExpand mex(igvn);
3078     // Last attempt to eliminate macro nodes.
3079     mex.eliminate_macro_nodes();
3080     if (failing()) {
3081       return;
3082     }
3083 
3084     print_method(PHASE_BEFORE_MACRO_EXPANSION, 3);
3085     // Do not allow new macro nodes once we start to eliminate and expand
3086     C->reset_allow_macro_nodes();
3087     // Last attempt to eliminate macro nodes before expand
3088     mex.eliminate_macro_nodes();
3089     if (failing()) {
3090       return;
3091     }
3092     mex.eliminate_opaque_looplimit_macro_nodes();
3093     if (failing()) {
3094       return;
3095     }
3096     print_method(PHASE_AFTER_MACRO_ELIMINATION, 2);
3097     if (mex.expand_macro_nodes()) {
3098       assert(failing(), "must bail out w/ explicit message");
3099       return;
3100     }
3101     print_method(PHASE_AFTER_MACRO_EXPANSION, 2);
3102   }
3103 
3104   // Process inline type nodes again and remove them. From here
3105   // on we don't need to keep track of field values anymore.
3106   process_inline_types(igvn, /* remove= */ true);
3107 
3108   {
3109     TracePhase tp(_t_barrierExpand);
3110     if (bs->expand_barriers(this, igvn)) {
3111       assert(failing(), "must bail out w/ explicit message");
3112       return;
3113     }
3114     print_method(PHASE_BARRIER_EXPANSION, 2);
3115   }
3116 
3117   if (C->max_vector_size() > 0) {
3118     C->optimize_logic_cones(igvn);
3119     igvn.optimize();
3120     if (failing()) return;
3121   }
3122 
3123   DEBUG_ONLY( _modified_nodes = nullptr; )
3124   DEBUG_ONLY( _late_inlines.clear(); )
3125 
3126   assert(igvn._worklist.size() == 0, "not empty");
3127  } // (End scope of igvn; run destructor if necessary for asserts.)
3128 
3129  check_no_dead_use();
3130 
3131  // We will never use the NodeHash table any more. Clear it so that final_graph_reshaping does not have
3132  // to remove hashes to unlock nodes for modifications.
3133  C->node_hash()->clear();
3134 
3135  // A method with only infinite loops has no edges entering loops from root
3136  {
3137    TracePhase tp(_t_graphReshaping);
3138    if (final_graph_reshaping()) {
3139      assert(failing(), "must bail out w/ explicit message");
3140      return;
3141    }
3142  }
3143 
3144  print_method(PHASE_OPTIMIZE_FINISHED, 2);
3145  DEBUG_ONLY(set_phase_optimize_finished();)
3146 }
3147 
3148 #ifdef ASSERT
3149 void Compile::check_no_dead_use() const {
3150   ResourceMark rm;
3151   Unique_Node_List wq;
3152   wq.push(root());
3153   for (uint i = 0; i < wq.size(); ++i) {
3154     Node* n = wq.at(i);
3155     for (DUIterator_Fast jmax, j = n->fast_outs(jmax); j < jmax; j++) {
3156       Node* u = n->fast_out(j);
3157       if (u->outcnt() == 0 && !u->is_Con()) {
3158         u->dump();
3159         fatal("no reachable node should have no use");
3160       }
3161       wq.push(u);
3162     }
3163   }
3164 }
3165 #endif
3166 
3167 void Compile::inline_vector_reboxing_calls() {
3168   if (C->_vector_reboxing_late_inlines.length() > 0) {
3169     _late_inlines_pos = C->_late_inlines.length();
3170     while (_vector_reboxing_late_inlines.length() > 0) {
3171       CallGenerator* cg = _vector_reboxing_late_inlines.pop();
3172       cg->do_late_inline();
3173       if (failing())  return;
3174       print_method(PHASE_INLINE_VECTOR_REBOX, 3, cg->call_node());
3175     }
3176     _vector_reboxing_late_inlines.trunc_to(0);
3177   }
3178 }
3179 
3180 bool Compile::has_vbox_nodes() {
3181   if (C->_vector_reboxing_late_inlines.length() > 0) {
3182     return true;
3183   }
3184   for (int macro_idx = C->macro_count() - 1; macro_idx >= 0; macro_idx--) {
3185     Node * n = C->macro_node(macro_idx);
3186     assert(n->is_macro(), "only macro nodes expected here");
3187     if (n->Opcode() == Op_VectorUnbox || n->Opcode() == Op_VectorBox || n->Opcode() == Op_VectorBoxAllocate) {
3188       return true;
3189     }
3190   }
3191   return false;
3192 }
3193 
3194 //---------------------------- Bitwise operation packing optimization ---------------------------
3195 
3196 static bool is_vector_unary_bitwise_op(Node* n) {
3197   return n->Opcode() == Op_XorV &&
3198          VectorNode::is_vector_bitwise_not_pattern(n);
3199 }
3200 
3201 static bool is_vector_binary_bitwise_op(Node* n) {
3202   switch (n->Opcode()) {
3203     case Op_AndV:
3204     case Op_OrV:
3205       return true;
3206 
3207     case Op_XorV:
3208       return !is_vector_unary_bitwise_op(n);
3209 
3210     default:
3211       return false;
3212   }
3213 }
3214 
3215 static bool is_vector_ternary_bitwise_op(Node* n) {
3216   return n->Opcode() == Op_MacroLogicV;
3217 }
3218 
3219 static bool is_vector_bitwise_op(Node* n) {
3220   return is_vector_unary_bitwise_op(n)  ||
3221          is_vector_binary_bitwise_op(n) ||
3222          is_vector_ternary_bitwise_op(n);
3223 }
3224 
3225 static bool is_vector_bitwise_cone_root(Node* n) {
3226   if (n->bottom_type()->isa_vectmask() || !is_vector_bitwise_op(n)) {
3227     return false;
3228   }
3229   for (DUIterator_Fast imax, i = n->fast_outs(imax); i < imax; i++) {
3230     if (is_vector_bitwise_op(n->fast_out(i))) {
3231       return false;
3232     }
3233   }
3234   return true;
3235 }
3236 
3237 static uint collect_unique_inputs(Node* n, Unique_Node_List& inputs) {
3238   uint cnt = 0;
3239   if (is_vector_bitwise_op(n)) {
3240     uint inp_cnt = n->is_predicated_vector() ? n->req()-1 : n->req();
3241     if (VectorNode::is_vector_bitwise_not_pattern(n)) {
3242       for (uint i = 1; i < inp_cnt; i++) {
3243         Node* in = n->in(i);
3244         bool skip = VectorNode::is_all_ones_vector(in);
3245         if (!skip && !inputs.member(in)) {
3246           inputs.push(in);
3247           cnt++;
3248         }
3249       }
3250       assert(cnt <= 1, "not unary");
3251     } else {
3252       uint last_req = inp_cnt;
3253       if (is_vector_ternary_bitwise_op(n)) {
3254         last_req = inp_cnt - 1; // skip last input
3255       }
3256       for (uint i = 1; i < last_req; i++) {
3257         Node* def = n->in(i);
3258         if (!inputs.member(def)) {
3259           inputs.push(def);
3260           cnt++;
3261         }
3262       }
3263     }
3264   } else { // not a bitwise operations
3265     if (!inputs.member(n)) {
3266       inputs.push(n);
3267       cnt++;
3268     }
3269   }
3270   return cnt;
3271 }
3272 
3273 void Compile::collect_logic_cone_roots(Unique_Node_List& list) {
3274   Unique_Node_List useful_nodes;
3275   C->identify_useful_nodes(useful_nodes);
3276 
3277   for (uint i = 0; i < useful_nodes.size(); i++) {
3278     Node* n = useful_nodes.at(i);
3279     if (is_vector_bitwise_cone_root(n)) {
3280       list.push(n);
3281     }
3282   }
3283 }
3284 
3285 Node* Compile::xform_to_MacroLogicV(PhaseIterGVN& igvn,
3286                                     const TypeVect* vt,
3287                                     Unique_Node_List& partition,
3288                                     Unique_Node_List& inputs) {
3289   assert(partition.size() == 2 || partition.size() == 3, "not supported");
3290   assert(inputs.size()    == 2 || inputs.size()    == 3, "not supported");
3291   assert(Matcher::match_rule_supported_vector(Op_MacroLogicV, vt->length(), vt->element_basic_type()), "not supported");
3292 
3293   Node* in1 = inputs.at(0);
3294   Node* in2 = inputs.at(1);
3295   Node* in3 = (inputs.size() == 3 ? inputs.at(2) : in2);
3296 
3297   uint func = compute_truth_table(partition, inputs);
3298 
3299   Node* pn = partition.at(partition.size() - 1);
3300   Node* mask = pn->is_predicated_vector() ? pn->in(pn->req()-1) : nullptr;
3301   return igvn.transform(MacroLogicVNode::make(igvn, in1, in2, in3, mask, func, vt));
3302 }
3303 
3304 static uint extract_bit(uint func, uint pos) {
3305   return (func & (1 << pos)) >> pos;
3306 }
3307 
3308 //
3309 //  A macro logic node represents a truth table. It has 4 inputs,
3310 //  First three inputs corresponds to 3 columns of a truth table
3311 //  and fourth input captures the logic function.
3312 //
3313 //  eg.  fn = (in1 AND in2) OR in3;
3314 //
3315 //      MacroNode(in1,in2,in3,fn)
3316 //
3317 //  -----------------
3318 //  in1 in2 in3  fn
3319 //  -----------------
3320 //  0    0   0    0
3321 //  0    0   1    1
3322 //  0    1   0    0
3323 //  0    1   1    1
3324 //  1    0   0    0
3325 //  1    0   1    1
3326 //  1    1   0    1
3327 //  1    1   1    1
3328 //
3329 
3330 uint Compile::eval_macro_logic_op(uint func, uint in1 , uint in2, uint in3) {
3331   int res = 0;
3332   for (int i = 0; i < 8; i++) {
3333     int bit1 = extract_bit(in1, i);
3334     int bit2 = extract_bit(in2, i);
3335     int bit3 = extract_bit(in3, i);
3336 
3337     int func_bit_pos = (bit1 << 2 | bit2 << 1 | bit3);
3338     int func_bit = extract_bit(func, func_bit_pos);
3339 
3340     res |= func_bit << i;
3341   }
3342   return res;
3343 }
3344 
3345 static uint eval_operand(Node* n, HashTable<Node*,uint>& eval_map) {
3346   assert(n != nullptr, "");
3347   assert(eval_map.contains(n), "absent");
3348   return *(eval_map.get(n));
3349 }
3350 
3351 static void eval_operands(Node* n,
3352                           uint& func1, uint& func2, uint& func3,
3353                           HashTable<Node*,uint>& eval_map) {
3354   assert(is_vector_bitwise_op(n), "");
3355 
3356   if (is_vector_unary_bitwise_op(n)) {
3357     Node* opnd = n->in(1);
3358     if (VectorNode::is_vector_bitwise_not_pattern(n) && VectorNode::is_all_ones_vector(opnd)) {
3359       opnd = n->in(2);
3360     }
3361     func1 = eval_operand(opnd, eval_map);
3362   } else if (is_vector_binary_bitwise_op(n)) {
3363     func1 = eval_operand(n->in(1), eval_map);
3364     func2 = eval_operand(n->in(2), eval_map);
3365   } else {
3366     assert(is_vector_ternary_bitwise_op(n), "unknown operation");
3367     func1 = eval_operand(n->in(1), eval_map);
3368     func2 = eval_operand(n->in(2), eval_map);
3369     func3 = eval_operand(n->in(3), eval_map);
3370   }
3371 }
3372 
3373 uint Compile::compute_truth_table(Unique_Node_List& partition, Unique_Node_List& inputs) {
3374   assert(inputs.size() <= 3, "sanity");
3375   ResourceMark rm;
3376   uint res = 0;
3377   HashTable<Node*,uint> eval_map;
3378 
3379   // Populate precomputed functions for inputs.
3380   // Each input corresponds to one column of 3 input truth-table.
3381   uint input_funcs[] = { 0xAA,   // (_, _, c) -> c
3382                          0xCC,   // (_, b, _) -> b
3383                          0xF0 }; // (a, _, _) -> a
3384   for (uint i = 0; i < inputs.size(); i++) {
3385     eval_map.put(inputs.at(i), input_funcs[2-i]);
3386   }
3387 
3388   for (uint i = 0; i < partition.size(); i++) {
3389     Node* n = partition.at(i);
3390 
3391     uint func1 = 0, func2 = 0, func3 = 0;
3392     eval_operands(n, func1, func2, func3, eval_map);
3393 
3394     switch (n->Opcode()) {
3395       case Op_OrV:
3396         assert(func3 == 0, "not binary");
3397         res = func1 | func2;
3398         break;
3399       case Op_AndV:
3400         assert(func3 == 0, "not binary");
3401         res = func1 & func2;
3402         break;
3403       case Op_XorV:
3404         if (VectorNode::is_vector_bitwise_not_pattern(n)) {
3405           assert(func2 == 0 && func3 == 0, "not unary");
3406           res = (~func1) & 0xFF;
3407         } else {
3408           assert(func3 == 0, "not binary");
3409           res = func1 ^ func2;
3410         }
3411         break;
3412       case Op_MacroLogicV:
3413         // Ordering of inputs may change during evaluation of sub-tree
3414         // containing MacroLogic node as a child node, thus a re-evaluation
3415         // makes sure that function is evaluated in context of current
3416         // inputs.
3417         res = eval_macro_logic_op(n->in(4)->get_int(), func1, func2, func3);
3418         break;
3419 
3420       default: assert(false, "not supported: %s", n->Name());
3421     }
3422     assert(res <= 0xFF, "invalid");
3423     eval_map.put(n, res);
3424   }
3425   return res;
3426 }
3427 
3428 // Criteria under which nodes gets packed into a macro logic node:-
3429 //  1) Parent and both child nodes are all unmasked or masked with
3430 //     same predicates.
3431 //  2) Masked parent can be packed with left child if it is predicated
3432 //     and both have same predicates.
3433 //  3) Masked parent can be packed with right child if its un-predicated
3434 //     or has matching predication condition.
3435 //  4) An unmasked parent can be packed with an unmasked child.
3436 bool Compile::compute_logic_cone(Node* n, Unique_Node_List& partition, Unique_Node_List& inputs) {
3437   assert(partition.size() == 0, "not empty");
3438   assert(inputs.size() == 0, "not empty");
3439   if (is_vector_ternary_bitwise_op(n)) {
3440     return false;
3441   }
3442 
3443   bool is_unary_op = is_vector_unary_bitwise_op(n);
3444   if (is_unary_op) {
3445     assert(collect_unique_inputs(n, inputs) == 1, "not unary");
3446     return false; // too few inputs
3447   }
3448 
3449   bool pack_left_child = true;
3450   bool pack_right_child = true;
3451 
3452   bool left_child_LOP = is_vector_bitwise_op(n->in(1));
3453   bool right_child_LOP = is_vector_bitwise_op(n->in(2));
3454 
3455   int left_child_input_cnt = 0;
3456   int right_child_input_cnt = 0;
3457 
3458   bool parent_is_predicated = n->is_predicated_vector();
3459   bool left_child_predicated = n->in(1)->is_predicated_vector();
3460   bool right_child_predicated = n->in(2)->is_predicated_vector();
3461 
3462   Node* parent_pred = parent_is_predicated ? n->in(n->req()-1) : nullptr;
3463   Node* left_child_pred = left_child_predicated ? n->in(1)->in(n->in(1)->req()-1) : nullptr;
3464   Node* right_child_pred = right_child_predicated ? n->in(1)->in(n->in(1)->req()-1) : nullptr;
3465 
3466   do {
3467     if (pack_left_child && left_child_LOP &&
3468         ((!parent_is_predicated && !left_child_predicated) ||
3469         ((parent_is_predicated && left_child_predicated &&
3470           parent_pred == left_child_pred)))) {
3471        partition.push(n->in(1));
3472        left_child_input_cnt = collect_unique_inputs(n->in(1), inputs);
3473     } else {
3474        inputs.push(n->in(1));
3475        left_child_input_cnt = 1;
3476     }
3477 
3478     if (pack_right_child && right_child_LOP &&
3479         (!right_child_predicated ||
3480          (right_child_predicated && parent_is_predicated &&
3481           parent_pred == right_child_pred))) {
3482        partition.push(n->in(2));
3483        right_child_input_cnt = collect_unique_inputs(n->in(2), inputs);
3484     } else {
3485        inputs.push(n->in(2));
3486        right_child_input_cnt = 1;
3487     }
3488 
3489     if (inputs.size() > 3) {
3490       assert(partition.size() > 0, "");
3491       inputs.clear();
3492       partition.clear();
3493       if (left_child_input_cnt > right_child_input_cnt) {
3494         pack_left_child = false;
3495       } else {
3496         pack_right_child = false;
3497       }
3498     } else {
3499       break;
3500     }
3501   } while(true);
3502 
3503   if(partition.size()) {
3504     partition.push(n);
3505   }
3506 
3507   return (partition.size() == 2 || partition.size() == 3) &&
3508          (inputs.size()    == 2 || inputs.size()    == 3);
3509 }
3510 
3511 void Compile::process_logic_cone_root(PhaseIterGVN &igvn, Node *n, VectorSet &visited) {
3512   assert(is_vector_bitwise_op(n), "not a root");
3513 
3514   visited.set(n->_idx);
3515 
3516   // 1) Do a DFS walk over the logic cone.
3517   for (uint i = 1; i < n->req(); i++) {
3518     Node* in = n->in(i);
3519     if (!visited.test(in->_idx) && is_vector_bitwise_op(in)) {
3520       process_logic_cone_root(igvn, in, visited);
3521     }
3522   }
3523 
3524   // 2) Bottom up traversal: Merge node[s] with
3525   // the parent to form macro logic node.
3526   Unique_Node_List partition;
3527   Unique_Node_List inputs;
3528   if (compute_logic_cone(n, partition, inputs)) {
3529     const TypeVect* vt = n->bottom_type()->is_vect();
3530     Node* pn = partition.at(partition.size() - 1);
3531     Node* mask = pn->is_predicated_vector() ? pn->in(pn->req()-1) : nullptr;
3532     if (mask == nullptr ||
3533         Matcher::match_rule_supported_vector_masked(Op_MacroLogicV, vt->length(), vt->element_basic_type())) {
3534       Node* macro_logic = xform_to_MacroLogicV(igvn, vt, partition, inputs);
3535       VectorNode::trace_new_vector(macro_logic, "MacroLogic");
3536       igvn.replace_node(n, macro_logic);
3537     }
3538   }
3539 }
3540 
3541 void Compile::optimize_logic_cones(PhaseIterGVN &igvn) {
3542   ResourceMark rm;
3543   if (Matcher::match_rule_supported(Op_MacroLogicV)) {
3544     Unique_Node_List list;
3545     collect_logic_cone_roots(list);
3546 
3547     while (list.size() > 0) {
3548       Node* n = list.pop();
3549       const TypeVect* vt = n->bottom_type()->is_vect();
3550       bool supported = Matcher::match_rule_supported_vector(Op_MacroLogicV, vt->length(), vt->element_basic_type());
3551       if (supported) {
3552         VectorSet visited(comp_arena());
3553         process_logic_cone_root(igvn, n, visited);
3554       }
3555     }
3556   }
3557 }
3558 
3559 //------------------------------Code_Gen---------------------------------------
3560 // Given a graph, generate code for it
3561 void Compile::Code_Gen() {
3562   if (failing()) {
3563     return;
3564   }
3565 
3566   // Perform instruction selection.  You might think we could reclaim Matcher
3567   // memory PDQ, but actually the Matcher is used in generating spill code.
3568   // Internals of the Matcher (including some VectorSets) must remain live
3569   // for awhile - thus I cannot reclaim Matcher memory lest a VectorSet usage
3570   // set a bit in reclaimed memory.
3571 
3572   // In debug mode can dump m._nodes.dump() for mapping of ideal to machine
3573   // nodes.  Mapping is only valid at the root of each matched subtree.
3574   NOT_PRODUCT( verify_graph_edges(); )
3575 
3576   Matcher matcher;
3577   _matcher = &matcher;
3578   {
3579     TracePhase tp(_t_matcher);
3580     matcher.match();
3581     if (failing()) {
3582       return;
3583     }
3584   }
3585   // In debug mode can dump m._nodes.dump() for mapping of ideal to machine
3586   // nodes.  Mapping is only valid at the root of each matched subtree.
3587   NOT_PRODUCT( verify_graph_edges(); )
3588 
3589   // If you have too many nodes, or if matching has failed, bail out
3590   check_node_count(0, "out of nodes matching instructions");
3591   if (failing()) {
3592     return;
3593   }
3594 
3595   print_method(PHASE_MATCHING, 2);
3596 
3597   // Build a proper-looking CFG
3598   PhaseCFG cfg(node_arena(), root(), matcher);
3599   if (failing()) {
3600     return;
3601   }
3602   _cfg = &cfg;
3603   {
3604     TracePhase tp(_t_scheduler);
3605     bool success = cfg.do_global_code_motion();
3606     if (!success) {
3607       return;
3608     }
3609 
3610     print_method(PHASE_GLOBAL_CODE_MOTION, 2);
3611     NOT_PRODUCT( verify_graph_edges(); )
3612     cfg.verify();
3613     if (failing()) {
3614       return;
3615     }
3616   }
3617 
3618   PhaseChaitin regalloc(unique(), cfg, matcher, false);
3619   _regalloc = &regalloc;
3620   {
3621     TracePhase tp(_t_registerAllocation);
3622     // Perform register allocation.  After Chaitin, use-def chains are
3623     // no longer accurate (at spill code) and so must be ignored.
3624     // Node->LRG->reg mappings are still accurate.
3625     _regalloc->Register_Allocate();
3626 
3627     // Bail out if the allocator builds too many nodes
3628     if (failing()) {
3629       return;
3630     }
3631 
3632     print_method(PHASE_REGISTER_ALLOCATION, 2);
3633   }
3634 
3635   // Prior to register allocation we kept empty basic blocks in case the
3636   // the allocator needed a place to spill.  After register allocation we
3637   // are not adding any new instructions.  If any basic block is empty, we
3638   // can now safely remove it.
3639   {
3640     TracePhase tp(_t_blockOrdering);
3641     cfg.remove_empty_blocks();
3642     if (do_freq_based_layout()) {
3643       PhaseBlockLayout layout(cfg);
3644     } else {
3645       cfg.set_loop_alignment();
3646     }
3647     cfg.fixup_flow();
3648     cfg.remove_unreachable_blocks();
3649     cfg.verify_dominator_tree();
3650     print_method(PHASE_BLOCK_ORDERING, 3);
3651   }
3652 
3653   // Apply peephole optimizations
3654   if( OptoPeephole ) {
3655     TracePhase tp(_t_peephole);
3656     PhasePeephole peep( _regalloc, cfg);
3657     peep.do_transform();
3658     print_method(PHASE_PEEPHOLE, 3);
3659   }
3660 
3661   // Do late expand if CPU requires this.
3662   if (Matcher::require_postalloc_expand) {
3663     TracePhase tp(_t_postalloc_expand);
3664     cfg.postalloc_expand(_regalloc);
3665     print_method(PHASE_POSTALLOC_EXPAND, 3);
3666   }
3667 
3668 #ifdef ASSERT
3669   {
3670     CompilationMemoryStatistic::do_test_allocations();
3671     if (failing()) return;
3672   }
3673 #endif
3674 
3675   // Convert Nodes to instruction bits in a buffer
3676   {
3677     TracePhase tp(_t_output);
3678     PhaseOutput output;
3679     output.Output();
3680     if (failing())  return;
3681     output.install();
3682     print_method(PHASE_FINAL_CODE, 1); // Compile::_output is not null here
3683   }
3684 
3685   // He's dead, Jim.
3686   _cfg     = (PhaseCFG*)((intptr_t)0xdeadbeef);
3687   _regalloc = (PhaseChaitin*)((intptr_t)0xdeadbeef);
3688 }
3689 
3690 //------------------------------Final_Reshape_Counts---------------------------
3691 // This class defines counters to help identify when a method
3692 // may/must be executed using hardware with only 24-bit precision.
3693 struct Final_Reshape_Counts : public StackObj {
3694   int  _call_count;             // count non-inlined 'common' calls
3695   int  _float_count;            // count float ops requiring 24-bit precision
3696   int  _double_count;           // count double ops requiring more precision
3697   int  _java_call_count;        // count non-inlined 'java' calls
3698   int  _inner_loop_count;       // count loops which need alignment
3699   VectorSet _visited;           // Visitation flags
3700   Node_List _tests;             // Set of IfNodes & PCTableNodes
3701 
3702   Final_Reshape_Counts() :
3703     _call_count(0), _float_count(0), _double_count(0),
3704     _java_call_count(0), _inner_loop_count(0) { }
3705 
3706   void inc_call_count  () { _call_count  ++; }
3707   void inc_float_count () { _float_count ++; }
3708   void inc_double_count() { _double_count++; }
3709   void inc_java_call_count() { _java_call_count++; }
3710   void inc_inner_loop_count() { _inner_loop_count++; }
3711 
3712   int  get_call_count  () const { return _call_count  ; }
3713   int  get_float_count () const { return _float_count ; }
3714   int  get_double_count() const { return _double_count; }
3715   int  get_java_call_count() const { return _java_call_count; }
3716   int  get_inner_loop_count() const { return _inner_loop_count; }
3717 };
3718 
3719 //------------------------------final_graph_reshaping_impl----------------------
3720 // Implement items 1-5 from final_graph_reshaping below.
3721 void Compile::final_graph_reshaping_impl(Node *n, Final_Reshape_Counts& frc, Unique_Node_List& dead_nodes) {
3722 
3723   if ( n->outcnt() == 0 ) return; // dead node
3724   uint nop = n->Opcode();
3725 
3726   // Check for 2-input instruction with "last use" on right input.
3727   // Swap to left input.  Implements item (2).
3728   if( n->req() == 3 &&          // two-input instruction
3729       n->in(1)->outcnt() > 1 && // left use is NOT a last use
3730       (!n->in(1)->is_Phi() || n->in(1)->in(2) != n) && // it is not data loop
3731       n->in(2)->outcnt() == 1 &&// right use IS a last use
3732       !n->in(2)->is_Con() ) {   // right use is not a constant
3733     // Check for commutative opcode
3734     switch( nop ) {
3735     case Op_AddI:  case Op_AddF:  case Op_AddD:  case Op_AddL:
3736     case Op_MaxI:  case Op_MaxL:  case Op_MaxF:  case Op_MaxD:
3737     case Op_MinI:  case Op_MinL:  case Op_MinF:  case Op_MinD:
3738     case Op_MulI:  case Op_MulF:  case Op_MulD:  case Op_MulL:
3739     case Op_AndL:  case Op_XorL:  case Op_OrL:
3740     case Op_AndI:  case Op_XorI:  case Op_OrI: {
3741       // Move "last use" input to left by swapping inputs
3742       n->swap_edges(1, 2);
3743       break;
3744     }
3745     default:
3746       break;
3747     }
3748   }
3749 
3750 #ifdef ASSERT
3751   if( n->is_Mem() ) {
3752     int alias_idx = get_alias_index(n->as_Mem()->adr_type());
3753     assert( n->in(0) != nullptr || alias_idx != Compile::AliasIdxRaw ||
3754             // oop will be recorded in oop map if load crosses safepoint
3755             (n->is_Load() && (n->as_Load()->bottom_type()->isa_oopptr() ||
3756                               LoadNode::is_immutable_value(n->in(MemNode::Address)))),
3757             "raw memory operations should have control edge");
3758   }
3759   if (n->is_MemBar()) {
3760     MemBarNode* mb = n->as_MemBar();
3761     if (mb->trailing_store() || mb->trailing_load_store()) {
3762       assert(mb->leading_membar()->trailing_membar() == mb, "bad membar pair");
3763       Node* mem = BarrierSet::barrier_set()->barrier_set_c2()->step_over_gc_barrier(mb->in(MemBarNode::Precedent));
3764       assert((mb->trailing_store() && mem->is_Store() && mem->as_Store()->is_release()) ||
3765              (mb->trailing_load_store() && mem->is_LoadStore()), "missing mem op");
3766     } else if (mb->leading()) {
3767       assert(mb->trailing_membar()->leading_membar() == mb, "bad membar pair");
3768     }
3769   }
3770 #endif
3771   // Count FPU ops and common calls, implements item (3)
3772   bool gc_handled = BarrierSet::barrier_set()->barrier_set_c2()->final_graph_reshaping(this, n, nop, dead_nodes);
3773   if (!gc_handled) {
3774     final_graph_reshaping_main_switch(n, frc, nop, dead_nodes);
3775   }
3776 
3777   // Collect CFG split points
3778   if (n->is_MultiBranch() && !n->is_RangeCheck()) {
3779     frc._tests.push(n);
3780   }
3781 }
3782 
3783 void Compile::handle_div_mod_op(Node* n, BasicType bt, bool is_unsigned) {
3784   if (!UseDivMod) {
3785     return;
3786   }
3787 
3788   // Check if "a % b" and "a / b" both exist
3789   Node* d = n->find_similar(Op_DivIL(bt, is_unsigned));
3790   if (d == nullptr) {
3791     return;
3792   }
3793 
3794   // Replace them with a fused divmod if supported
3795   if (Matcher::has_match_rule(Op_DivModIL(bt, is_unsigned))) {
3796     DivModNode* divmod = DivModNode::make(n, bt, is_unsigned);
3797     // If the divisor input for a Div (or Mod etc.) is not zero, then the control input of the Div is set to zero.
3798     // It could be that the divisor input is found not zero because its type is narrowed down by a CastII in the
3799     // subgraph for that input. Range check CastIIs are removed during final graph reshape. To preserve the dependency
3800     // carried by a CastII, precedence edges are added to the Div node. We need to transfer the precedence edges to the
3801     // DivMod node so the dependency is not lost.
3802     divmod->add_prec_from(n);
3803     divmod->add_prec_from(d);
3804     d->subsume_by(divmod->div_proj(), this);
3805     n->subsume_by(divmod->mod_proj(), this);
3806   } else {
3807     // Replace "a % b" with "a - ((a / b) * b)"
3808     Node* mult = MulNode::make(d, d->in(2), bt);
3809     Node* sub = SubNode::make(d->in(1), mult, bt);
3810     n->subsume_by(sub, this);
3811   }
3812 }
3813 
3814 void Compile::final_graph_reshaping_main_switch(Node* n, Final_Reshape_Counts& frc, uint nop, Unique_Node_List& dead_nodes) {
3815   switch( nop ) {
3816   // Count all float operations that may use FPU
3817   case Op_AddF:
3818   case Op_SubF:
3819   case Op_MulF:
3820   case Op_DivF:
3821   case Op_NegF:
3822   case Op_ModF:
3823   case Op_ConvI2F:
3824   case Op_ConF:
3825   case Op_CmpF:
3826   case Op_CmpF3:
3827   case Op_StoreF:
3828   case Op_LoadF:
3829   // case Op_ConvL2F: // longs are split into 32-bit halves
3830     frc.inc_float_count();
3831     break;
3832 
3833   case Op_ConvF2D:
3834   case Op_ConvD2F:
3835     frc.inc_float_count();
3836     frc.inc_double_count();
3837     break;
3838 
3839   // Count all double operations that may use FPU
3840   case Op_AddD:
3841   case Op_SubD:
3842   case Op_MulD:
3843   case Op_DivD:
3844   case Op_NegD:
3845   case Op_ModD:
3846   case Op_ConvI2D:
3847   case Op_ConvD2I:
3848   // case Op_ConvL2D: // handled by leaf call
3849   // case Op_ConvD2L: // handled by leaf call
3850   case Op_ConD:
3851   case Op_CmpD:
3852   case Op_CmpD3:
3853   case Op_StoreD:
3854   case Op_LoadD:
3855   case Op_LoadD_unaligned:
3856     frc.inc_double_count();
3857     break;
3858   case Op_Opaque1:              // Remove Opaque Nodes before matching
3859     n->subsume_by(n->in(1), this);
3860     break;
3861   case Op_CallLeafPure: {
3862     // If the pure call is not supported, then lower to a CallLeaf.
3863     if (!Matcher::match_rule_supported(Op_CallLeafPure)) {
3864       CallNode* call = n->as_Call();
3865       CallNode* new_call = new CallLeafNode(call->tf(), call->entry_point(),
3866                                             call->_name, TypeRawPtr::BOTTOM);
3867       new_call->init_req(TypeFunc::Control, call->in(TypeFunc::Control));
3868       new_call->init_req(TypeFunc::I_O, C->top());
3869       new_call->init_req(TypeFunc::Memory, C->top());
3870       new_call->init_req(TypeFunc::ReturnAdr, C->top());
3871       new_call->init_req(TypeFunc::FramePtr, C->top());
3872       for (unsigned int i = TypeFunc::Parms; i < call->tf()->domain_sig()->cnt(); i++) {
3873         new_call->init_req(i, call->in(i));
3874       }
3875       n->subsume_by(new_call, this);
3876     }
3877     frc.inc_call_count();
3878     break;
3879   }
3880   case Op_CallStaticJava:
3881   case Op_CallJava:
3882   case Op_CallDynamicJava:
3883     frc.inc_java_call_count(); // Count java call site;
3884   case Op_CallRuntime:
3885   case Op_CallLeaf:
3886   case Op_CallLeafVector:
3887   case Op_CallLeafNoFP: {
3888     assert (n->is_Call(), "");
3889     CallNode *call = n->as_Call();
3890     // Count call sites where the FP mode bit would have to be flipped.
3891     // Do not count uncommon runtime calls:
3892     // uncommon_trap, _complete_monitor_locking, _complete_monitor_unlocking,
3893     // _new_Java, _new_typeArray, _new_objArray, _rethrow_Java, ...
3894     if (!call->is_CallStaticJava() || !call->as_CallStaticJava()->_name) {
3895       frc.inc_call_count();   // Count the call site
3896     } else {                  // See if uncommon argument is shared
3897       Node *n = call->in(TypeFunc::Parms);
3898       int nop = n->Opcode();
3899       // Clone shared simple arguments to uncommon calls, item (1).
3900       if (n->outcnt() > 1 &&
3901           !n->is_Proj() &&
3902           nop != Op_CreateEx &&
3903           nop != Op_CheckCastPP &&
3904           nop != Op_DecodeN &&
3905           nop != Op_DecodeNKlass &&
3906           !n->is_Mem() &&
3907           !n->is_Phi()) {
3908         Node *x = n->clone();
3909         call->set_req(TypeFunc::Parms, x);
3910       }
3911     }
3912     break;
3913   }
3914   case Op_StoreB:
3915   case Op_StoreC:
3916   case Op_StoreI:
3917   case Op_StoreL:
3918   case Op_StoreLSpecial:
3919   case Op_CompareAndSwapB:
3920   case Op_CompareAndSwapS:
3921   case Op_CompareAndSwapI:
3922   case Op_CompareAndSwapL:
3923   case Op_CompareAndSwapP:
3924   case Op_CompareAndSwapN:
3925   case Op_WeakCompareAndSwapB:
3926   case Op_WeakCompareAndSwapS:
3927   case Op_WeakCompareAndSwapI:
3928   case Op_WeakCompareAndSwapL:
3929   case Op_WeakCompareAndSwapP:
3930   case Op_WeakCompareAndSwapN:
3931   case Op_CompareAndExchangeB:
3932   case Op_CompareAndExchangeS:
3933   case Op_CompareAndExchangeI:
3934   case Op_CompareAndExchangeL:
3935   case Op_CompareAndExchangeP:
3936   case Op_CompareAndExchangeN:
3937   case Op_GetAndAddS:
3938   case Op_GetAndAddB:
3939   case Op_GetAndAddI:
3940   case Op_GetAndAddL:
3941   case Op_GetAndSetS:
3942   case Op_GetAndSetB:
3943   case Op_GetAndSetI:
3944   case Op_GetAndSetL:
3945   case Op_GetAndSetP:
3946   case Op_GetAndSetN:
3947   case Op_StoreP:
3948   case Op_StoreN:
3949   case Op_StoreNKlass:
3950   case Op_LoadB:
3951   case Op_LoadUB:
3952   case Op_LoadUS:
3953   case Op_LoadI:
3954   case Op_LoadKlass:
3955   case Op_LoadNKlass:
3956   case Op_LoadL:
3957   case Op_LoadL_unaligned:
3958   case Op_LoadP:
3959   case Op_LoadN:
3960   case Op_LoadRange:
3961   case Op_LoadS:
3962     break;
3963 
3964   case Op_AddP: {               // Assert sane base pointers
3965     Node *addp = n->in(AddPNode::Address);
3966     assert( !addp->is_AddP() ||
3967             addp->in(AddPNode::Base)->is_top() || // Top OK for allocation
3968             addp->in(AddPNode::Base) == n->in(AddPNode::Base),
3969             "Base pointers must match (addp %u)", addp->_idx );
3970 #ifdef _LP64
3971     if ((UseCompressedOops || UseCompressedClassPointers) &&
3972         addp->Opcode() == Op_ConP &&
3973         addp == n->in(AddPNode::Base) &&
3974         n->in(AddPNode::Offset)->is_Con()) {
3975       // If the transformation of ConP to ConN+DecodeN is beneficial depends
3976       // on the platform and on the compressed oops mode.
3977       // Use addressing with narrow klass to load with offset on x86.
3978       // Some platforms can use the constant pool to load ConP.
3979       // Do this transformation here since IGVN will convert ConN back to ConP.
3980       const Type* t = addp->bottom_type();
3981       bool is_oop   = t->isa_oopptr() != nullptr;
3982       bool is_klass = t->isa_klassptr() != nullptr;
3983 
3984       if ((is_oop   && UseCompressedOops          && Matcher::const_oop_prefer_decode()  ) ||
3985           (is_klass && UseCompressedClassPointers && Matcher::const_klass_prefer_decode() &&
3986            t->isa_klassptr()->exact_klass()->is_in_encoding_range())) {
3987         Node* nn = nullptr;
3988 
3989         int op = is_oop ? Op_ConN : Op_ConNKlass;
3990 
3991         // Look for existing ConN node of the same exact type.
3992         Node* r  = root();
3993         uint cnt = r->outcnt();
3994         for (uint i = 0; i < cnt; i++) {
3995           Node* m = r->raw_out(i);
3996           if (m!= nullptr && m->Opcode() == op &&
3997               m->bottom_type()->make_ptr() == t) {
3998             nn = m;
3999             break;
4000           }
4001         }
4002         if (nn != nullptr) {
4003           // Decode a narrow oop to match address
4004           // [R12 + narrow_oop_reg<<3 + offset]
4005           if (is_oop) {
4006             nn = new DecodeNNode(nn, t);
4007           } else {
4008             nn = new DecodeNKlassNode(nn, t);
4009           }
4010           // Check for succeeding AddP which uses the same Base.
4011           // Otherwise we will run into the assertion above when visiting that guy.
4012           for (uint i = 0; i < n->outcnt(); ++i) {
4013             Node *out_i = n->raw_out(i);
4014             if (out_i && out_i->is_AddP() && out_i->in(AddPNode::Base) == addp) {
4015               out_i->set_req(AddPNode::Base, nn);
4016 #ifdef ASSERT
4017               for (uint j = 0; j < out_i->outcnt(); ++j) {
4018                 Node *out_j = out_i->raw_out(j);
4019                 assert(out_j == nullptr || !out_j->is_AddP() || out_j->in(AddPNode::Base) != addp,
4020                        "more than 2 AddP nodes in a chain (out_j %u)", out_j->_idx);
4021               }
4022 #endif
4023             }
4024           }
4025           n->set_req(AddPNode::Base, nn);
4026           n->set_req(AddPNode::Address, nn);
4027           if (addp->outcnt() == 0) {
4028             addp->disconnect_inputs(this);
4029           }
4030         }
4031       }
4032     }
4033 #endif
4034     break;
4035   }
4036 
4037   case Op_CastPP: {
4038     // Remove CastPP nodes to gain more freedom during scheduling but
4039     // keep the dependency they encode as control or precedence edges
4040     // (if control is set already) on memory operations. Some CastPP
4041     // nodes don't have a control (don't carry a dependency): skip
4042     // those.
4043     if (n->in(0) != nullptr) {
4044       ResourceMark rm;
4045       Unique_Node_List wq;
4046       wq.push(n);
4047       for (uint next = 0; next < wq.size(); ++next) {
4048         Node *m = wq.at(next);
4049         for (DUIterator_Fast imax, i = m->fast_outs(imax); i < imax; i++) {
4050           Node* use = m->fast_out(i);
4051           if (use->is_Mem() || use->is_EncodeNarrowPtr()) {
4052             use->ensure_control_or_add_prec(n->in(0));
4053           } else {
4054             switch(use->Opcode()) {
4055             case Op_AddP:
4056             case Op_DecodeN:
4057             case Op_DecodeNKlass:
4058             case Op_CheckCastPP:
4059             case Op_CastPP:
4060               wq.push(use);
4061               break;
4062             }
4063           }
4064         }
4065       }
4066     }
4067     const bool is_LP64 = LP64_ONLY(true) NOT_LP64(false);
4068     if (is_LP64 && n->in(1)->is_DecodeN() && Matcher::gen_narrow_oop_implicit_null_checks()) {
4069       Node* in1 = n->in(1);
4070       const Type* t = n->bottom_type();
4071       Node* new_in1 = in1->clone();
4072       new_in1->as_DecodeN()->set_type(t);
4073 
4074       if (!Matcher::narrow_oop_use_complex_address()) {
4075         //
4076         // x86, ARM and friends can handle 2 adds in addressing mode
4077         // and Matcher can fold a DecodeN node into address by using
4078         // a narrow oop directly and do implicit null check in address:
4079         //
4080         // [R12 + narrow_oop_reg<<3 + offset]
4081         // NullCheck narrow_oop_reg
4082         //
4083         // On other platforms (Sparc) we have to keep new DecodeN node and
4084         // use it to do implicit null check in address:
4085         //
4086         // decode_not_null narrow_oop_reg, base_reg
4087         // [base_reg + offset]
4088         // NullCheck base_reg
4089         //
4090         // Pin the new DecodeN node to non-null path on these platform (Sparc)
4091         // to keep the information to which null check the new DecodeN node
4092         // corresponds to use it as value in implicit_null_check().
4093         //
4094         new_in1->set_req(0, n->in(0));
4095       }
4096 
4097       n->subsume_by(new_in1, this);
4098       if (in1->outcnt() == 0) {
4099         in1->disconnect_inputs(this);
4100       }
4101     } else {
4102       n->subsume_by(n->in(1), this);
4103       if (n->outcnt() == 0) {
4104         n->disconnect_inputs(this);
4105       }
4106     }
4107     break;
4108   }
4109   case Op_CastII: {
4110     n->as_CastII()->remove_range_check_cast(this);
4111     break;
4112   }
4113 #ifdef _LP64
4114   case Op_CmpP:
4115     // Do this transformation here to preserve CmpPNode::sub() and
4116     // other TypePtr related Ideal optimizations (for example, ptr nullness).
4117     if (n->in(1)->is_DecodeNarrowPtr() || n->in(2)->is_DecodeNarrowPtr()) {
4118       Node* in1 = n->in(1);
4119       Node* in2 = n->in(2);
4120       if (!in1->is_DecodeNarrowPtr()) {
4121         in2 = in1;
4122         in1 = n->in(2);
4123       }
4124       assert(in1->is_DecodeNarrowPtr(), "sanity");
4125 
4126       Node* new_in2 = nullptr;
4127       if (in2->is_DecodeNarrowPtr()) {
4128         assert(in2->Opcode() == in1->Opcode(), "must be same node type");
4129         new_in2 = in2->in(1);
4130       } else if (in2->Opcode() == Op_ConP) {
4131         const Type* t = in2->bottom_type();
4132         if (t == TypePtr::NULL_PTR) {
4133           assert(in1->is_DecodeN(), "compare klass to null?");
4134           // Don't convert CmpP null check into CmpN if compressed
4135           // oops implicit null check is not generated.
4136           // This will allow to generate normal oop implicit null check.
4137           if (Matcher::gen_narrow_oop_implicit_null_checks())
4138             new_in2 = ConNode::make(TypeNarrowOop::NULL_PTR);
4139           //
4140           // This transformation together with CastPP transformation above
4141           // will generated code for implicit null checks for compressed oops.
4142           //
4143           // The original code after Optimize()
4144           //
4145           //    LoadN memory, narrow_oop_reg
4146           //    decode narrow_oop_reg, base_reg
4147           //    CmpP base_reg, nullptr
4148           //    CastPP base_reg // NotNull
4149           //    Load [base_reg + offset], val_reg
4150           //
4151           // after these transformations will be
4152           //
4153           //    LoadN memory, narrow_oop_reg
4154           //    CmpN narrow_oop_reg, nullptr
4155           //    decode_not_null narrow_oop_reg, base_reg
4156           //    Load [base_reg + offset], val_reg
4157           //
4158           // and the uncommon path (== nullptr) will use narrow_oop_reg directly
4159           // since narrow oops can be used in debug info now (see the code in
4160           // final_graph_reshaping_walk()).
4161           //
4162           // At the end the code will be matched to
4163           // on x86:
4164           //
4165           //    Load_narrow_oop memory, narrow_oop_reg
4166           //    Load [R12 + narrow_oop_reg<<3 + offset], val_reg
4167           //    NullCheck narrow_oop_reg
4168           //
4169           // and on sparc:
4170           //
4171           //    Load_narrow_oop memory, narrow_oop_reg
4172           //    decode_not_null narrow_oop_reg, base_reg
4173           //    Load [base_reg + offset], val_reg
4174           //    NullCheck base_reg
4175           //
4176         } else if (t->isa_oopptr()) {
4177           new_in2 = ConNode::make(t->make_narrowoop());
4178         } else if (t->isa_klassptr()) {
4179           ciKlass* klass = t->is_klassptr()->exact_klass();
4180           if (klass->is_in_encoding_range()) {
4181             new_in2 = ConNode::make(t->make_narrowklass());
4182           }
4183         }
4184       }
4185       if (new_in2 != nullptr) {
4186         Node* cmpN = new CmpNNode(in1->in(1), new_in2);
4187         n->subsume_by(cmpN, this);
4188         if (in1->outcnt() == 0) {
4189           in1->disconnect_inputs(this);
4190         }
4191         if (in2->outcnt() == 0) {
4192           in2->disconnect_inputs(this);
4193         }
4194       }
4195     }
4196     break;
4197 
4198   case Op_DecodeN:
4199   case Op_DecodeNKlass:
4200     assert(!n->in(1)->is_EncodeNarrowPtr(), "should be optimized out");
4201     // DecodeN could be pinned when it can't be fold into
4202     // an address expression, see the code for Op_CastPP above.
4203     assert(n->in(0) == nullptr || (UseCompressedOops && !Matcher::narrow_oop_use_complex_address()), "no control");
4204     break;
4205 
4206   case Op_EncodeP:
4207   case Op_EncodePKlass: {
4208     Node* in1 = n->in(1);
4209     if (in1->is_DecodeNarrowPtr()) {
4210       n->subsume_by(in1->in(1), this);
4211     } else if (in1->Opcode() == Op_ConP) {
4212       const Type* t = in1->bottom_type();
4213       if (t == TypePtr::NULL_PTR) {
4214         assert(t->isa_oopptr(), "null klass?");
4215         n->subsume_by(ConNode::make(TypeNarrowOop::NULL_PTR), this);
4216       } else if (t->isa_oopptr()) {
4217         n->subsume_by(ConNode::make(t->make_narrowoop()), this);
4218       } else if (t->isa_klassptr()) {
4219         ciKlass* klass = t->is_klassptr()->exact_klass();
4220         if (klass->is_in_encoding_range()) {
4221           n->subsume_by(ConNode::make(t->make_narrowklass()), this);
4222         } else {
4223           assert(false, "unencodable klass in ConP -> EncodeP");
4224           C->record_failure("unencodable klass in ConP -> EncodeP");
4225         }
4226       }
4227     }
4228     if (in1->outcnt() == 0) {
4229       in1->disconnect_inputs(this);
4230     }
4231     break;
4232   }
4233 
4234   case Op_Proj: {
4235     if (OptimizeStringConcat || IncrementalInline) {
4236       ProjNode* proj = n->as_Proj();
4237       if (proj->_is_io_use) {
4238         assert(proj->_con == TypeFunc::I_O || proj->_con == TypeFunc::Memory, "");
4239         // Separate projections were used for the exception path which
4240         // are normally removed by a late inline.  If it wasn't inlined
4241         // then they will hang around and should just be replaced with
4242         // the original one. Merge them.
4243         Node* non_io_proj = proj->in(0)->as_Multi()->proj_out_or_null(proj->_con, false /*is_io_use*/);
4244         if (non_io_proj  != nullptr) {
4245           proj->subsume_by(non_io_proj , this);
4246         }
4247       }
4248     }
4249     break;
4250   }
4251 
4252   case Op_Phi:
4253     if (n->as_Phi()->bottom_type()->isa_narrowoop() || n->as_Phi()->bottom_type()->isa_narrowklass()) {
4254       // The EncodeP optimization may create Phi with the same edges
4255       // for all paths. It is not handled well by Register Allocator.
4256       Node* unique_in = n->in(1);
4257       assert(unique_in != nullptr, "");
4258       uint cnt = n->req();
4259       for (uint i = 2; i < cnt; i++) {
4260         Node* m = n->in(i);
4261         assert(m != nullptr, "");
4262         if (unique_in != m)
4263           unique_in = nullptr;
4264       }
4265       if (unique_in != nullptr) {
4266         n->subsume_by(unique_in, this);
4267       }
4268     }
4269     break;
4270 
4271 #endif
4272 
4273   case Op_ModI:
4274     handle_div_mod_op(n, T_INT, false);
4275     break;
4276 
4277   case Op_ModL:
4278     handle_div_mod_op(n, T_LONG, false);
4279     break;
4280 
4281   case Op_UModI:
4282     handle_div_mod_op(n, T_INT, true);
4283     break;
4284 
4285   case Op_UModL:
4286     handle_div_mod_op(n, T_LONG, true);
4287     break;
4288 
4289   case Op_LoadVector:
4290   case Op_StoreVector:
4291 #ifdef ASSERT
4292     // Add VerifyVectorAlignment node between adr and load / store.
4293     if (VerifyAlignVector && Matcher::has_match_rule(Op_VerifyVectorAlignment)) {
4294       bool must_verify_alignment = n->is_LoadVector() ? n->as_LoadVector()->must_verify_alignment() :
4295                                                         n->as_StoreVector()->must_verify_alignment();
4296       if (must_verify_alignment) {
4297         jlong vector_width = n->is_LoadVector() ? n->as_LoadVector()->memory_size() :
4298                                                   n->as_StoreVector()->memory_size();
4299         // The memory access should be aligned to the vector width in bytes.
4300         // However, the underlying array is possibly less well aligned, but at least
4301         // to ObjectAlignmentInBytes. Hence, even if multiple arrays are accessed in
4302         // a loop we can expect at least the following alignment:
4303         jlong guaranteed_alignment = MIN2(vector_width, (jlong)ObjectAlignmentInBytes);
4304         assert(2 <= guaranteed_alignment && guaranteed_alignment <= 64, "alignment must be in range");
4305         assert(is_power_of_2(guaranteed_alignment), "alignment must be power of 2");
4306         // Create mask from alignment. e.g. 0b1000 -> 0b0111
4307         jlong mask = guaranteed_alignment - 1;
4308         Node* mask_con = ConLNode::make(mask);
4309         VerifyVectorAlignmentNode* va = new VerifyVectorAlignmentNode(n->in(MemNode::Address), mask_con);
4310         n->set_req(MemNode::Address, va);
4311       }
4312     }
4313 #endif
4314     break;
4315 
4316   case Op_LoadVectorGather:
4317   case Op_StoreVectorScatter:
4318   case Op_LoadVectorGatherMasked:
4319   case Op_StoreVectorScatterMasked:
4320   case Op_VectorCmpMasked:
4321   case Op_VectorMaskGen:
4322   case Op_LoadVectorMasked:
4323   case Op_StoreVectorMasked:
4324     break;
4325 
4326   case Op_AddReductionVI:
4327   case Op_AddReductionVL:
4328   case Op_AddReductionVF:
4329   case Op_AddReductionVD:
4330   case Op_MulReductionVI:
4331   case Op_MulReductionVL:
4332   case Op_MulReductionVF:
4333   case Op_MulReductionVD:
4334   case Op_MinReductionV:
4335   case Op_MaxReductionV:
4336   case Op_AndReductionV:
4337   case Op_OrReductionV:
4338   case Op_XorReductionV:
4339     break;
4340 
4341   case Op_PackB:
4342   case Op_PackS:
4343   case Op_PackI:
4344   case Op_PackF:
4345   case Op_PackL:
4346   case Op_PackD:
4347     if (n->req()-1 > 2) {
4348       // Replace many operand PackNodes with a binary tree for matching
4349       PackNode* p = (PackNode*) n;
4350       Node* btp = p->binary_tree_pack(1, n->req());
4351       n->subsume_by(btp, this);
4352     }
4353     break;
4354   case Op_Loop:
4355     assert(!n->as_Loop()->is_loop_nest_inner_loop() || _loop_opts_cnt == 0, "should have been turned into a counted loop");
4356   case Op_CountedLoop:
4357   case Op_LongCountedLoop:
4358   case Op_OuterStripMinedLoop:
4359     if (n->as_Loop()->is_inner_loop()) {
4360       frc.inc_inner_loop_count();
4361     }
4362     n->as_Loop()->verify_strip_mined(0);
4363     break;
4364   case Op_LShiftI:
4365   case Op_RShiftI:
4366   case Op_URShiftI:
4367   case Op_LShiftL:
4368   case Op_RShiftL:
4369   case Op_URShiftL:
4370     if (Matcher::need_masked_shift_count) {
4371       // The cpu's shift instructions don't restrict the count to the
4372       // lower 5/6 bits. We need to do the masking ourselves.
4373       Node* in2 = n->in(2);
4374       juint mask = (n->bottom_type() == TypeInt::INT) ? (BitsPerInt - 1) : (BitsPerLong - 1);
4375       const TypeInt* t = in2->find_int_type();
4376       if (t != nullptr && t->is_con()) {
4377         juint shift = t->get_con();
4378         if (shift > mask) { // Unsigned cmp
4379           n->set_req(2, ConNode::make(TypeInt::make(shift & mask)));
4380         }
4381       } else {
4382         if (t == nullptr || t->_lo < 0 || t->_hi > (int)mask) {
4383           Node* shift = new AndINode(in2, ConNode::make(TypeInt::make(mask)));
4384           n->set_req(2, shift);
4385         }
4386       }
4387       if (in2->outcnt() == 0) { // Remove dead node
4388         in2->disconnect_inputs(this);
4389       }
4390     }
4391     break;
4392   case Op_MemBarStoreStore:
4393   case Op_MemBarRelease:
4394     // Break the link with AllocateNode: it is no longer useful and
4395     // confuses register allocation.
4396     if (n->req() > MemBarNode::Precedent) {
4397       n->set_req(MemBarNode::Precedent, top());
4398     }
4399     break;
4400   case Op_MemBarAcquire: {
4401     if (n->as_MemBar()->trailing_load() && n->req() > MemBarNode::Precedent) {
4402       // At parse time, the trailing MemBarAcquire for a volatile load
4403       // is created with an edge to the load. After optimizations,
4404       // that input may be a chain of Phis. If those phis have no
4405       // other use, then the MemBarAcquire keeps them alive and
4406       // register allocation can be confused.
4407       dead_nodes.push(n->in(MemBarNode::Precedent));
4408       n->set_req(MemBarNode::Precedent, top());
4409     }
4410     break;
4411   }
4412   case Op_Blackhole:
4413     break;
4414   case Op_RangeCheck: {
4415     RangeCheckNode* rc = n->as_RangeCheck();
4416     Node* iff = new IfNode(rc->in(0), rc->in(1), rc->_prob, rc->_fcnt);
4417     n->subsume_by(iff, this);
4418     frc._tests.push(iff);
4419     break;
4420   }
4421   case Op_ConvI2L: {
4422     if (!Matcher::convi2l_type_required) {
4423       // Code generation on some platforms doesn't need accurate
4424       // ConvI2L types. Widening the type can help remove redundant
4425       // address computations.
4426       n->as_Type()->set_type(TypeLong::INT);
4427       ResourceMark rm;
4428       Unique_Node_List wq;
4429       wq.push(n);
4430       for (uint next = 0; next < wq.size(); next++) {
4431         Node *m = wq.at(next);
4432 
4433         for(;;) {
4434           // Loop over all nodes with identical inputs edges as m
4435           Node* k = m->find_similar(m->Opcode());
4436           if (k == nullptr) {
4437             break;
4438           }
4439           // Push their uses so we get a chance to remove node made
4440           // redundant
4441           for (DUIterator_Fast imax, i = k->fast_outs(imax); i < imax; i++) {
4442             Node* u = k->fast_out(i);
4443             if (u->Opcode() == Op_LShiftL ||
4444                 u->Opcode() == Op_AddL ||
4445                 u->Opcode() == Op_SubL ||
4446                 u->Opcode() == Op_AddP) {
4447               wq.push(u);
4448             }
4449           }
4450           // Replace all nodes with identical edges as m with m
4451           k->subsume_by(m, this);
4452         }
4453       }
4454     }
4455     break;
4456   }
4457   case Op_CmpUL: {
4458     if (!Matcher::has_match_rule(Op_CmpUL)) {
4459       // No support for unsigned long comparisons
4460       ConINode* sign_pos = new ConINode(TypeInt::make(BitsPerLong - 1));
4461       Node* sign_bit_mask = new RShiftLNode(n->in(1), sign_pos);
4462       Node* orl = new OrLNode(n->in(1), sign_bit_mask);
4463       ConLNode* remove_sign_mask = new ConLNode(TypeLong::make(max_jlong));
4464       Node* andl = new AndLNode(orl, remove_sign_mask);
4465       Node* cmp = new CmpLNode(andl, n->in(2));
4466       n->subsume_by(cmp, this);
4467     }
4468     break;
4469   }
4470 #ifdef ASSERT
4471   case Op_InlineType: {
4472     n->dump(-1);
4473     assert(false, "inline type node was not removed");
4474     break;
4475   }
4476   case Op_ConNKlass: {
4477     const TypePtr* tp = n->as_Type()->type()->make_ptr();
4478     ciKlass* klass = tp->is_klassptr()->exact_klass();
4479     assert(klass->is_in_encoding_range(), "klass cannot be compressed");
4480     break;
4481   }
4482 #endif
4483   default:
4484     assert(!n->is_Call(), "");
4485     assert(!n->is_Mem(), "");
4486     assert(nop != Op_ProfileBoolean, "should be eliminated during IGVN");
4487     break;
4488   }
4489 }
4490 
4491 //------------------------------final_graph_reshaping_walk---------------------
4492 // Replacing Opaque nodes with their input in final_graph_reshaping_impl(),
4493 // requires that the walk visits a node's inputs before visiting the node.
4494 void Compile::final_graph_reshaping_walk(Node_Stack& nstack, Node* root, Final_Reshape_Counts& frc, Unique_Node_List& dead_nodes) {
4495   Unique_Node_List sfpt;
4496 
4497   frc._visited.set(root->_idx); // first, mark node as visited
4498   uint cnt = root->req();
4499   Node *n = root;
4500   uint  i = 0;
4501   while (true) {
4502     if (i < cnt) {
4503       // Place all non-visited non-null inputs onto stack
4504       Node* m = n->in(i);
4505       ++i;
4506       if (m != nullptr && !frc._visited.test_set(m->_idx)) {
4507         if (m->is_SafePoint() && m->as_SafePoint()->jvms() != nullptr) {
4508           // compute worst case interpreter size in case of a deoptimization
4509           update_interpreter_frame_size(m->as_SafePoint()->jvms()->interpreter_frame_size());
4510 
4511           sfpt.push(m);
4512         }
4513         cnt = m->req();
4514         nstack.push(n, i); // put on stack parent and next input's index
4515         n = m;
4516         i = 0;
4517       }
4518     } else {
4519       // Now do post-visit work
4520       final_graph_reshaping_impl(n, frc, dead_nodes);
4521       if (nstack.is_empty())
4522         break;             // finished
4523       n = nstack.node();   // Get node from stack
4524       cnt = n->req();
4525       i = nstack.index();
4526       nstack.pop();        // Shift to the next node on stack
4527     }
4528   }
4529 
4530   // Skip next transformation if compressed oops are not used.
4531   if ((UseCompressedOops && !Matcher::gen_narrow_oop_implicit_null_checks()) ||
4532       (!UseCompressedOops && !UseCompressedClassPointers))
4533     return;
4534 
4535   // Go over safepoints nodes to skip DecodeN/DecodeNKlass nodes for debug edges.
4536   // It could be done for an uncommon traps or any safepoints/calls
4537   // if the DecodeN/DecodeNKlass node is referenced only in a debug info.
4538   while (sfpt.size() > 0) {
4539     n = sfpt.pop();
4540     JVMState *jvms = n->as_SafePoint()->jvms();
4541     assert(jvms != nullptr, "sanity");
4542     int start = jvms->debug_start();
4543     int end   = n->req();
4544     bool is_uncommon = (n->is_CallStaticJava() &&
4545                         n->as_CallStaticJava()->uncommon_trap_request() != 0);
4546     for (int j = start; j < end; j++) {
4547       Node* in = n->in(j);
4548       if (in->is_DecodeNarrowPtr()) {
4549         bool safe_to_skip = true;
4550         if (!is_uncommon ) {
4551           // Is it safe to skip?
4552           for (uint i = 0; i < in->outcnt(); i++) {
4553             Node* u = in->raw_out(i);
4554             if (!u->is_SafePoint() ||
4555                 (u->is_Call() && u->as_Call()->has_non_debug_use(n))) {
4556               safe_to_skip = false;
4557             }
4558           }
4559         }
4560         if (safe_to_skip) {
4561           n->set_req(j, in->in(1));
4562         }
4563         if (in->outcnt() == 0) {
4564           in->disconnect_inputs(this);
4565         }
4566       }
4567     }
4568   }
4569 }
4570 
4571 //------------------------------final_graph_reshaping--------------------------
4572 // Final Graph Reshaping.
4573 //
4574 // (1) Clone simple inputs to uncommon calls, so they can be scheduled late
4575 //     and not commoned up and forced early.  Must come after regular
4576 //     optimizations to avoid GVN undoing the cloning.  Clone constant
4577 //     inputs to Loop Phis; these will be split by the allocator anyways.
4578 //     Remove Opaque nodes.
4579 // (2) Move last-uses by commutative operations to the left input to encourage
4580 //     Intel update-in-place two-address operations and better register usage
4581 //     on RISCs.  Must come after regular optimizations to avoid GVN Ideal
4582 //     calls canonicalizing them back.
4583 // (3) Count the number of double-precision FP ops, single-precision FP ops
4584 //     and call sites.  On Intel, we can get correct rounding either by
4585 //     forcing singles to memory (requires extra stores and loads after each
4586 //     FP bytecode) or we can set a rounding mode bit (requires setting and
4587 //     clearing the mode bit around call sites).  The mode bit is only used
4588 //     if the relative frequency of single FP ops to calls is low enough.
4589 //     This is a key transform for SPEC mpeg_audio.
4590 // (4) Detect infinite loops; blobs of code reachable from above but not
4591 //     below.  Several of the Code_Gen algorithms fail on such code shapes,
4592 //     so we simply bail out.  Happens a lot in ZKM.jar, but also happens
4593 //     from time to time in other codes (such as -Xcomp finalizer loops, etc).
4594 //     Detection is by looking for IfNodes where only 1 projection is
4595 //     reachable from below or CatchNodes missing some targets.
4596 // (5) Assert for insane oop offsets in debug mode.
4597 
4598 bool Compile::final_graph_reshaping() {
4599   // an infinite loop may have been eliminated by the optimizer,
4600   // in which case the graph will be empty.
4601   if (root()->req() == 1) {
4602     // Do not compile method that is only a trivial infinite loop,
4603     // since the content of the loop may have been eliminated.
4604     record_method_not_compilable("trivial infinite loop");
4605     return true;
4606   }
4607 
4608   // Expensive nodes have their control input set to prevent the GVN
4609   // from freely commoning them. There's no GVN beyond this point so
4610   // no need to keep the control input. We want the expensive nodes to
4611   // be freely moved to the least frequent code path by gcm.
4612   assert(OptimizeExpensiveOps || expensive_count() == 0, "optimization off but list non empty?");
4613   for (int i = 0; i < expensive_count(); i++) {
4614     _expensive_nodes.at(i)->set_req(0, nullptr);
4615   }
4616 
4617   Final_Reshape_Counts frc;
4618 
4619   // Visit everybody reachable!
4620   // Allocate stack of size C->live_nodes()/2 to avoid frequent realloc
4621   Node_Stack nstack(live_nodes() >> 1);
4622   Unique_Node_List dead_nodes;
4623   final_graph_reshaping_walk(nstack, root(), frc, dead_nodes);
4624 
4625   // Check for unreachable (from below) code (i.e., infinite loops).
4626   for( uint i = 0; i < frc._tests.size(); i++ ) {
4627     MultiBranchNode *n = frc._tests[i]->as_MultiBranch();
4628     // Get number of CFG targets.
4629     // Note that PCTables include exception targets after calls.
4630     uint required_outcnt = n->required_outcnt();
4631     if (n->outcnt() != required_outcnt) {
4632       // Check for a few special cases.  Rethrow Nodes never take the
4633       // 'fall-thru' path, so expected kids is 1 less.
4634       if (n->is_PCTable() && n->in(0) && n->in(0)->in(0)) {
4635         if (n->in(0)->in(0)->is_Call()) {
4636           CallNode* call = n->in(0)->in(0)->as_Call();
4637           if (call->entry_point() == OptoRuntime::rethrow_stub()) {
4638             required_outcnt--;      // Rethrow always has 1 less kid
4639           } else if (call->req() > TypeFunc::Parms &&
4640                      call->is_CallDynamicJava()) {
4641             // Check for null receiver. In such case, the optimizer has
4642             // detected that the virtual call will always result in a null
4643             // pointer exception. The fall-through projection of this CatchNode
4644             // will not be populated.
4645             Node* arg0 = call->in(TypeFunc::Parms);
4646             if (arg0->is_Type() &&
4647                 arg0->as_Type()->type()->higher_equal(TypePtr::NULL_PTR)) {
4648               required_outcnt--;
4649             }
4650           } else if (call->entry_point() == OptoRuntime::new_array_Java() ||
4651                      call->entry_point() == OptoRuntime::new_array_nozero_Java()) {
4652             // Check for illegal array length. In such case, the optimizer has
4653             // detected that the allocation attempt will always result in an
4654             // exception. There is no fall-through projection of this CatchNode .
4655             assert(call->is_CallStaticJava(), "static call expected");
4656             assert(call->req() == call->jvms()->endoff() + 1, "missing extra input");
4657             uint valid_length_test_input = call->req() - 1;
4658             Node* valid_length_test = call->in(valid_length_test_input);
4659             call->del_req(valid_length_test_input);
4660             if (valid_length_test->find_int_con(1) == 0) {
4661               required_outcnt--;
4662             }
4663             dead_nodes.push(valid_length_test);
4664             assert(n->outcnt() == required_outcnt, "malformed control flow");
4665             continue;
4666           }
4667         }
4668       }
4669 
4670       // Recheck with a better notion of 'required_outcnt'
4671       if (n->outcnt() != required_outcnt) {
4672         record_method_not_compilable("malformed control flow");
4673         return true;            // Not all targets reachable!
4674       }
4675     } else if (n->is_PCTable() && n->in(0) && n->in(0)->in(0) && n->in(0)->in(0)->is_Call()) {
4676       CallNode* call = n->in(0)->in(0)->as_Call();
4677       if (call->entry_point() == OptoRuntime::new_array_Java() ||
4678           call->entry_point() == OptoRuntime::new_array_nozero_Java()) {
4679         assert(call->is_CallStaticJava(), "static call expected");
4680         assert(call->req() == call->jvms()->endoff() + 1, "missing extra input");
4681         uint valid_length_test_input = call->req() - 1;
4682         dead_nodes.push(call->in(valid_length_test_input));
4683         call->del_req(valid_length_test_input); // valid length test useless now
4684       }
4685     }
4686     // Check that I actually visited all kids.  Unreached kids
4687     // must be infinite loops.
4688     for (DUIterator_Fast jmax, j = n->fast_outs(jmax); j < jmax; j++)
4689       if (!frc._visited.test(n->fast_out(j)->_idx)) {
4690         record_method_not_compilable("infinite loop");
4691         return true;            // Found unvisited kid; must be unreach
4692       }
4693 
4694     // Here so verification code in final_graph_reshaping_walk()
4695     // always see an OuterStripMinedLoopEnd
4696     if (n->is_OuterStripMinedLoopEnd() || n->is_LongCountedLoopEnd()) {
4697       IfNode* init_iff = n->as_If();
4698       Node* iff = new IfNode(init_iff->in(0), init_iff->in(1), init_iff->_prob, init_iff->_fcnt);
4699       n->subsume_by(iff, this);
4700     }
4701   }
4702 
4703   while (dead_nodes.size() > 0) {
4704     Node* m = dead_nodes.pop();
4705     if (m->outcnt() == 0 && m != top()) {
4706       for (uint j = 0; j < m->req(); j++) {
4707         Node* in = m->in(j);
4708         if (in != nullptr) {
4709           dead_nodes.push(in);
4710         }
4711       }
4712       m->disconnect_inputs(this);
4713     }
4714   }
4715 
4716   set_java_calls(frc.get_java_call_count());
4717   set_inner_loops(frc.get_inner_loop_count());
4718 
4719   // No infinite loops, no reason to bail out.
4720   return false;
4721 }
4722 
4723 //-----------------------------too_many_traps----------------------------------
4724 // Report if there are too many traps at the current method and bci.
4725 // Return true if there was a trap, and/or PerMethodTrapLimit is exceeded.
4726 bool Compile::too_many_traps(ciMethod* method,
4727                              int bci,
4728                              Deoptimization::DeoptReason reason) {
4729   ciMethodData* md = method->method_data();
4730   if (md->is_empty()) {
4731     // Assume the trap has not occurred, or that it occurred only
4732     // because of a transient condition during start-up in the interpreter.
4733     return false;
4734   }
4735   ciMethod* m = Deoptimization::reason_is_speculate(reason) ? this->method() : nullptr;
4736   if (md->has_trap_at(bci, m, reason) != 0) {
4737     // Assume PerBytecodeTrapLimit==0, for a more conservative heuristic.
4738     // Also, if there are multiple reasons, or if there is no per-BCI record,
4739     // assume the worst.
4740     if (log())
4741       log()->elem("observe trap='%s' count='%d'",
4742                   Deoptimization::trap_reason_name(reason),
4743                   md->trap_count(reason));
4744     return true;
4745   } else {
4746     // Ignore method/bci and see if there have been too many globally.
4747     return too_many_traps(reason, md);
4748   }
4749 }
4750 
4751 // Less-accurate variant which does not require a method and bci.
4752 bool Compile::too_many_traps(Deoptimization::DeoptReason reason,
4753                              ciMethodData* logmd) {
4754   if (trap_count(reason) >= Deoptimization::per_method_trap_limit(reason)) {
4755     // Too many traps globally.
4756     // Note that we use cumulative trap_count, not just md->trap_count.
4757     if (log()) {
4758       int mcount = (logmd == nullptr)? -1: (int)logmd->trap_count(reason);
4759       log()->elem("observe trap='%s' count='0' mcount='%d' ccount='%d'",
4760                   Deoptimization::trap_reason_name(reason),
4761                   mcount, trap_count(reason));
4762     }
4763     return true;
4764   } else {
4765     // The coast is clear.
4766     return false;
4767   }
4768 }
4769 
4770 //--------------------------too_many_recompiles--------------------------------
4771 // Report if there are too many recompiles at the current method and bci.
4772 // Consults PerBytecodeRecompilationCutoff and PerMethodRecompilationCutoff.
4773 // Is not eager to return true, since this will cause the compiler to use
4774 // Action_none for a trap point, to avoid too many recompilations.
4775 bool Compile::too_many_recompiles(ciMethod* method,
4776                                   int bci,
4777                                   Deoptimization::DeoptReason reason) {
4778   ciMethodData* md = method->method_data();
4779   if (md->is_empty()) {
4780     // Assume the trap has not occurred, or that it occurred only
4781     // because of a transient condition during start-up in the interpreter.
4782     return false;
4783   }
4784   // Pick a cutoff point well within PerBytecodeRecompilationCutoff.
4785   uint bc_cutoff = (uint) PerBytecodeRecompilationCutoff / 8;
4786   uint m_cutoff  = (uint) PerMethodRecompilationCutoff / 2 + 1;  // not zero
4787   Deoptimization::DeoptReason per_bc_reason
4788     = Deoptimization::reason_recorded_per_bytecode_if_any(reason);
4789   ciMethod* m = Deoptimization::reason_is_speculate(reason) ? this->method() : nullptr;
4790   if ((per_bc_reason == Deoptimization::Reason_none
4791        || md->has_trap_at(bci, m, reason) != 0)
4792       // The trap frequency measure we care about is the recompile count:
4793       && md->trap_recompiled_at(bci, m)
4794       && md->overflow_recompile_count() >= bc_cutoff) {
4795     // Do not emit a trap here if it has already caused recompilations.
4796     // Also, if there are multiple reasons, or if there is no per-BCI record,
4797     // assume the worst.
4798     if (log())
4799       log()->elem("observe trap='%s recompiled' count='%d' recompiles2='%d'",
4800                   Deoptimization::trap_reason_name(reason),
4801                   md->trap_count(reason),
4802                   md->overflow_recompile_count());
4803     return true;
4804   } else if (trap_count(reason) != 0
4805              && decompile_count() >= m_cutoff) {
4806     // Too many recompiles globally, and we have seen this sort of trap.
4807     // Use cumulative decompile_count, not just md->decompile_count.
4808     if (log())
4809       log()->elem("observe trap='%s' count='%d' mcount='%d' decompiles='%d' mdecompiles='%d'",
4810                   Deoptimization::trap_reason_name(reason),
4811                   md->trap_count(reason), trap_count(reason),
4812                   md->decompile_count(), decompile_count());
4813     return true;
4814   } else {
4815     // The coast is clear.
4816     return false;
4817   }
4818 }
4819 
4820 // Compute when not to trap. Used by matching trap based nodes and
4821 // NullCheck optimization.
4822 void Compile::set_allowed_deopt_reasons() {
4823   _allowed_reasons = 0;
4824   if (is_method_compilation()) {
4825     for (int rs = (int)Deoptimization::Reason_none+1; rs < Compile::trapHistLength; rs++) {
4826       assert(rs < BitsPerInt, "recode bit map");
4827       if (!too_many_traps((Deoptimization::DeoptReason) rs)) {
4828         _allowed_reasons |= nth_bit(rs);
4829       }
4830     }
4831   }
4832 }
4833 
4834 bool Compile::needs_clinit_barrier(ciMethod* method, ciMethod* accessing_method) {
4835   return method->is_static() && needs_clinit_barrier(method->holder(), accessing_method);
4836 }
4837 
4838 bool Compile::needs_clinit_barrier(ciField* field, ciMethod* accessing_method) {
4839   return field->is_static() && needs_clinit_barrier(field->holder(), accessing_method);
4840 }
4841 
4842 bool Compile::needs_clinit_barrier(ciInstanceKlass* holder, ciMethod* accessing_method) {
4843   if (holder->is_initialized()) {
4844     return false;
4845   }
4846   if (holder->is_being_initialized()) {
4847     if (accessing_method->holder() == holder) {
4848       // Access inside a class. The barrier can be elided when access happens in <clinit>,
4849       // <init>, or a static method. In all those cases, there was an initialization
4850       // barrier on the holder klass passed.
4851       if (accessing_method->is_class_initializer() ||
4852           accessing_method->is_object_constructor() ||
4853           accessing_method->is_static()) {
4854         return false;
4855       }
4856     } else if (accessing_method->holder()->is_subclass_of(holder)) {
4857       // Access from a subclass. The barrier can be elided only when access happens in <clinit>.
4858       // In case of <init> or a static method, the barrier is on the subclass is not enough:
4859       // child class can become fully initialized while its parent class is still being initialized.
4860       if (accessing_method->is_class_initializer()) {
4861         return false;
4862       }
4863     }
4864     ciMethod* root = method(); // the root method of compilation
4865     if (root != accessing_method) {
4866       return needs_clinit_barrier(holder, root); // check access in the context of compilation root
4867     }
4868   }
4869   return true;
4870 }
4871 
4872 #ifndef PRODUCT
4873 //------------------------------verify_bidirectional_edges---------------------
4874 // For each input edge to a node (ie - for each Use-Def edge), verify that
4875 // there is a corresponding Def-Use edge.
4876 void Compile::verify_bidirectional_edges(Unique_Node_List& visited, const Unique_Node_List* root_and_safepoints) const {
4877   // Allocate stack of size C->live_nodes()/16 to avoid frequent realloc
4878   uint stack_size = live_nodes() >> 4;
4879   Node_List nstack(MAX2(stack_size, (uint) OptoNodeListSize));
4880   if (root_and_safepoints != nullptr) {
4881     assert(root_and_safepoints->member(_root), "root is not in root_and_safepoints");
4882     for (uint i = 0, limit = root_and_safepoints->size(); i < limit; i++) {
4883       Node* root_or_safepoint = root_and_safepoints->at(i);
4884       // If the node is a safepoint, let's check if it still has a control input
4885       // Lack of control input signifies that this node was killed by CCP or
4886       // recursively by remove_globally_dead_node and it shouldn't be a starting
4887       // point.
4888       if (!root_or_safepoint->is_SafePoint() || root_or_safepoint->in(0) != nullptr) {
4889         nstack.push(root_or_safepoint);
4890       }
4891     }
4892   } else {
4893     nstack.push(_root);
4894   }
4895 
4896   while (nstack.size() > 0) {
4897     Node* n = nstack.pop();
4898     if (visited.member(n)) {
4899       continue;
4900     }
4901     visited.push(n);
4902 
4903     // Walk over all input edges, checking for correspondence
4904     uint length = n->len();
4905     for (uint i = 0; i < length; i++) {
4906       Node* in = n->in(i);
4907       if (in != nullptr && !visited.member(in)) {
4908         nstack.push(in); // Put it on stack
4909       }
4910       if (in != nullptr && !in->is_top()) {
4911         // Count instances of `next`
4912         int cnt = 0;
4913         for (uint idx = 0; idx < in->_outcnt; idx++) {
4914           if (in->_out[idx] == n) {
4915             cnt++;
4916           }
4917         }
4918         assert(cnt > 0, "Failed to find Def-Use edge.");
4919         // Check for duplicate edges
4920         // walk the input array downcounting the input edges to n
4921         for (uint j = 0; j < length; j++) {
4922           if (n->in(j) == in) {
4923             cnt--;
4924           }
4925         }
4926         assert(cnt == 0, "Mismatched edge count.");
4927       } else if (in == nullptr) {
4928         assert(i == 0 || i >= n->req() ||
4929                n->is_Region() || n->is_Phi() || n->is_ArrayCopy() ||
4930                (n->is_Allocate() && i >= AllocateNode::InlineType) ||
4931                (n->is_Unlock() && i == (n->req() - 1)) ||
4932                (n->is_MemBar() && i == 5), // the precedence edge to a membar can be removed during macro node expansion
4933               "only region, phi, arraycopy, allocate, unlock or membar nodes have null data edges");
4934       } else {
4935         assert(in->is_top(), "sanity");
4936         // Nothing to check.
4937       }
4938     }
4939   }
4940 }
4941 
4942 //------------------------------verify_graph_edges---------------------------
4943 // Walk the Graph and verify that there is a one-to-one correspondence
4944 // between Use-Def edges and Def-Use edges in the graph.
4945 void Compile::verify_graph_edges(bool no_dead_code, const Unique_Node_List* root_and_safepoints) const {
4946   if (VerifyGraphEdges) {
4947     Unique_Node_List visited;
4948 
4949     // Call graph walk to check edges
4950     verify_bidirectional_edges(visited, root_and_safepoints);
4951     if (no_dead_code) {
4952       // Now make sure that no visited node is used by an unvisited node.
4953       bool dead_nodes = false;
4954       Unique_Node_List checked;
4955       while (visited.size() > 0) {
4956         Node* n = visited.pop();
4957         checked.push(n);
4958         for (uint i = 0; i < n->outcnt(); i++) {
4959           Node* use = n->raw_out(i);
4960           if (checked.member(use))  continue;  // already checked
4961           if (visited.member(use))  continue;  // already in the graph
4962           if (use->is_Con())        continue;  // a dead ConNode is OK
4963           // At this point, we have found a dead node which is DU-reachable.
4964           if (!dead_nodes) {
4965             tty->print_cr("*** Dead nodes reachable via DU edges:");
4966             dead_nodes = true;
4967           }
4968           use->dump(2);
4969           tty->print_cr("---");
4970           checked.push(use);  // No repeats; pretend it is now checked.
4971         }
4972       }
4973       assert(!dead_nodes, "using nodes must be reachable from root");
4974     }
4975   }
4976 }
4977 #endif
4978 
4979 // The Compile object keeps track of failure reasons separately from the ciEnv.
4980 // This is required because there is not quite a 1-1 relation between the
4981 // ciEnv and its compilation task and the Compile object.  Note that one
4982 // ciEnv might use two Compile objects, if C2Compiler::compile_method decides
4983 // to backtrack and retry without subsuming loads.  Other than this backtracking
4984 // behavior, the Compile's failure reason is quietly copied up to the ciEnv
4985 // by the logic in C2Compiler.
4986 void Compile::record_failure(const char* reason DEBUG_ONLY(COMMA bool allow_multiple_failures)) {
4987   if (log() != nullptr) {
4988     log()->elem("failure reason='%s' phase='compile'", reason);
4989   }
4990   if (_failure_reason.get() == nullptr) {
4991     // Record the first failure reason.
4992     _failure_reason.set(reason);
4993     if (CaptureBailoutInformation) {
4994       _first_failure_details = new CompilationFailureInfo(reason);
4995     }
4996   } else {
4997     assert(!StressBailout || allow_multiple_failures, "should have handled previous failure.");
4998   }
4999 
5000   if (!C->failure_reason_is(C2Compiler::retry_no_subsuming_loads())) {
5001     C->print_method(PHASE_FAILURE, 1);
5002   }
5003   _root = nullptr;  // flush the graph, too
5004 }
5005 
5006 Compile::TracePhase::TracePhase(const char* name, PhaseTraceId id)
5007   : TraceTime(name, &Phase::timers[id], CITime, CITimeVerbose),
5008     _compile(Compile::current()),
5009     _log(nullptr),
5010     _dolog(CITimeVerbose)
5011 {
5012   assert(_compile != nullptr, "sanity check");
5013   assert(id != PhaseTraceId::_t_none, "Don't use none");
5014   if (_dolog) {
5015     _log = _compile->log();
5016   }
5017   if (_log != nullptr) {
5018     _log->begin_head("phase name='%s' nodes='%d' live='%d'", phase_name(), _compile->unique(), _compile->live_nodes());
5019     _log->stamp();
5020     _log->end_head();
5021   }
5022 
5023   // Inform memory statistic, if enabled
5024   if (CompilationMemoryStatistic::enabled()) {
5025     CompilationMemoryStatistic::on_phase_start((int)id, name);
5026   }
5027 }
5028 
5029 Compile::TracePhase::TracePhase(PhaseTraceId id)
5030   : TracePhase(Phase::get_phase_trace_id_text(id), id) {}
5031 
5032 Compile::TracePhase::~TracePhase() {
5033 
5034   // Inform memory statistic, if enabled
5035   if (CompilationMemoryStatistic::enabled()) {
5036     CompilationMemoryStatistic::on_phase_end();
5037   }
5038 
5039   if (_compile->failing_internal()) {
5040     if (_log != nullptr) {
5041       _log->done("phase");
5042     }
5043     return; // timing code, not stressing bailouts.
5044   }
5045 #ifdef ASSERT
5046   if (PrintIdealNodeCount) {
5047     tty->print_cr("phase name='%s' nodes='%d' live='%d' live_graph_walk='%d'",
5048                   phase_name(), _compile->unique(), _compile->live_nodes(), _compile->count_live_nodes_by_graph_walk());
5049   }
5050 
5051   if (VerifyIdealNodeCount) {
5052     _compile->print_missing_nodes();
5053   }
5054 #endif
5055 
5056   if (_log != nullptr) {
5057     _log->done("phase name='%s' nodes='%d' live='%d'", phase_name(), _compile->unique(), _compile->live_nodes());
5058   }
5059 }
5060 
5061 //----------------------------static_subtype_check-----------------------------
5062 // Shortcut important common cases when superklass is exact:
5063 // (0) superklass is java.lang.Object (can occur in reflective code)
5064 // (1) subklass is already limited to a subtype of superklass => always ok
5065 // (2) subklass does not overlap with superklass => always fail
5066 // (3) superklass has NO subtypes and we can check with a simple compare.
5067 Compile::SubTypeCheckResult Compile::static_subtype_check(const TypeKlassPtr* superk, const TypeKlassPtr* subk, bool skip) {
5068   if (skip) {
5069     return SSC_full_test;       // Let caller generate the general case.
5070   }
5071 
5072   if (subk->is_java_subtype_of(superk)) {
5073     return SSC_always_true; // (0) and (1)  this test cannot fail
5074   }
5075 
5076   if (!subk->maybe_java_subtype_of(superk)) {
5077     return SSC_always_false; // (2) true path dead; no dynamic test needed
5078   }
5079 
5080   const Type* superelem = superk;
5081   if (superk->isa_aryklassptr()) {
5082     int ignored;
5083     superelem = superk->is_aryklassptr()->base_element_type(ignored);
5084 
5085     // Do not fold the subtype check to an array klass pointer comparison for null-able inline type arrays
5086     // because null-free [LMyValue <: null-able [LMyValue but the klasses are different. Perform a full test.
5087     if (!superk->is_aryklassptr()->is_null_free() && superk->is_aryklassptr()->elem()->isa_instklassptr() &&
5088         superk->is_aryklassptr()->elem()->is_instklassptr()->instance_klass()->is_inlinetype()) {
5089       return SSC_full_test;
5090     }
5091   }
5092 
5093   if (superelem->isa_instklassptr()) {
5094     ciInstanceKlass* ik = superelem->is_instklassptr()->instance_klass();
5095     if (!ik->has_subklass()) {
5096       if (!ik->is_final()) {
5097         // Add a dependency if there is a chance of a later subclass.
5098         dependencies()->assert_leaf_type(ik);
5099       }
5100       if (!superk->maybe_java_subtype_of(subk)) {
5101         return SSC_always_false;
5102       }
5103       return SSC_easy_test;     // (3) caller can do a simple ptr comparison
5104     }
5105   } else {
5106     // A primitive array type has no subtypes.
5107     return SSC_easy_test;       // (3) caller can do a simple ptr comparison
5108   }
5109 
5110   return SSC_full_test;
5111 }
5112 
5113 Node* Compile::conv_I2X_index(PhaseGVN* phase, Node* idx, const TypeInt* sizetype, Node* ctrl) {
5114 #ifdef _LP64
5115   // The scaled index operand to AddP must be a clean 64-bit value.
5116   // Java allows a 32-bit int to be incremented to a negative
5117   // value, which appears in a 64-bit register as a large
5118   // positive number.  Using that large positive number as an
5119   // operand in pointer arithmetic has bad consequences.
5120   // On the other hand, 32-bit overflow is rare, and the possibility
5121   // can often be excluded, if we annotate the ConvI2L node with
5122   // a type assertion that its value is known to be a small positive
5123   // number.  (The prior range check has ensured this.)
5124   // This assertion is used by ConvI2LNode::Ideal.
5125   int index_max = max_jint - 1;  // array size is max_jint, index is one less
5126   if (sizetype != nullptr && sizetype->_hi > 0) {
5127     index_max = sizetype->_hi - 1;
5128   }
5129   const TypeInt* iidxtype = TypeInt::make(0, index_max, Type::WidenMax);
5130   idx = constrained_convI2L(phase, idx, iidxtype, ctrl);
5131 #endif
5132   return idx;
5133 }
5134 
5135 // Convert integer value to a narrowed long type dependent on ctrl (for example, a range check)
5136 Node* Compile::constrained_convI2L(PhaseGVN* phase, Node* value, const TypeInt* itype, Node* ctrl, bool carry_dependency) {
5137   if (ctrl != nullptr) {
5138     // Express control dependency by a CastII node with a narrow type.
5139     // Make the CastII node dependent on the control input to prevent the narrowed ConvI2L
5140     // node from floating above the range check during loop optimizations. Otherwise, the
5141     // ConvI2L node may be eliminated independently of the range check, causing the data path
5142     // to become TOP while the control path is still there (although it's unreachable).
5143     value = new CastIINode(ctrl, value, itype, carry_dependency ? ConstraintCastNode::StrongDependency : ConstraintCastNode::RegularDependency, true /* range check dependency */);
5144     value = phase->transform(value);
5145   }
5146   const TypeLong* ltype = TypeLong::make(itype->_lo, itype->_hi, itype->_widen);
5147   return phase->transform(new ConvI2LNode(value, ltype));
5148 }
5149 
5150 void Compile::dump_print_inlining() {
5151   inline_printer()->print_on(tty);
5152 }
5153 
5154 void Compile::log_late_inline(CallGenerator* cg) {
5155   if (log() != nullptr) {
5156     log()->head("late_inline method='%d' inline_id='" JLONG_FORMAT "'", log()->identify(cg->method()),
5157                 cg->unique_id());
5158     JVMState* p = cg->call_node()->jvms();
5159     while (p != nullptr) {
5160       log()->elem("jvms bci='%d' method='%d'", p->bci(), log()->identify(p->method()));
5161       p = p->caller();
5162     }
5163     log()->tail("late_inline");
5164   }
5165 }
5166 
5167 void Compile::log_late_inline_failure(CallGenerator* cg, const char* msg) {
5168   log_late_inline(cg);
5169   if (log() != nullptr) {
5170     log()->inline_fail(msg);
5171   }
5172 }
5173 
5174 void Compile::log_inline_id(CallGenerator* cg) {
5175   if (log() != nullptr) {
5176     // The LogCompilation tool needs a unique way to identify late
5177     // inline call sites. This id must be unique for this call site in
5178     // this compilation. Try to have it unique across compilations as
5179     // well because it can be convenient when grepping through the log
5180     // file.
5181     // Distinguish OSR compilations from others in case CICountOSR is
5182     // on.
5183     jlong id = ((jlong)unique()) + (((jlong)compile_id()) << 33) + (CICountOSR && is_osr_compilation() ? ((jlong)1) << 32 : 0);
5184     cg->set_unique_id(id);
5185     log()->elem("inline_id id='" JLONG_FORMAT "'", id);
5186   }
5187 }
5188 
5189 void Compile::log_inline_failure(const char* msg) {
5190   if (C->log() != nullptr) {
5191     C->log()->inline_fail(msg);
5192   }
5193 }
5194 
5195 
5196 // Dump inlining replay data to the stream.
5197 // Don't change thread state and acquire any locks.
5198 void Compile::dump_inline_data(outputStream* out) {
5199   InlineTree* inl_tree = ilt();
5200   if (inl_tree != nullptr) {
5201     out->print(" inline %d", inl_tree->count());
5202     inl_tree->dump_replay_data(out);
5203   }
5204 }
5205 
5206 void Compile::dump_inline_data_reduced(outputStream* out) {
5207   assert(ReplayReduce, "");
5208 
5209   InlineTree* inl_tree = ilt();
5210   if (inl_tree == nullptr) {
5211     return;
5212   }
5213   // Enable iterative replay file reduction
5214   // Output "compile" lines for depth 1 subtrees,
5215   // simulating that those trees were compiled
5216   // instead of inlined.
5217   for (int i = 0; i < inl_tree->subtrees().length(); ++i) {
5218     InlineTree* sub = inl_tree->subtrees().at(i);
5219     if (sub->inline_level() != 1) {
5220       continue;
5221     }
5222 
5223     ciMethod* method = sub->method();
5224     int entry_bci = -1;
5225     int comp_level = env()->task()->comp_level();
5226     out->print("compile ");
5227     method->dump_name_as_ascii(out);
5228     out->print(" %d %d", entry_bci, comp_level);
5229     out->print(" inline %d", sub->count());
5230     sub->dump_replay_data(out, -1);
5231     out->cr();
5232   }
5233 }
5234 
5235 int Compile::cmp_expensive_nodes(Node* n1, Node* n2) {
5236   if (n1->Opcode() < n2->Opcode())      return -1;
5237   else if (n1->Opcode() > n2->Opcode()) return 1;
5238 
5239   assert(n1->req() == n2->req(), "can't compare %s nodes: n1->req() = %d, n2->req() = %d", NodeClassNames[n1->Opcode()], n1->req(), n2->req());
5240   for (uint i = 1; i < n1->req(); i++) {
5241     if (n1->in(i) < n2->in(i))      return -1;
5242     else if (n1->in(i) > n2->in(i)) return 1;
5243   }
5244 
5245   return 0;
5246 }
5247 
5248 int Compile::cmp_expensive_nodes(Node** n1p, Node** n2p) {
5249   Node* n1 = *n1p;
5250   Node* n2 = *n2p;
5251 
5252   return cmp_expensive_nodes(n1, n2);
5253 }
5254 
5255 void Compile::sort_expensive_nodes() {
5256   if (!expensive_nodes_sorted()) {
5257     _expensive_nodes.sort(cmp_expensive_nodes);
5258   }
5259 }
5260 
5261 bool Compile::expensive_nodes_sorted() const {
5262   for (int i = 1; i < _expensive_nodes.length(); i++) {
5263     if (cmp_expensive_nodes(_expensive_nodes.adr_at(i), _expensive_nodes.adr_at(i-1)) < 0) {
5264       return false;
5265     }
5266   }
5267   return true;
5268 }
5269 
5270 bool Compile::should_optimize_expensive_nodes(PhaseIterGVN &igvn) {
5271   if (_expensive_nodes.length() == 0) {
5272     return false;
5273   }
5274 
5275   assert(OptimizeExpensiveOps, "optimization off?");
5276 
5277   // Take this opportunity to remove dead nodes from the list
5278   int j = 0;
5279   for (int i = 0; i < _expensive_nodes.length(); i++) {
5280     Node* n = _expensive_nodes.at(i);
5281     if (!n->is_unreachable(igvn)) {
5282       assert(n->is_expensive(), "should be expensive");
5283       _expensive_nodes.at_put(j, n);
5284       j++;
5285     }
5286   }
5287   _expensive_nodes.trunc_to(j);
5288 
5289   // Then sort the list so that similar nodes are next to each other
5290   // and check for at least two nodes of identical kind with same data
5291   // inputs.
5292   sort_expensive_nodes();
5293 
5294   for (int i = 0; i < _expensive_nodes.length()-1; i++) {
5295     if (cmp_expensive_nodes(_expensive_nodes.adr_at(i), _expensive_nodes.adr_at(i+1)) == 0) {
5296       return true;
5297     }
5298   }
5299 
5300   return false;
5301 }
5302 
5303 void Compile::cleanup_expensive_nodes(PhaseIterGVN &igvn) {
5304   if (_expensive_nodes.length() == 0) {
5305     return;
5306   }
5307 
5308   assert(OptimizeExpensiveOps, "optimization off?");
5309 
5310   // Sort to bring similar nodes next to each other and clear the
5311   // control input of nodes for which there's only a single copy.
5312   sort_expensive_nodes();
5313 
5314   int j = 0;
5315   int identical = 0;
5316   int i = 0;
5317   bool modified = false;
5318   for (; i < _expensive_nodes.length()-1; i++) {
5319     assert(j <= i, "can't write beyond current index");
5320     if (_expensive_nodes.at(i)->Opcode() == _expensive_nodes.at(i+1)->Opcode()) {
5321       identical++;
5322       _expensive_nodes.at_put(j++, _expensive_nodes.at(i));
5323       continue;
5324     }
5325     if (identical > 0) {
5326       _expensive_nodes.at_put(j++, _expensive_nodes.at(i));
5327       identical = 0;
5328     } else {
5329       Node* n = _expensive_nodes.at(i);
5330       igvn.replace_input_of(n, 0, nullptr);
5331       igvn.hash_insert(n);
5332       modified = true;
5333     }
5334   }
5335   if (identical > 0) {
5336     _expensive_nodes.at_put(j++, _expensive_nodes.at(i));
5337   } else if (_expensive_nodes.length() >= 1) {
5338     Node* n = _expensive_nodes.at(i);
5339     igvn.replace_input_of(n, 0, nullptr);
5340     igvn.hash_insert(n);
5341     modified = true;
5342   }
5343   _expensive_nodes.trunc_to(j);
5344   if (modified) {
5345     igvn.optimize();
5346   }
5347 }
5348 
5349 void Compile::add_expensive_node(Node * n) {
5350   assert(!_expensive_nodes.contains(n), "duplicate entry in expensive list");
5351   assert(n->is_expensive(), "expensive nodes with non-null control here only");
5352   assert(!n->is_CFG() && !n->is_Mem(), "no cfg or memory nodes here");
5353   if (OptimizeExpensiveOps) {
5354     _expensive_nodes.append(n);
5355   } else {
5356     // Clear control input and let IGVN optimize expensive nodes if
5357     // OptimizeExpensiveOps is off.
5358     n->set_req(0, nullptr);
5359   }
5360 }
5361 
5362 /**
5363  * Track coarsened Lock and Unlock nodes.
5364  */
5365 
5366 class Lock_List : public Node_List {
5367   uint _origin_cnt;
5368 public:
5369   Lock_List(Arena *a, uint cnt) : Node_List(a), _origin_cnt(cnt) {}
5370   uint origin_cnt() const { return _origin_cnt; }
5371 };
5372 
5373 void Compile::add_coarsened_locks(GrowableArray<AbstractLockNode*>& locks) {
5374   int length = locks.length();
5375   if (length > 0) {
5376     // Have to keep this list until locks elimination during Macro nodes elimination.
5377     Lock_List* locks_list = new (comp_arena()) Lock_List(comp_arena(), length);
5378     AbstractLockNode* alock = locks.at(0);
5379     BoxLockNode* box = alock->box_node()->as_BoxLock();
5380     for (int i = 0; i < length; i++) {
5381       AbstractLockNode* lock = locks.at(i);
5382       assert(lock->is_coarsened(), "expecting only coarsened AbstractLock nodes, but got '%s'[%d] node", lock->Name(), lock->_idx);
5383       locks_list->push(lock);
5384       BoxLockNode* this_box = lock->box_node()->as_BoxLock();
5385       if (this_box != box) {
5386         // Locking regions (BoxLock) could be Unbalanced here:
5387         //  - its coarsened locks were eliminated in earlier
5388         //    macro nodes elimination followed by loop unroll
5389         //  - it is OSR locking region (no Lock node)
5390         // Preserve Unbalanced status in such cases.
5391         if (!this_box->is_unbalanced()) {
5392           this_box->set_coarsened();
5393         }
5394         if (!box->is_unbalanced()) {
5395           box->set_coarsened();
5396         }
5397       }
5398     }
5399     _coarsened_locks.append(locks_list);
5400   }
5401 }
5402 
5403 void Compile::remove_useless_coarsened_locks(Unique_Node_List& useful) {
5404   int count = coarsened_count();
5405   for (int i = 0; i < count; i++) {
5406     Node_List* locks_list = _coarsened_locks.at(i);
5407     for (uint j = 0; j < locks_list->size(); j++) {
5408       Node* lock = locks_list->at(j);
5409       assert(lock->is_AbstractLock(), "sanity");
5410       if (!useful.member(lock)) {
5411         locks_list->yank(lock);
5412       }
5413     }
5414   }
5415 }
5416 
5417 void Compile::remove_coarsened_lock(Node* n) {
5418   if (n->is_AbstractLock()) {
5419     int count = coarsened_count();
5420     for (int i = 0; i < count; i++) {
5421       Node_List* locks_list = _coarsened_locks.at(i);
5422       locks_list->yank(n);
5423     }
5424   }
5425 }
5426 
5427 bool Compile::coarsened_locks_consistent() {
5428   int count = coarsened_count();
5429   for (int i = 0; i < count; i++) {
5430     bool unbalanced = false;
5431     bool modified = false; // track locks kind modifications
5432     Lock_List* locks_list = (Lock_List*)_coarsened_locks.at(i);
5433     uint size = locks_list->size();
5434     if (size == 0) {
5435       unbalanced = false; // All locks were eliminated - good
5436     } else if (size != locks_list->origin_cnt()) {
5437       unbalanced = true; // Some locks were removed from list
5438     } else {
5439       for (uint j = 0; j < size; j++) {
5440         Node* lock = locks_list->at(j);
5441         // All nodes in group should have the same state (modified or not)
5442         if (!lock->as_AbstractLock()->is_coarsened()) {
5443           if (j == 0) {
5444             // first on list was modified, the rest should be too for consistency
5445             modified = true;
5446           } else if (!modified) {
5447             // this lock was modified but previous locks on the list were not
5448             unbalanced = true;
5449             break;
5450           }
5451         } else if (modified) {
5452           // previous locks on list were modified but not this lock
5453           unbalanced = true;
5454           break;
5455         }
5456       }
5457     }
5458     if (unbalanced) {
5459       // unbalanced monitor enter/exit - only some [un]lock nodes were removed or modified
5460 #ifdef ASSERT
5461       if (PrintEliminateLocks) {
5462         tty->print_cr("=== unbalanced coarsened locks ===");
5463         for (uint l = 0; l < size; l++) {
5464           locks_list->at(l)->dump();
5465         }
5466       }
5467 #endif
5468       record_failure(C2Compiler::retry_no_locks_coarsening());
5469       return false;
5470     }
5471   }
5472   return true;
5473 }
5474 
5475 // Mark locking regions (identified by BoxLockNode) as unbalanced if
5476 // locks coarsening optimization removed Lock/Unlock nodes from them.
5477 // Such regions become unbalanced because coarsening only removes part
5478 // of Lock/Unlock nodes in region. As result we can't execute other
5479 // locks elimination optimizations which assume all code paths have
5480 // corresponding pair of Lock/Unlock nodes - they are balanced.
5481 void Compile::mark_unbalanced_boxes() const {
5482   int count = coarsened_count();
5483   for (int i = 0; i < count; i++) {
5484     Node_List* locks_list = _coarsened_locks.at(i);
5485     uint size = locks_list->size();
5486     if (size > 0) {
5487       AbstractLockNode* alock = locks_list->at(0)->as_AbstractLock();
5488       BoxLockNode* box = alock->box_node()->as_BoxLock();
5489       if (alock->is_coarsened()) {
5490         // coarsened_locks_consistent(), which is called before this method, verifies
5491         // that the rest of Lock/Unlock nodes on locks_list are also coarsened.
5492         assert(!box->is_eliminated(), "regions with coarsened locks should not be marked as eliminated");
5493         for (uint j = 1; j < size; j++) {
5494           assert(locks_list->at(j)->as_AbstractLock()->is_coarsened(), "only coarsened locks are expected here");
5495           BoxLockNode* this_box = locks_list->at(j)->as_AbstractLock()->box_node()->as_BoxLock();
5496           if (box != this_box) {
5497             assert(!this_box->is_eliminated(), "regions with coarsened locks should not be marked as eliminated");
5498             box->set_unbalanced();
5499             this_box->set_unbalanced();
5500           }
5501         }
5502       }
5503     }
5504   }
5505 }
5506 
5507 /**
5508  * Remove the speculative part of types and clean up the graph
5509  */
5510 void Compile::remove_speculative_types(PhaseIterGVN &igvn) {
5511   if (UseTypeSpeculation) {
5512     Unique_Node_List worklist;
5513     worklist.push(root());
5514     int modified = 0;
5515     // Go over all type nodes that carry a speculative type, drop the
5516     // speculative part of the type and enqueue the node for an igvn
5517     // which may optimize it out.
5518     for (uint next = 0; next < worklist.size(); ++next) {
5519       Node *n  = worklist.at(next);
5520       if (n->is_Type()) {
5521         TypeNode* tn = n->as_Type();
5522         const Type* t = tn->type();
5523         const Type* t_no_spec = t->remove_speculative();
5524         if (t_no_spec != t) {
5525           bool in_hash = igvn.hash_delete(n);
5526           assert(in_hash || n->hash() == Node::NO_HASH, "node should be in igvn hash table");
5527           tn->set_type(t_no_spec);
5528           igvn.hash_insert(n);
5529           igvn._worklist.push(n); // give it a chance to go away
5530           modified++;
5531         }
5532       }
5533       // Iterate over outs - endless loops is unreachable from below
5534       for (DUIterator_Fast imax, i = n->fast_outs(imax); i < imax; i++) {
5535         Node *m = n->fast_out(i);
5536         if (not_a_node(m)) {
5537           continue;
5538         }
5539         worklist.push(m);
5540       }
5541     }
5542     // Drop the speculative part of all types in the igvn's type table
5543     igvn.remove_speculative_types();
5544     if (modified > 0) {
5545       igvn.optimize();
5546       if (failing())  return;
5547     }
5548 #ifdef ASSERT
5549     // Verify that after the IGVN is over no speculative type has resurfaced
5550     worklist.clear();
5551     worklist.push(root());
5552     for (uint next = 0; next < worklist.size(); ++next) {
5553       Node *n  = worklist.at(next);
5554       const Type* t = igvn.type_or_null(n);
5555       assert((t == nullptr) || (t == t->remove_speculative()), "no more speculative types");
5556       if (n->is_Type()) {
5557         t = n->as_Type()->type();
5558         assert(t == t->remove_speculative(), "no more speculative types");
5559       }
5560       // Iterate over outs - endless loops is unreachable from below
5561       for (DUIterator_Fast imax, i = n->fast_outs(imax); i < imax; i++) {
5562         Node *m = n->fast_out(i);
5563         if (not_a_node(m)) {
5564           continue;
5565         }
5566         worklist.push(m);
5567       }
5568     }
5569     igvn.check_no_speculative_types();
5570 #endif
5571   }
5572 }
5573 
5574 Node* Compile::optimize_acmp(PhaseGVN* phase, Node* a, Node* b) {
5575   const TypeInstPtr* ta = phase->type(a)->isa_instptr();
5576   const TypeInstPtr* tb = phase->type(b)->isa_instptr();
5577   if (!EnableValhalla || ta == nullptr || tb == nullptr ||
5578       ta->is_zero_type() || tb->is_zero_type() ||
5579       !ta->can_be_inline_type() || !tb->can_be_inline_type()) {
5580     // Use old acmp if one operand is null or not an inline type
5581     return new CmpPNode(a, b);
5582   } else if (ta->is_inlinetypeptr() || tb->is_inlinetypeptr()) {
5583     // We know that one operand is an inline type. Therefore,
5584     // new acmp will only return true if both operands are nullptr.
5585     // Check if both operands are null by or'ing the oops.
5586     a = phase->transform(new CastP2XNode(nullptr, a));
5587     b = phase->transform(new CastP2XNode(nullptr, b));
5588     a = phase->transform(new OrXNode(a, b));
5589     return new CmpXNode(a, phase->MakeConX(0));
5590   }
5591   // Use new acmp
5592   return nullptr;
5593 }
5594 
5595 // Auxiliary methods to support randomized stressing/fuzzing.
5596 
5597 void Compile::initialize_stress_seed(const DirectiveSet* directive) {
5598   if (FLAG_IS_DEFAULT(StressSeed) || (FLAG_IS_ERGO(StressSeed) && directive->RepeatCompilationOption)) {
5599     _stress_seed = static_cast<uint>(Ticks::now().nanoseconds());
5600     FLAG_SET_ERGO(StressSeed, _stress_seed);
5601   } else {
5602     _stress_seed = StressSeed;
5603   }
5604   if (_log != nullptr) {
5605     _log->elem("stress_test seed='%u'", _stress_seed);
5606   }
5607 }
5608 
5609 int Compile::random() {
5610   _stress_seed = os::next_random(_stress_seed);
5611   return static_cast<int>(_stress_seed);
5612 }
5613 
5614 // This method can be called the arbitrary number of times, with current count
5615 // as the argument. The logic allows selecting a single candidate from the
5616 // running list of candidates as follows:
5617 //    int count = 0;
5618 //    Cand* selected = null;
5619 //    while(cand = cand->next()) {
5620 //      if (randomized_select(++count)) {
5621 //        selected = cand;
5622 //      }
5623 //    }
5624 //
5625 // Including count equalizes the chances any candidate is "selected".
5626 // This is useful when we don't have the complete list of candidates to choose
5627 // from uniformly. In this case, we need to adjust the randomicity of the
5628 // selection, or else we will end up biasing the selection towards the latter
5629 // candidates.
5630 //
5631 // Quick back-envelope calculation shows that for the list of n candidates
5632 // the equal probability for the candidate to persist as "best" can be
5633 // achieved by replacing it with "next" k-th candidate with the probability
5634 // of 1/k. It can be easily shown that by the end of the run, the
5635 // probability for any candidate is converged to 1/n, thus giving the
5636 // uniform distribution among all the candidates.
5637 //
5638 // We don't care about the domain size as long as (RANDOMIZED_DOMAIN / count) is large.
5639 #define RANDOMIZED_DOMAIN_POW 29
5640 #define RANDOMIZED_DOMAIN (1 << RANDOMIZED_DOMAIN_POW)
5641 #define RANDOMIZED_DOMAIN_MASK ((1 << (RANDOMIZED_DOMAIN_POW + 1)) - 1)
5642 bool Compile::randomized_select(int count) {
5643   assert(count > 0, "only positive");
5644   return (random() & RANDOMIZED_DOMAIN_MASK) < (RANDOMIZED_DOMAIN / count);
5645 }
5646 
5647 #ifdef ASSERT
5648 // Failures are geometrically distributed with probability 1/StressBailoutMean.
5649 bool Compile::fail_randomly() {
5650   if ((random() % StressBailoutMean) != 0) {
5651     return false;
5652   }
5653   record_failure("StressBailout");
5654   return true;
5655 }
5656 
5657 bool Compile::failure_is_artificial() {
5658   return C->failure_reason_is("StressBailout");
5659 }
5660 #endif
5661 
5662 CloneMap&     Compile::clone_map()                 { return _clone_map; }
5663 void          Compile::set_clone_map(Dict* d)      { _clone_map._dict = d; }
5664 
5665 void NodeCloneInfo::dump_on(outputStream* st) const {
5666   st->print(" {%d:%d} ", idx(), gen());
5667 }
5668 
5669 void CloneMap::clone(Node* old, Node* nnn, int gen) {
5670   uint64_t val = value(old->_idx);
5671   NodeCloneInfo cio(val);
5672   assert(val != 0, "old node should be in the map");
5673   NodeCloneInfo cin(cio.idx(), gen + cio.gen());
5674   insert(nnn->_idx, cin.get());
5675 #ifndef PRODUCT
5676   if (is_debug()) {
5677     tty->print_cr("CloneMap::clone inserted node %d info {%d:%d} into CloneMap", nnn->_idx, cin.idx(), cin.gen());
5678   }
5679 #endif
5680 }
5681 
5682 void CloneMap::verify_insert_and_clone(Node* old, Node* nnn, int gen) {
5683   NodeCloneInfo cio(value(old->_idx));
5684   if (cio.get() == 0) {
5685     cio.set(old->_idx, 0);
5686     insert(old->_idx, cio.get());
5687 #ifndef PRODUCT
5688     if (is_debug()) {
5689       tty->print_cr("CloneMap::verify_insert_and_clone inserted node %d info {%d:%d} into CloneMap", old->_idx, cio.idx(), cio.gen());
5690     }
5691 #endif
5692   }
5693   clone(old, nnn, gen);
5694 }
5695 
5696 int CloneMap::max_gen() const {
5697   int g = 0;
5698   DictI di(_dict);
5699   for(; di.test(); ++di) {
5700     int t = gen(di._key);
5701     if (g < t) {
5702       g = t;
5703 #ifndef PRODUCT
5704       if (is_debug()) {
5705         tty->print_cr("CloneMap::max_gen() update max=%d from %d", g, _2_node_idx_t(di._key));
5706       }
5707 #endif
5708     }
5709   }
5710   return g;
5711 }
5712 
5713 void CloneMap::dump(node_idx_t key, outputStream* st) const {
5714   uint64_t val = value(key);
5715   if (val != 0) {
5716     NodeCloneInfo ni(val);
5717     ni.dump_on(st);
5718   }
5719 }
5720 
5721 void Compile::shuffle_macro_nodes() {
5722   if (_macro_nodes.length() < 2) {
5723     return;
5724   }
5725   for (uint i = _macro_nodes.length() - 1; i >= 1; i--) {
5726     uint j = C->random() % (i + 1);
5727     swap(_macro_nodes.at(i), _macro_nodes.at(j));
5728   }
5729 }
5730 
5731 // Move Allocate nodes to the start of the list
5732 void Compile::sort_macro_nodes() {
5733   int count = macro_count();
5734   int allocates = 0;
5735   for (int i = 0; i < count; i++) {
5736     Node* n = macro_node(i);
5737     if (n->is_Allocate()) {
5738       if (i != allocates) {
5739         Node* tmp = macro_node(allocates);
5740         _macro_nodes.at_put(allocates, n);
5741         _macro_nodes.at_put(i, tmp);
5742       }
5743       allocates++;
5744     }
5745   }
5746 }
5747 
5748 void Compile::print_method(CompilerPhaseType cpt, int level, Node* n) {
5749   if (failing_internal()) { return; } // failing_internal to not stress bailouts from printing code.
5750   EventCompilerPhase event(UNTIMED);
5751   if (event.should_commit()) {
5752     CompilerEvent::PhaseEvent::post(event, C->_latest_stage_start_counter, cpt, C->_compile_id, level);
5753   }
5754 #ifndef PRODUCT
5755   ResourceMark rm;
5756   stringStream ss;
5757   ss.print_raw(CompilerPhaseTypeHelper::to_description(cpt));
5758   int iter = ++_igv_phase_iter[cpt];
5759   if (iter > 1) {
5760     ss.print(" %d", iter);
5761   }
5762   if (n != nullptr) {
5763     ss.print(": %d %s", n->_idx, NodeClassNames[n->Opcode()]);
5764     if (n->is_Call()) {
5765       CallNode* call = n->as_Call();
5766       if (call->_name != nullptr) {
5767         // E.g. uncommon traps etc.
5768         ss.print(" - %s", call->_name);
5769       } else if (call->is_CallJava()) {
5770         CallJavaNode* call_java = call->as_CallJava();
5771         if (call_java->method() != nullptr) {
5772           ss.print(" -");
5773           call_java->method()->print_short_name(&ss);
5774         }
5775       }
5776     }
5777   }
5778 
5779   const char* name = ss.as_string();
5780   if (should_print_igv(level)) {
5781     _igv_printer->print_graph(name);
5782   }
5783   if (should_print_phase(level)) {
5784     print_phase(name);
5785   }
5786   if (should_print_ideal_phase(cpt)) {
5787     print_ideal_ir(CompilerPhaseTypeHelper::to_name(cpt));
5788   }
5789 #endif
5790   C->_latest_stage_start_counter.stamp();
5791 }
5792 
5793 // Only used from CompileWrapper
5794 void Compile::begin_method() {
5795 #ifndef PRODUCT
5796   if (_method != nullptr && should_print_igv(1)) {
5797     _igv_printer->begin_method();
5798   }
5799 #endif
5800   C->_latest_stage_start_counter.stamp();
5801 }
5802 
5803 // Only used from CompileWrapper
5804 void Compile::end_method() {
5805   EventCompilerPhase event(UNTIMED);
5806   if (event.should_commit()) {
5807     CompilerEvent::PhaseEvent::post(event, C->_latest_stage_start_counter, PHASE_END, C->_compile_id, 1);
5808   }
5809 
5810 #ifndef PRODUCT
5811   if (_method != nullptr && should_print_igv(1)) {
5812     _igv_printer->end_method();
5813   }
5814 #endif
5815 }
5816 
5817 #ifndef PRODUCT
5818 bool Compile::should_print_phase(const int level) const {
5819   return PrintPhaseLevel > 0 && directive()->PhasePrintLevelOption >= level &&
5820          _method != nullptr; // Do not print phases for stubs.
5821 }
5822 
5823 bool Compile::should_print_ideal_phase(CompilerPhaseType cpt) const {
5824   return _directive->should_print_ideal_phase(cpt);
5825 }
5826 
5827 void Compile::init_igv() {
5828   if (_igv_printer == nullptr) {
5829     _igv_printer = IdealGraphPrinter::printer();
5830     _igv_printer->set_compile(this);
5831   }
5832 }
5833 
5834 bool Compile::should_print_igv(const int level) {
5835   PRODUCT_RETURN_(return false;);
5836 
5837   if (PrintIdealGraphLevel < 0) { // disabled by the user
5838     return false;
5839   }
5840 
5841   bool need = directive()->IGVPrintLevelOption >= level;
5842   if (need) {
5843     Compile::init_igv();
5844   }
5845   return need;
5846 }
5847 
5848 IdealGraphPrinter* Compile::_debug_file_printer = nullptr;
5849 IdealGraphPrinter* Compile::_debug_network_printer = nullptr;
5850 
5851 // Called from debugger. Prints method to the default file with the default phase name.
5852 // This works regardless of any Ideal Graph Visualizer flags set or not.
5853 // Use in debugger (gdb/rr): p igv_print($sp, $fp, $pc).
5854 void igv_print(void* sp, void* fp, void* pc) {
5855   frame fr(sp, fp, pc);
5856   Compile::current()->igv_print_method_to_file(nullptr, false, &fr);
5857 }
5858 
5859 // Same as igv_print() above but with a specified phase name.
5860 void igv_print(const char* phase_name, void* sp, void* fp, void* pc) {
5861   frame fr(sp, fp, pc);
5862   Compile::current()->igv_print_method_to_file(phase_name, false, &fr);
5863 }
5864 
5865 // Called from debugger. Prints method with the default phase name to the default network or the one specified with
5866 // the network flags for the Ideal Graph Visualizer, or to the default file depending on the 'network' argument.
5867 // This works regardless of any Ideal Graph Visualizer flags set or not.
5868 // Use in debugger (gdb/rr): p igv_print(true, $sp, $fp, $pc).
5869 void igv_print(bool network, void* sp, void* fp, void* pc) {
5870   frame fr(sp, fp, pc);
5871   if (network) {
5872     Compile::current()->igv_print_method_to_network(nullptr, &fr);
5873   } else {
5874     Compile::current()->igv_print_method_to_file(nullptr, false, &fr);
5875   }
5876 }
5877 
5878 // Same as igv_print(bool network, ...) above but with a specified phase name.
5879 // Use in debugger (gdb/rr): p igv_print(true, "MyPhase", $sp, $fp, $pc).
5880 void igv_print(bool network, const char* phase_name, void* sp, void* fp, void* pc) {
5881   frame fr(sp, fp, pc);
5882   if (network) {
5883     Compile::current()->igv_print_method_to_network(phase_name, &fr);
5884   } else {
5885     Compile::current()->igv_print_method_to_file(phase_name, false, &fr);
5886   }
5887 }
5888 
5889 // Called from debugger. Normal write to the default _printer. Only works if Ideal Graph Visualizer printing flags are set.
5890 void igv_print_default() {
5891   Compile::current()->print_method(PHASE_DEBUG, 0);
5892 }
5893 
5894 // Called from debugger, especially when replaying a trace in which the program state cannot be altered like with rr replay.
5895 // A method is appended to an existing default file with the default phase name. This means that igv_append() must follow
5896 // an earlier igv_print(*) call which sets up the file. This works regardless of any Ideal Graph Visualizer flags set or not.
5897 // Use in debugger (gdb/rr): p igv_append($sp, $fp, $pc).
5898 void igv_append(void* sp, void* fp, void* pc) {
5899   frame fr(sp, fp, pc);
5900   Compile::current()->igv_print_method_to_file(nullptr, true, &fr);
5901 }
5902 
5903 // Same as igv_append(...) above but with a specified phase name.
5904 // Use in debugger (gdb/rr): p igv_append("MyPhase", $sp, $fp, $pc).
5905 void igv_append(const char* phase_name, void* sp, void* fp, void* pc) {
5906   frame fr(sp, fp, pc);
5907   Compile::current()->igv_print_method_to_file(phase_name, true, &fr);
5908 }
5909 
5910 void Compile::igv_print_method_to_file(const char* phase_name, bool append, const frame* fr) {
5911   const char* file_name = "custom_debug.xml";
5912   if (_debug_file_printer == nullptr) {
5913     _debug_file_printer = new IdealGraphPrinter(C, file_name, append);
5914   } else {
5915     _debug_file_printer->update_compiled_method(C->method());
5916   }
5917   tty->print_cr("Method %s to %s", append ? "appended" : "printed", file_name);
5918   _debug_file_printer->print_graph(phase_name, fr);
5919 }
5920 
5921 void Compile::igv_print_method_to_network(const char* phase_name, const frame* fr) {
5922   ResourceMark rm;
5923   GrowableArray<const Node*> empty_list;
5924   igv_print_graph_to_network(phase_name, empty_list, fr);
5925 }
5926 
5927 void Compile::igv_print_graph_to_network(const char* name, GrowableArray<const Node*>& visible_nodes, const frame* fr) {
5928   if (_debug_network_printer == nullptr) {
5929     _debug_network_printer = new IdealGraphPrinter(C);
5930   } else {
5931     _debug_network_printer->update_compiled_method(C->method());
5932   }
5933   tty->print_cr("Method printed over network stream to IGV");
5934   _debug_network_printer->print(name, C->root(), visible_nodes, fr);
5935 }
5936 #endif // !PRODUCT
5937 
5938 Node* Compile::narrow_value(BasicType bt, Node* value, const Type* type, PhaseGVN* phase, bool transform_res) {
5939   if (type != nullptr && phase->type(value)->higher_equal(type)) {
5940     return value;
5941   }
5942   Node* result = nullptr;
5943   if (bt == T_BYTE) {
5944     result = phase->transform(new LShiftINode(value, phase->intcon(24)));
5945     result = new RShiftINode(result, phase->intcon(24));
5946   } else if (bt == T_BOOLEAN) {
5947     result = new AndINode(value, phase->intcon(0xFF));
5948   } else if (bt == T_CHAR) {
5949     result = new AndINode(value,phase->intcon(0xFFFF));
5950   } else if (bt == T_FLOAT) {
5951     result = new MoveI2FNode(value);
5952   } else {
5953     assert(bt == T_SHORT, "unexpected narrow type");
5954     result = phase->transform(new LShiftINode(value, phase->intcon(16)));
5955     result = new RShiftINode(result, phase->intcon(16));
5956   }
5957   if (transform_res) {
5958     result = phase->transform(result);
5959   }
5960   return result;
5961 }
5962 
5963 void Compile::record_method_not_compilable_oom() {
5964   record_method_not_compilable(CompilationMemoryStatistic::failure_reason_memlimit());
5965 }
5966 
5967 #ifndef PRODUCT
5968 // Collects all the control inputs from nodes on the worklist and from their data dependencies
5969 static void find_candidate_control_inputs(Unique_Node_List& worklist, Unique_Node_List& candidates) {
5970   // Follow non-control edges until we reach CFG nodes
5971   for (uint i = 0; i < worklist.size(); i++) {
5972     const Node* n = worklist.at(i);
5973     for (uint j = 0; j < n->req(); j++) {
5974       Node* in = n->in(j);
5975       if (in == nullptr || in->is_Root()) {
5976         continue;
5977       }
5978       if (in->is_CFG()) {
5979         if (in->is_Call()) {
5980           // The return value of a call is only available if the call did not result in an exception
5981           Node* control_proj_use = in->as_Call()->proj_out(TypeFunc::Control)->unique_out();
5982           if (control_proj_use->is_Catch()) {
5983             Node* fall_through = control_proj_use->as_Catch()->proj_out(CatchProjNode::fall_through_index);
5984             candidates.push(fall_through);
5985             continue;
5986           }
5987         }
5988 
5989         if (in->is_Multi()) {
5990           // We got here by following data inputs so we should only have one control use
5991           // (no IfNode, etc)
5992           assert(!n->is_MultiBranch(), "unexpected node type: %s", n->Name());
5993           candidates.push(in->as_Multi()->proj_out(TypeFunc::Control));
5994         } else {
5995           candidates.push(in);
5996         }
5997       } else {
5998         worklist.push(in);
5999       }
6000     }
6001   }
6002 }
6003 
6004 // Returns the candidate node that is a descendant to all the other candidates
6005 static Node* pick_control(Unique_Node_List& candidates) {
6006   Unique_Node_List worklist;
6007   worklist.copy(candidates);
6008 
6009   // Traverse backwards through the CFG
6010   for (uint i = 0; i < worklist.size(); i++) {
6011     const Node* n = worklist.at(i);
6012     if (n->is_Root()) {
6013       continue;
6014     }
6015     for (uint j = 0; j < n->req(); j++) {
6016       // Skip backedge of loops to avoid cycles
6017       if (n->is_Loop() && j == LoopNode::LoopBackControl) {
6018         continue;
6019       }
6020 
6021       Node* pred = n->in(j);
6022       if (pred != nullptr && pred != n && pred->is_CFG()) {
6023         worklist.push(pred);
6024         // if pred is an ancestor of n, then pred is an ancestor to at least one candidate
6025         candidates.remove(pred);
6026       }
6027     }
6028   }
6029 
6030   assert(candidates.size() == 1, "unexpected control flow");
6031   return candidates.at(0);
6032 }
6033 
6034 // Initialize a parameter input for a debug print call, using a placeholder for jlong and jdouble
6035 static void debug_print_init_parm(Node* call, Node* parm, Node* half, int* pos) {
6036   call->init_req((*pos)++, parm);
6037   const BasicType bt = parm->bottom_type()->basic_type();
6038   if (bt == T_LONG || bt == T_DOUBLE) {
6039     call->init_req((*pos)++, half);
6040   }
6041 }
6042 
6043 Node* Compile::make_debug_print_call(const char* str, address call_addr, PhaseGVN* gvn,
6044                               Node* parm0, Node* parm1,
6045                               Node* parm2, Node* parm3,
6046                               Node* parm4, Node* parm5,
6047                               Node* parm6) const {
6048   Node* str_node = gvn->transform(new ConPNode(TypeRawPtr::make(((address) str))));
6049   const TypeFunc* type = OptoRuntime::debug_print_Type(parm0, parm1, parm2, parm3, parm4, parm5, parm6);
6050   Node* call = new CallLeafNode(type, call_addr, "debug_print", TypeRawPtr::BOTTOM);
6051 
6052   // find the most suitable control input
6053   Unique_Node_List worklist, candidates;
6054   if (parm0 != nullptr) { worklist.push(parm0);
6055   if (parm1 != nullptr) { worklist.push(parm1);
6056   if (parm2 != nullptr) { worklist.push(parm2);
6057   if (parm3 != nullptr) { worklist.push(parm3);
6058   if (parm4 != nullptr) { worklist.push(parm4);
6059   if (parm5 != nullptr) { worklist.push(parm5);
6060   if (parm6 != nullptr) { worklist.push(parm6);
6061   /* close each nested if ===> */  } } } } } } }
6062   find_candidate_control_inputs(worklist, candidates);
6063   Node* control = nullptr;
6064   if (candidates.size() == 0) {
6065     control = C->start()->proj_out(TypeFunc::Control);
6066   } else {
6067     control = pick_control(candidates);
6068   }
6069 
6070   // find all the previous users of the control we picked
6071   GrowableArray<Node*> users_of_control;
6072   for (DUIterator_Fast kmax, i = control->fast_outs(kmax); i < kmax; i++) {
6073     Node* use = control->fast_out(i);
6074     if (use->is_CFG() && use != control) {
6075       users_of_control.push(use);
6076     }
6077   }
6078 
6079   // we do not actually care about IO and memory as it uses neither
6080   call->init_req(TypeFunc::Control,   control);
6081   call->init_req(TypeFunc::I_O,       top());
6082   call->init_req(TypeFunc::Memory,    top());
6083   call->init_req(TypeFunc::FramePtr,  C->start()->proj_out(TypeFunc::FramePtr));
6084   call->init_req(TypeFunc::ReturnAdr, top());
6085 
6086   int pos = TypeFunc::Parms;
6087   call->init_req(pos++, str_node);
6088   if (parm0 != nullptr) { debug_print_init_parm(call, parm0, top(), &pos);
6089   if (parm1 != nullptr) { debug_print_init_parm(call, parm1, top(), &pos);
6090   if (parm2 != nullptr) { debug_print_init_parm(call, parm2, top(), &pos);
6091   if (parm3 != nullptr) { debug_print_init_parm(call, parm3, top(), &pos);
6092   if (parm4 != nullptr) { debug_print_init_parm(call, parm4, top(), &pos);
6093   if (parm5 != nullptr) { debug_print_init_parm(call, parm5, top(), &pos);
6094   if (parm6 != nullptr) { debug_print_init_parm(call, parm6, top(), &pos);
6095   /* close each nested if ===> */  } } } } } } }
6096   assert(call->in(call->req()-1) != nullptr, "must initialize all parms");
6097 
6098   call = gvn->transform(call);
6099   Node* call_control_proj = gvn->transform(new ProjNode(call, TypeFunc::Control));
6100 
6101   // rewire previous users to have the new call as control instead
6102   PhaseIterGVN* igvn = gvn->is_IterGVN();
6103   for (int i = 0; i < users_of_control.length(); i++) {
6104     Node* use = users_of_control.at(i);
6105     for (uint j = 0; j < use->req(); j++) {
6106       if (use->in(j) == control) {
6107         if (igvn != nullptr) {
6108           igvn->replace_input_of(use, j, call_control_proj);
6109         } else {
6110           gvn->hash_delete(use);
6111           use->set_req(j, call_control_proj);
6112           gvn->hash_insert(use);
6113         }
6114       }
6115     }
6116   }
6117 
6118   return call;
6119 }
6120 #endif // !PRODUCT