1 /*
   2  * Copyright (c) 1997, 2025, Oracle and/or its affiliates. All rights reserved.
   3  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
   4  *
   5  * This code is free software; you can redistribute it and/or modify it
   6  * under the terms of the GNU General Public License version 2 only, as
   7  * published by the Free Software Foundation.
   8  *
   9  * This code is distributed in the hope that it will be useful, but WITHOUT
  10  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  11  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  12  * version 2 for more details (a copy is included in the LICENSE file that
  13  * accompanied this code).
  14  *
  15  * You should have received a copy of the GNU General Public License version
  16  * 2 along with this work; if not, write to the Free Software Foundation,
  17  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
  18  *
  19  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
  20  * or visit www.oracle.com if you need additional information or have any
  21  * questions.
  22  *
  23  */
  24 
  25 #include "asm/macroAssembler.hpp"
  26 #include "asm/macroAssembler.inline.hpp"
  27 #include "ci/ciReplay.hpp"
  28 #include "classfile/javaClasses.hpp"
  29 #include "code/exceptionHandlerTable.hpp"
  30 #include "code/nmethod.hpp"
  31 #include "compiler/compilationFailureInfo.hpp"
  32 #include "compiler/compilationMemoryStatistic.hpp"
  33 #include "compiler/compileBroker.hpp"
  34 #include "compiler/compileLog.hpp"
  35 #include "compiler/compilerOracle.hpp"
  36 #include "compiler/compiler_globals.hpp"
  37 #include "compiler/disassembler.hpp"
  38 #include "compiler/oopMap.hpp"
  39 #include "gc/shared/barrierSet.hpp"
  40 #include "gc/shared/c2/barrierSetC2.hpp"
  41 #include "jfr/jfrEvents.hpp"
  42 #include "jvm_io.h"
  43 #include "memory/allocation.hpp"
  44 #include "memory/arena.hpp"
  45 #include "memory/resourceArea.hpp"
  46 #include "opto/addnode.hpp"
  47 #include "opto/block.hpp"
  48 #include "opto/c2compiler.hpp"
  49 #include "opto/callGenerator.hpp"
  50 #include "opto/callnode.hpp"
  51 #include "opto/castnode.hpp"
  52 #include "opto/cfgnode.hpp"
  53 #include "opto/chaitin.hpp"
  54 #include "opto/compile.hpp"
  55 #include "opto/connode.hpp"
  56 #include "opto/convertnode.hpp"
  57 #include "opto/divnode.hpp"
  58 #include "opto/escape.hpp"
  59 #include "opto/idealGraphPrinter.hpp"
  60 #include "opto/locknode.hpp"
  61 #include "opto/loopnode.hpp"
  62 #include "opto/machnode.hpp"
  63 #include "opto/macro.hpp"
  64 #include "opto/matcher.hpp"
  65 #include "opto/mathexactnode.hpp"
  66 #include "opto/memnode.hpp"
  67 #include "opto/mulnode.hpp"
  68 #include "opto/narrowptrnode.hpp"
  69 #include "opto/node.hpp"
  70 #include "opto/opaquenode.hpp"
  71 #include "opto/opcodes.hpp"
  72 #include "opto/output.hpp"
  73 #include "opto/parse.hpp"
  74 #include "opto/phaseX.hpp"
  75 #include "opto/rootnode.hpp"
  76 #include "opto/runtime.hpp"
  77 #include "opto/stringopts.hpp"
  78 #include "opto/type.hpp"
  79 #include "opto/vector.hpp"
  80 #include "opto/vectornode.hpp"
  81 #include "runtime/globals_extension.hpp"
  82 #include "runtime/sharedRuntime.hpp"
  83 #include "runtime/signature.hpp"
  84 #include "runtime/stubRoutines.hpp"
  85 #include "runtime/timer.hpp"
  86 #include "utilities/align.hpp"
  87 #include "utilities/copy.hpp"
  88 #include "utilities/macros.hpp"
  89 #include "utilities/resourceHash.hpp"
  90 
  91 // -------------------- Compile::mach_constant_base_node -----------------------
  92 // Constant table base node singleton.
  93 MachConstantBaseNode* Compile::mach_constant_base_node() {
  94   if (_mach_constant_base_node == nullptr) {
  95     _mach_constant_base_node = new MachConstantBaseNode();
  96     _mach_constant_base_node->add_req(C->root());
  97   }
  98   return _mach_constant_base_node;
  99 }
 100 
 101 
 102 /// Support for intrinsics.
 103 
 104 // Return the index at which m must be inserted (or already exists).
 105 // The sort order is by the address of the ciMethod, with is_virtual as minor key.
 106 class IntrinsicDescPair {
 107  private:
 108   ciMethod* _m;
 109   bool _is_virtual;
 110  public:
 111   IntrinsicDescPair(ciMethod* m, bool is_virtual) : _m(m), _is_virtual(is_virtual) {}
 112   static int compare(IntrinsicDescPair* const& key, CallGenerator* const& elt) {
 113     ciMethod* m= elt->method();
 114     ciMethod* key_m = key->_m;
 115     if (key_m < m)      return -1;
 116     else if (key_m > m) return 1;
 117     else {
 118       bool is_virtual = elt->is_virtual();
 119       bool key_virtual = key->_is_virtual;
 120       if (key_virtual < is_virtual)      return -1;
 121       else if (key_virtual > is_virtual) return 1;
 122       else                               return 0;
 123     }
 124   }
 125 };
 126 int Compile::intrinsic_insertion_index(ciMethod* m, bool is_virtual, bool& found) {
 127 #ifdef ASSERT
 128   for (int i = 1; i < _intrinsics.length(); i++) {
 129     CallGenerator* cg1 = _intrinsics.at(i-1);
 130     CallGenerator* cg2 = _intrinsics.at(i);
 131     assert(cg1->method() != cg2->method()
 132            ? cg1->method()     < cg2->method()
 133            : cg1->is_virtual() < cg2->is_virtual(),
 134            "compiler intrinsics list must stay sorted");
 135   }
 136 #endif
 137   IntrinsicDescPair pair(m, is_virtual);
 138   return _intrinsics.find_sorted<IntrinsicDescPair*, IntrinsicDescPair::compare>(&pair, found);
 139 }
 140 
 141 void Compile::register_intrinsic(CallGenerator* cg) {
 142   bool found = false;
 143   int index = intrinsic_insertion_index(cg->method(), cg->is_virtual(), found);
 144   assert(!found, "registering twice");
 145   _intrinsics.insert_before(index, cg);
 146   assert(find_intrinsic(cg->method(), cg->is_virtual()) == cg, "registration worked");
 147 }
 148 
 149 CallGenerator* Compile::find_intrinsic(ciMethod* m, bool is_virtual) {
 150   assert(m->is_loaded(), "don't try this on unloaded methods");
 151   if (_intrinsics.length() > 0) {
 152     bool found = false;
 153     int index = intrinsic_insertion_index(m, is_virtual, found);
 154      if (found) {
 155       return _intrinsics.at(index);
 156     }
 157   }
 158   // Lazily create intrinsics for intrinsic IDs well-known in the runtime.
 159   if (m->intrinsic_id() != vmIntrinsics::_none &&
 160       m->intrinsic_id() <= vmIntrinsics::LAST_COMPILER_INLINE) {
 161     CallGenerator* cg = make_vm_intrinsic(m, is_virtual);
 162     if (cg != nullptr) {
 163       // Save it for next time:
 164       register_intrinsic(cg);
 165       return cg;
 166     } else {
 167       gather_intrinsic_statistics(m->intrinsic_id(), is_virtual, _intrinsic_disabled);
 168     }
 169   }
 170   return nullptr;
 171 }
 172 
 173 // Compile::make_vm_intrinsic is defined in library_call.cpp.
 174 
 175 #ifndef PRODUCT
 176 // statistics gathering...
 177 
 178 juint  Compile::_intrinsic_hist_count[vmIntrinsics::number_of_intrinsics()] = {0};
 179 jubyte Compile::_intrinsic_hist_flags[vmIntrinsics::number_of_intrinsics()] = {0};
 180 
 181 inline int as_int(vmIntrinsics::ID id) {
 182   return vmIntrinsics::as_int(id);
 183 }
 184 
 185 bool Compile::gather_intrinsic_statistics(vmIntrinsics::ID id, bool is_virtual, int flags) {
 186   assert(id > vmIntrinsics::_none && id < vmIntrinsics::ID_LIMIT, "oob");
 187   int oflags = _intrinsic_hist_flags[as_int(id)];
 188   assert(flags != 0, "what happened?");
 189   if (is_virtual) {
 190     flags |= _intrinsic_virtual;
 191   }
 192   bool changed = (flags != oflags);
 193   if ((flags & _intrinsic_worked) != 0) {
 194     juint count = (_intrinsic_hist_count[as_int(id)] += 1);
 195     if (count == 1) {
 196       changed = true;           // first time
 197     }
 198     // increment the overall count also:
 199     _intrinsic_hist_count[as_int(vmIntrinsics::_none)] += 1;
 200   }
 201   if (changed) {
 202     if (((oflags ^ flags) & _intrinsic_virtual) != 0) {
 203       // Something changed about the intrinsic's virtuality.
 204       if ((flags & _intrinsic_virtual) != 0) {
 205         // This is the first use of this intrinsic as a virtual call.
 206         if (oflags != 0) {
 207           // We already saw it as a non-virtual, so note both cases.
 208           flags |= _intrinsic_both;
 209         }
 210       } else if ((oflags & _intrinsic_both) == 0) {
 211         // This is the first use of this intrinsic as a non-virtual
 212         flags |= _intrinsic_both;
 213       }
 214     }
 215     _intrinsic_hist_flags[as_int(id)] = (jubyte) (oflags | flags);
 216   }
 217   // update the overall flags also:
 218   _intrinsic_hist_flags[as_int(vmIntrinsics::_none)] |= (jubyte) flags;
 219   return changed;
 220 }
 221 
 222 static char* format_flags(int flags, char* buf) {
 223   buf[0] = 0;
 224   if ((flags & Compile::_intrinsic_worked) != 0)    strcat(buf, ",worked");
 225   if ((flags & Compile::_intrinsic_failed) != 0)    strcat(buf, ",failed");
 226   if ((flags & Compile::_intrinsic_disabled) != 0)  strcat(buf, ",disabled");
 227   if ((flags & Compile::_intrinsic_virtual) != 0)   strcat(buf, ",virtual");
 228   if ((flags & Compile::_intrinsic_both) != 0)      strcat(buf, ",nonvirtual");
 229   if (buf[0] == 0)  strcat(buf, ",");
 230   assert(buf[0] == ',', "must be");
 231   return &buf[1];
 232 }
 233 
 234 void Compile::print_intrinsic_statistics() {
 235   char flagsbuf[100];
 236   ttyLocker ttyl;
 237   if (xtty != nullptr)  xtty->head("statistics type='intrinsic'");
 238   tty->print_cr("Compiler intrinsic usage:");
 239   juint total = _intrinsic_hist_count[as_int(vmIntrinsics::_none)];
 240   if (total == 0)  total = 1;  // avoid div0 in case of no successes
 241   #define PRINT_STAT_LINE(name, c, f) \
 242     tty->print_cr("  %4d (%4.1f%%) %s (%s)", (int)(c), ((c) * 100.0) / total, name, f);
 243   for (auto id : EnumRange<vmIntrinsicID>{}) {
 244     int   flags = _intrinsic_hist_flags[as_int(id)];
 245     juint count = _intrinsic_hist_count[as_int(id)];
 246     if ((flags | count) != 0) {
 247       PRINT_STAT_LINE(vmIntrinsics::name_at(id), count, format_flags(flags, flagsbuf));
 248     }
 249   }
 250   PRINT_STAT_LINE("total", total, format_flags(_intrinsic_hist_flags[as_int(vmIntrinsics::_none)], flagsbuf));
 251   if (xtty != nullptr)  xtty->tail("statistics");
 252 }
 253 
 254 void Compile::print_statistics() {
 255   { ttyLocker ttyl;
 256     if (xtty != nullptr)  xtty->head("statistics type='opto'");
 257     Parse::print_statistics();
 258     PhaseStringOpts::print_statistics();
 259     PhaseCCP::print_statistics();
 260     PhaseRegAlloc::print_statistics();
 261     PhaseOutput::print_statistics();
 262     PhasePeephole::print_statistics();
 263     PhaseIdealLoop::print_statistics();
 264     ConnectionGraph::print_statistics();
 265     PhaseMacroExpand::print_statistics();
 266     if (xtty != nullptr)  xtty->tail("statistics");
 267   }
 268   if (_intrinsic_hist_flags[as_int(vmIntrinsics::_none)] != 0) {
 269     // put this under its own <statistics> element.
 270     print_intrinsic_statistics();
 271   }
 272 }
 273 #endif //PRODUCT
 274 
 275 void Compile::gvn_replace_by(Node* n, Node* nn) {
 276   for (DUIterator_Last imin, i = n->last_outs(imin); i >= imin; ) {
 277     Node* use = n->last_out(i);
 278     bool is_in_table = initial_gvn()->hash_delete(use);
 279     uint uses_found = 0;
 280     for (uint j = 0; j < use->len(); j++) {
 281       if (use->in(j) == n) {
 282         if (j < use->req())
 283           use->set_req(j, nn);
 284         else
 285           use->set_prec(j, nn);
 286         uses_found++;
 287       }
 288     }
 289     if (is_in_table) {
 290       // reinsert into table
 291       initial_gvn()->hash_find_insert(use);
 292     }
 293     record_for_igvn(use);
 294     PhaseIterGVN::add_users_of_use_to_worklist(nn, use, *_igvn_worklist);
 295     i -= uses_found;    // we deleted 1 or more copies of this edge
 296   }
 297 }
 298 
 299 
 300 // Identify all nodes that are reachable from below, useful.
 301 // Use breadth-first pass that records state in a Unique_Node_List,
 302 // recursive traversal is slower.
 303 void Compile::identify_useful_nodes(Unique_Node_List &useful) {
 304   int estimated_worklist_size = live_nodes();
 305   useful.map( estimated_worklist_size, nullptr );  // preallocate space
 306 
 307   // Initialize worklist
 308   if (root() != nullptr)  { useful.push(root()); }
 309   // If 'top' is cached, declare it useful to preserve cached node
 310   if (cached_top_node())  { useful.push(cached_top_node()); }
 311 
 312   // Push all useful nodes onto the list, breadthfirst
 313   for( uint next = 0; next < useful.size(); ++next ) {
 314     assert( next < unique(), "Unique useful nodes < total nodes");
 315     Node *n  = useful.at(next);
 316     uint max = n->len();
 317     for( uint i = 0; i < max; ++i ) {
 318       Node *m = n->in(i);
 319       if (not_a_node(m))  continue;
 320       useful.push(m);
 321     }
 322   }
 323 }
 324 
 325 // Update dead_node_list with any missing dead nodes using useful
 326 // list. Consider all non-useful nodes to be useless i.e., dead nodes.
 327 void Compile::update_dead_node_list(Unique_Node_List &useful) {
 328   uint max_idx = unique();
 329   VectorSet& useful_node_set = useful.member_set();
 330 
 331   for (uint node_idx = 0; node_idx < max_idx; node_idx++) {
 332     // If node with index node_idx is not in useful set,
 333     // mark it as dead in dead node list.
 334     if (!useful_node_set.test(node_idx)) {
 335       record_dead_node(node_idx);
 336     }
 337   }
 338 }
 339 
 340 void Compile::remove_useless_late_inlines(GrowableArray<CallGenerator*>* inlines, Unique_Node_List &useful) {
 341   int shift = 0;
 342   for (int i = 0; i < inlines->length(); i++) {
 343     CallGenerator* cg = inlines->at(i);
 344     if (useful.member(cg->call_node())) {
 345       if (shift > 0) {
 346         inlines->at_put(i - shift, cg);
 347       }
 348     } else {
 349       shift++; // skip over the dead element
 350     }
 351   }
 352   if (shift > 0) {
 353     inlines->trunc_to(inlines->length() - shift); // remove last elements from compacted array
 354   }
 355 }
 356 
 357 void Compile::remove_useless_late_inlines(GrowableArray<CallGenerator*>* inlines, Node* dead) {
 358   assert(dead != nullptr && dead->is_Call(), "sanity");
 359   int found = 0;
 360   for (int i = 0; i < inlines->length(); i++) {
 361     if (inlines->at(i)->call_node() == dead) {
 362       inlines->remove_at(i);
 363       found++;
 364       NOT_DEBUG( break; ) // elements are unique, so exit early
 365     }
 366   }
 367   assert(found <= 1, "not unique");
 368 }
 369 
 370 template<typename N, ENABLE_IF_SDEFN(std::is_base_of<Node, N>::value)>
 371 void Compile::remove_useless_nodes(GrowableArray<N*>& node_list, Unique_Node_List& useful) {
 372   for (int i = node_list.length() - 1; i >= 0; i--) {
 373     N* node = node_list.at(i);
 374     if (!useful.member(node)) {
 375       node_list.delete_at(i); // replaces i-th with last element which is known to be useful (already processed)
 376     }
 377   }
 378 }
 379 
 380 void Compile::remove_useless_node(Node* dead) {
 381   remove_modified_node(dead);
 382 
 383   // Constant node that has no out-edges and has only one in-edge from
 384   // root is usually dead. However, sometimes reshaping walk makes
 385   // it reachable by adding use edges. So, we will NOT count Con nodes
 386   // as dead to be conservative about the dead node count at any
 387   // given time.
 388   if (!dead->is_Con()) {
 389     record_dead_node(dead->_idx);
 390   }
 391   if (dead->is_macro()) {
 392     remove_macro_node(dead);
 393   }
 394   if (dead->is_expensive()) {
 395     remove_expensive_node(dead);
 396   }
 397   if (dead->is_OpaqueTemplateAssertionPredicate()) {
 398     remove_template_assertion_predicate_opaque(dead->as_OpaqueTemplateAssertionPredicate());
 399   }
 400   if (dead->is_ParsePredicate()) {
 401     remove_parse_predicate(dead->as_ParsePredicate());
 402   }
 403   if (dead->for_post_loop_opts_igvn()) {
 404     remove_from_post_loop_opts_igvn(dead);
 405   }
 406   if (dead->for_merge_stores_igvn()) {
 407     remove_from_merge_stores_igvn(dead);
 408   }
 409   if (dead->is_Call()) {
 410     remove_useless_late_inlines(                &_late_inlines, dead);
 411     remove_useless_late_inlines(         &_string_late_inlines, dead);
 412     remove_useless_late_inlines(         &_boxing_late_inlines, dead);
 413     remove_useless_late_inlines(&_vector_reboxing_late_inlines, dead);
 414 
 415     if (dead->is_CallStaticJava()) {
 416       remove_unstable_if_trap(dead->as_CallStaticJava(), false);
 417     }
 418   }
 419   BarrierSetC2* bs = BarrierSet::barrier_set()->barrier_set_c2();
 420   bs->unregister_potential_barrier_node(dead);
 421 }
 422 
 423 // Disconnect all useless nodes by disconnecting those at the boundary.
 424 void Compile::disconnect_useless_nodes(Unique_Node_List& useful, Unique_Node_List& worklist, const Unique_Node_List* root_and_safepoints) {
 425   uint next = 0;
 426   while (next < useful.size()) {
 427     Node *n = useful.at(next++);
 428     if (n->is_SafePoint()) {
 429       // We're done with a parsing phase. Replaced nodes are not valid
 430       // beyond that point.
 431       n->as_SafePoint()->delete_replaced_nodes();
 432     }
 433     // Use raw traversal of out edges since this code removes out edges
 434     int max = n->outcnt();
 435     for (int j = 0; j < max; ++j) {
 436       Node* child = n->raw_out(j);
 437       if (!useful.member(child)) {
 438         assert(!child->is_top() || child != top(),
 439                "If top is cached in Compile object it is in useful list");
 440         // Only need to remove this out-edge to the useless node
 441         n->raw_del_out(j);
 442         --j;
 443         --max;
 444         if (child->is_data_proj_of_pure_function(n)) {
 445           worklist.push(n);
 446         }
 447       }
 448     }
 449     if (n->outcnt() == 1 && n->has_special_unique_user()) {
 450       assert(useful.member(n->unique_out()), "do not push a useless node");
 451       worklist.push(n->unique_out());
 452     }
 453   }
 454 
 455   remove_useless_nodes(_macro_nodes,        useful); // remove useless macro nodes
 456   remove_useless_nodes(_parse_predicates,   useful); // remove useless Parse Predicate nodes
 457   // Remove useless Template Assertion Predicate opaque nodes
 458   remove_useless_nodes(_template_assertion_predicate_opaques, useful);
 459   remove_useless_nodes(_expensive_nodes,    useful); // remove useless expensive nodes
 460   remove_useless_nodes(_for_post_loop_igvn, useful); // remove useless node recorded for post loop opts IGVN pass
 461   remove_useless_nodes(_for_merge_stores_igvn, useful); // remove useless node recorded for merge stores IGVN pass
 462   remove_useless_unstable_if_traps(useful);          // remove useless unstable_if traps
 463   remove_useless_coarsened_locks(useful);            // remove useless coarsened locks nodes
 464 #ifdef ASSERT
 465   if (_modified_nodes != nullptr) {
 466     _modified_nodes->remove_useless_nodes(useful.member_set());
 467   }
 468 #endif
 469 
 470   BarrierSetC2* bs = BarrierSet::barrier_set()->barrier_set_c2();
 471   bs->eliminate_useless_gc_barriers(useful, this);
 472   // clean up the late inline lists
 473   remove_useless_late_inlines(                &_late_inlines, useful);
 474   remove_useless_late_inlines(         &_string_late_inlines, useful);
 475   remove_useless_late_inlines(         &_boxing_late_inlines, useful);
 476   remove_useless_late_inlines(&_vector_reboxing_late_inlines, useful);
 477   DEBUG_ONLY(verify_graph_edges(true /*check for no_dead_code*/, root_and_safepoints);)
 478 }
 479 
 480 // ============================================================================
 481 //------------------------------CompileWrapper---------------------------------
 482 class CompileWrapper : public StackObj {
 483   Compile *const _compile;
 484  public:
 485   CompileWrapper(Compile* compile);
 486 
 487   ~CompileWrapper();
 488 };
 489 
 490 CompileWrapper::CompileWrapper(Compile* compile) : _compile(compile) {
 491   // the Compile* pointer is stored in the current ciEnv:
 492   ciEnv* env = compile->env();
 493   assert(env == ciEnv::current(), "must already be a ciEnv active");
 494   assert(env->compiler_data() == nullptr, "compile already active?");
 495   env->set_compiler_data(compile);
 496   assert(compile == Compile::current(), "sanity");
 497 
 498   compile->set_type_dict(nullptr);
 499   compile->set_clone_map(new Dict(cmpkey, hashkey, _compile->comp_arena()));
 500   compile->clone_map().set_clone_idx(0);
 501   compile->set_type_last_size(0);
 502   compile->set_last_tf(nullptr, nullptr);
 503   compile->set_indexSet_arena(nullptr);
 504   compile->set_indexSet_free_block_list(nullptr);
 505   compile->init_type_arena();
 506   Type::Initialize(compile);
 507   _compile->begin_method();
 508   _compile->clone_map().set_debug(_compile->has_method() && _compile->directive()->CloneMapDebugOption);
 509 }
 510 CompileWrapper::~CompileWrapper() {
 511   // simulate crash during compilation
 512   assert(CICrashAt < 0 || _compile->compile_id() != CICrashAt, "just as planned");
 513 
 514   _compile->end_method();
 515   _compile->env()->set_compiler_data(nullptr);
 516 }
 517 
 518 
 519 //----------------------------print_compile_messages---------------------------
 520 void Compile::print_compile_messages() {
 521 #ifndef PRODUCT
 522   // Check if recompiling
 523   if (!subsume_loads() && PrintOpto) {
 524     // Recompiling without allowing machine instructions to subsume loads
 525     tty->print_cr("*********************************************************");
 526     tty->print_cr("** Bailout: Recompile without subsuming loads          **");
 527     tty->print_cr("*********************************************************");
 528   }
 529   if ((do_escape_analysis() != DoEscapeAnalysis) && PrintOpto) {
 530     // Recompiling without escape analysis
 531     tty->print_cr("*********************************************************");
 532     tty->print_cr("** Bailout: Recompile without escape analysis          **");
 533     tty->print_cr("*********************************************************");
 534   }
 535   if (do_iterative_escape_analysis() != DoEscapeAnalysis && PrintOpto) {
 536     // Recompiling without iterative escape analysis
 537     tty->print_cr("*********************************************************");
 538     tty->print_cr("** Bailout: Recompile without iterative escape analysis**");
 539     tty->print_cr("*********************************************************");
 540   }
 541   if (do_reduce_allocation_merges() != ReduceAllocationMerges && PrintOpto) {
 542     // Recompiling without reducing allocation merges
 543     tty->print_cr("*********************************************************");
 544     tty->print_cr("** Bailout: Recompile without reduce allocation merges **");
 545     tty->print_cr("*********************************************************");
 546   }
 547   if ((eliminate_boxing() != EliminateAutoBox) && PrintOpto) {
 548     // Recompiling without boxing elimination
 549     tty->print_cr("*********************************************************");
 550     tty->print_cr("** Bailout: Recompile without boxing elimination       **");
 551     tty->print_cr("*********************************************************");
 552   }
 553   if ((do_locks_coarsening() != EliminateLocks) && PrintOpto) {
 554     // Recompiling without locks coarsening
 555     tty->print_cr("*********************************************************");
 556     tty->print_cr("** Bailout: Recompile without locks coarsening         **");
 557     tty->print_cr("*********************************************************");
 558   }
 559   if (env()->break_at_compile()) {
 560     // Open the debugger when compiling this method.
 561     tty->print("### Breaking when compiling: ");
 562     method()->print_short_name();
 563     tty->cr();
 564     BREAKPOINT;
 565   }
 566 
 567   if( PrintOpto ) {
 568     if (is_osr_compilation()) {
 569       tty->print("[OSR]%3d", _compile_id);
 570     } else {
 571       tty->print("%3d", _compile_id);
 572     }
 573   }
 574 #endif
 575 }
 576 
 577 #ifndef PRODUCT
 578 void Compile::print_ideal_ir(const char* phase_name) {
 579   // keep the following output all in one block
 580   // This output goes directly to the tty, not the compiler log.
 581   // To enable tools to match it up with the compilation activity,
 582   // be sure to tag this tty output with the compile ID.
 583 
 584   // Node dumping can cause a safepoint, which can break the tty lock.
 585   // Buffer all node dumps, so that all safepoints happen before we lock.
 586   ResourceMark rm;
 587   stringStream ss;
 588 
 589   if (_output == nullptr) {
 590     ss.print_cr("AFTER: %s", phase_name);
 591     // Print out all nodes in ascending order of index.
 592     root()->dump_bfs(MaxNodeLimit, nullptr, "+S$", &ss);
 593   } else {
 594     // Dump the node blockwise if we have a scheduling
 595     _output->print_scheduling(&ss);
 596   }
 597 
 598   // Check that the lock is not broken by a safepoint.
 599   NoSafepointVerifier nsv;
 600   ttyLocker ttyl;
 601   if (xtty != nullptr) {
 602     xtty->head("ideal compile_id='%d'%s compile_phase='%s'",
 603                compile_id(),
 604                is_osr_compilation() ? " compile_kind='osr'" : "",
 605                phase_name);
 606   }
 607 
 608   tty->print("%s", ss.as_string());
 609 
 610   if (xtty != nullptr) {
 611     xtty->tail("ideal");
 612   }
 613 }
 614 #endif
 615 
 616 // ============================================================================
 617 //------------------------------Compile standard-------------------------------
 618 
 619 // Compile a method.  entry_bci is -1 for normal compilations and indicates
 620 // the continuation bci for on stack replacement.
 621 
 622 
 623 Compile::Compile(ciEnv* ci_env, ciMethod* target, int osr_bci,
 624                  Options options, DirectiveSet* directive)
 625     : Phase(Compiler),
 626       _compile_id(ci_env->compile_id()),
 627       _options(options),
 628       _method(target),
 629       _entry_bci(osr_bci),
 630       _ilt(nullptr),
 631       _stub_function(nullptr),
 632       _stub_name(nullptr),
 633       _stub_entry_point(nullptr),
 634       _max_node_limit(MaxNodeLimit),
 635       _post_loop_opts_phase(false),
 636       _merge_stores_phase(false),
 637       _allow_macro_nodes(true),
 638       _inlining_progress(false),
 639       _inlining_incrementally(false),
 640       _do_cleanup(false),
 641       _has_reserved_stack_access(target->has_reserved_stack_access()),
 642 #ifndef PRODUCT
 643       _igv_idx(0),
 644       _trace_opto_output(directive->TraceOptoOutputOption),
 645 #endif
 646       _has_method_handle_invokes(false),
 647       _clinit_barrier_on_entry(false),
 648       _stress_seed(0),
 649       _comp_arena(mtCompiler, Arena::Tag::tag_comp),
 650       _barrier_set_state(BarrierSet::barrier_set()->barrier_set_c2()->create_barrier_state(comp_arena())),
 651       _env(ci_env),
 652       _directive(directive),
 653       _log(ci_env->log()),
 654       _first_failure_details(nullptr),
 655       _intrinsics(comp_arena(), 0, 0, nullptr),
 656       _macro_nodes(comp_arena(), 8, 0, nullptr),
 657       _parse_predicates(comp_arena(), 8, 0, nullptr),
 658       _template_assertion_predicate_opaques(comp_arena(), 8, 0, nullptr),
 659       _expensive_nodes(comp_arena(), 8, 0, nullptr),
 660       _for_post_loop_igvn(comp_arena(), 8, 0, nullptr),
 661       _for_merge_stores_igvn(comp_arena(), 8, 0, nullptr),
 662       _unstable_if_traps(comp_arena(), 8, 0, nullptr),
 663       _coarsened_locks(comp_arena(), 8, 0, nullptr),
 664       _congraph(nullptr),
 665       NOT_PRODUCT(_igv_printer(nullptr) COMMA)
 666           _unique(0),
 667       _dead_node_count(0),
 668       _dead_node_list(comp_arena()),
 669       _node_arena_one(mtCompiler, Arena::Tag::tag_node),
 670       _node_arena_two(mtCompiler, Arena::Tag::tag_node),
 671       _node_arena(&_node_arena_one),
 672       _mach_constant_base_node(nullptr),
 673       _Compile_types(mtCompiler, Arena::Tag::tag_type),
 674       _initial_gvn(nullptr),
 675       _igvn_worklist(nullptr),
 676       _types(nullptr),
 677       _node_hash(nullptr),
 678       _late_inlines(comp_arena(), 2, 0, nullptr),
 679       _string_late_inlines(comp_arena(), 2, 0, nullptr),
 680       _boxing_late_inlines(comp_arena(), 2, 0, nullptr),
 681       _vector_reboxing_late_inlines(comp_arena(), 2, 0, nullptr),
 682       _late_inlines_pos(0),
 683       _number_of_mh_late_inlines(0),
 684       _oom(false),
 685       _replay_inline_data(nullptr),
 686       _inline_printer(this),
 687       _java_calls(0),
 688       _inner_loops(0),
 689       _interpreter_frame_size(0),
 690       _output(nullptr)
 691 #ifndef PRODUCT
 692       ,
 693       _in_dump_cnt(0)
 694 #endif
 695 {
 696   C = this;
 697   CompileWrapper cw(this);
 698 
 699   TraceTime t1("Total compilation time", &_t_totalCompilation, CITime, CITimeVerbose);
 700   TraceTime t2(nullptr, &_t_methodCompilation, CITime, false);
 701 
 702 #if defined(SUPPORT_ASSEMBLY) || defined(SUPPORT_ABSTRACT_ASSEMBLY)
 703   bool print_opto_assembly = directive->PrintOptoAssemblyOption;
 704   // We can always print a disassembly, either abstract (hex dump) or
 705   // with the help of a suitable hsdis library. Thus, we should not
 706   // couple print_assembly and print_opto_assembly controls.
 707   // But: always print opto and regular assembly on compile command 'print'.
 708   bool print_assembly = directive->PrintAssemblyOption;
 709   set_print_assembly(print_opto_assembly || print_assembly);
 710 #else
 711   set_print_assembly(false); // must initialize.
 712 #endif
 713 
 714 #ifndef PRODUCT
 715   set_parsed_irreducible_loop(false);
 716 #endif
 717 
 718   if (directive->ReplayInlineOption) {
 719     _replay_inline_data = ciReplay::load_inline_data(method(), entry_bci(), ci_env->comp_level());
 720   }
 721   set_print_inlining(directive->PrintInliningOption || PrintOptoInlining);
 722   set_print_intrinsics(directive->PrintIntrinsicsOption);
 723   set_has_irreducible_loop(true); // conservative until build_loop_tree() reset it
 724 
 725   if (ProfileTraps) {
 726     // Make sure the method being compiled gets its own MDO,
 727     // so we can at least track the decompile_count().
 728     method()->ensure_method_data();
 729   }
 730 
 731   if (StressLCM || StressGCM || StressIGVN || StressCCP ||
 732       StressIncrementalInlining || StressMacroExpansion || StressUnstableIfTraps || StressBailout) {
 733     initialize_stress_seed(directive);
 734   }
 735 
 736   Init(/*do_aliasing=*/ true);
 737 
 738   print_compile_messages();
 739 
 740   _ilt = InlineTree::build_inline_tree_root();
 741 
 742   // Even if NO memory addresses are used, MergeMem nodes must have at least 1 slice
 743   assert(num_alias_types() >= AliasIdxRaw, "");
 744 
 745 #define MINIMUM_NODE_HASH  1023
 746 
 747   // GVN that will be run immediately on new nodes
 748   uint estimated_size = method()->code_size()*4+64;
 749   estimated_size = (estimated_size < MINIMUM_NODE_HASH ? MINIMUM_NODE_HASH : estimated_size);
 750   _igvn_worklist = new (comp_arena()) Unique_Node_List(comp_arena());
 751   _types = new (comp_arena()) Type_Array(comp_arena());
 752   _node_hash = new (comp_arena()) NodeHash(comp_arena(), estimated_size);
 753   PhaseGVN gvn;
 754   set_initial_gvn(&gvn);
 755 
 756   { // Scope for timing the parser
 757     TracePhase tp(_t_parser);
 758 
 759     // Put top into the hash table ASAP.
 760     initial_gvn()->transform(top());
 761 
 762     // Set up tf(), start(), and find a CallGenerator.
 763     CallGenerator* cg = nullptr;
 764     if (is_osr_compilation()) {
 765       const TypeTuple *domain = StartOSRNode::osr_domain();
 766       const TypeTuple *range = TypeTuple::make_range(method()->signature());
 767       init_tf(TypeFunc::make(domain, range));
 768       StartNode* s = new StartOSRNode(root(), domain);
 769       initial_gvn()->set_type_bottom(s);
 770       verify_start(s);
 771       cg = CallGenerator::for_osr(method(), entry_bci());
 772     } else {
 773       // Normal case.
 774       init_tf(TypeFunc::make(method()));
 775       StartNode* s = new StartNode(root(), tf()->domain());
 776       initial_gvn()->set_type_bottom(s);
 777       verify_start(s);
 778       if (method()->intrinsic_id() == vmIntrinsics::_Reference_get) {
 779         // With java.lang.ref.reference.get() we must go through the
 780         // intrinsic - even when get() is the root
 781         // method of the compile - so that, if necessary, the value in
 782         // the referent field of the reference object gets recorded by
 783         // the pre-barrier code.
 784         cg = find_intrinsic(method(), false);
 785       }
 786       if (cg == nullptr) {
 787         float past_uses = method()->interpreter_invocation_count();
 788         float expected_uses = past_uses;
 789         cg = CallGenerator::for_inline(method(), expected_uses);
 790       }
 791     }
 792     if (failing())  return;
 793     if (cg == nullptr) {
 794       const char* reason = InlineTree::check_can_parse(method());
 795       assert(reason != nullptr, "expect reason for parse failure");
 796       stringStream ss;
 797       ss.print("cannot parse method: %s", reason);
 798       record_method_not_compilable(ss.as_string());
 799       return;
 800     }
 801 
 802     gvn.set_type(root(), root()->bottom_type());
 803 
 804     JVMState* jvms = build_start_state(start(), tf());
 805     if ((jvms = cg->generate(jvms)) == nullptr) {
 806       assert(failure_reason() != nullptr, "expect reason for parse failure");
 807       stringStream ss;
 808       ss.print("method parse failed: %s", failure_reason());
 809       record_method_not_compilable(ss.as_string() DEBUG_ONLY(COMMA true));
 810       return;
 811     }
 812     GraphKit kit(jvms);
 813 
 814     if (!kit.stopped()) {
 815       // Accept return values, and transfer control we know not where.
 816       // This is done by a special, unique ReturnNode bound to root.
 817       return_values(kit.jvms());
 818     }
 819 
 820     if (kit.has_exceptions()) {
 821       // Any exceptions that escape from this call must be rethrown
 822       // to whatever caller is dynamically above us on the stack.
 823       // This is done by a special, unique RethrowNode bound to root.
 824       rethrow_exceptions(kit.transfer_exceptions_into_jvms());
 825     }
 826 
 827     assert(IncrementalInline || (_late_inlines.length() == 0 && !has_mh_late_inlines()), "incremental inlining is off");
 828 
 829     if (_late_inlines.length() == 0 && !has_mh_late_inlines() && !failing() && has_stringbuilder()) {
 830       inline_string_calls(true);
 831     }
 832 
 833     if (failing())  return;
 834 
 835     // Remove clutter produced by parsing.
 836     if (!failing()) {
 837       ResourceMark rm;
 838       PhaseRemoveUseless pru(initial_gvn(), *igvn_worklist());
 839     }
 840   }
 841 
 842   // Note:  Large methods are capped off in do_one_bytecode().
 843   if (failing())  return;
 844 
 845   // After parsing, node notes are no longer automagic.
 846   // They must be propagated by register_new_node_with_optimizer(),
 847   // clone(), or the like.
 848   set_default_node_notes(nullptr);
 849 
 850 #ifndef PRODUCT
 851   if (should_print_igv(1)) {
 852     _igv_printer->print_inlining();
 853   }
 854 #endif
 855 
 856   if (failing())  return;
 857   NOT_PRODUCT( verify_graph_edges(); )
 858 
 859   // Now optimize
 860   Optimize();
 861   if (failing())  return;
 862   NOT_PRODUCT( verify_graph_edges(); )
 863 
 864 #ifndef PRODUCT
 865   if (should_print_ideal()) {
 866     print_ideal_ir("print_ideal");
 867   }
 868 #endif
 869 
 870 #ifdef ASSERT
 871   BarrierSetC2* bs = BarrierSet::barrier_set()->barrier_set_c2();
 872   bs->verify_gc_barriers(this, BarrierSetC2::BeforeCodeGen);
 873 #endif
 874 
 875   // Dump compilation data to replay it.
 876   if (directive->DumpReplayOption) {
 877     env()->dump_replay_data(_compile_id);
 878   }
 879   if (directive->DumpInlineOption && (ilt() != nullptr)) {
 880     env()->dump_inline_data(_compile_id);
 881   }
 882 
 883   // Now that we know the size of all the monitors we can add a fixed slot
 884   // for the original deopt pc.
 885   int next_slot = fixed_slots() + (sizeof(address) / VMRegImpl::stack_slot_size);
 886   set_fixed_slots(next_slot);
 887 
 888   // Compute when to use implicit null checks. Used by matching trap based
 889   // nodes and NullCheck optimization.
 890   set_allowed_deopt_reasons();
 891 
 892   // Now generate code
 893   Code_Gen();
 894 }
 895 
 896 //------------------------------Compile----------------------------------------
 897 // Compile a runtime stub
 898 Compile::Compile(ciEnv* ci_env,
 899                  TypeFunc_generator generator,
 900                  address stub_function,
 901                  const char* stub_name,
 902                  int is_fancy_jump,
 903                  bool pass_tls,
 904                  bool return_pc,
 905                  DirectiveSet* directive)
 906     : Phase(Compiler),
 907       _compile_id(0),
 908       _options(Options::for_runtime_stub()),
 909       _method(nullptr),
 910       _entry_bci(InvocationEntryBci),
 911       _stub_function(stub_function),
 912       _stub_name(stub_name),
 913       _stub_entry_point(nullptr),
 914       _max_node_limit(MaxNodeLimit),
 915       _post_loop_opts_phase(false),
 916       _merge_stores_phase(false),
 917       _allow_macro_nodes(true),
 918       _inlining_progress(false),
 919       _inlining_incrementally(false),
 920       _has_reserved_stack_access(false),
 921 #ifndef PRODUCT
 922       _igv_idx(0),
 923       _trace_opto_output(directive->TraceOptoOutputOption),
 924 #endif
 925       _has_method_handle_invokes(false),
 926       _clinit_barrier_on_entry(false),
 927       _stress_seed(0),
 928       _comp_arena(mtCompiler, Arena::Tag::tag_comp),
 929       _barrier_set_state(BarrierSet::barrier_set()->barrier_set_c2()->create_barrier_state(comp_arena())),
 930       _env(ci_env),
 931       _directive(directive),
 932       _log(ci_env->log()),
 933       _first_failure_details(nullptr),
 934       _for_post_loop_igvn(comp_arena(), 8, 0, nullptr),
 935       _for_merge_stores_igvn(comp_arena(), 8, 0, nullptr),
 936       _congraph(nullptr),
 937       NOT_PRODUCT(_igv_printer(nullptr) COMMA)
 938           _unique(0),
 939       _dead_node_count(0),
 940       _dead_node_list(comp_arena()),
 941       _node_arena_one(mtCompiler, Arena::Tag::tag_node),
 942       _node_arena_two(mtCompiler, Arena::Tag::tag_node),
 943       _node_arena(&_node_arena_one),
 944       _mach_constant_base_node(nullptr),
 945       _Compile_types(mtCompiler, Arena::Tag::tag_type),
 946       _initial_gvn(nullptr),
 947       _igvn_worklist(nullptr),
 948       _types(nullptr),
 949       _node_hash(nullptr),
 950       _number_of_mh_late_inlines(0),
 951       _oom(false),
 952       _replay_inline_data(nullptr),
 953       _inline_printer(this),
 954       _java_calls(0),
 955       _inner_loops(0),
 956       _interpreter_frame_size(0),
 957       _output(nullptr),
 958 #ifndef PRODUCT
 959       _in_dump_cnt(0),
 960 #endif
 961       _allowed_reasons(0) {
 962   C = this;
 963 
 964   TraceTime t1(nullptr, &_t_totalCompilation, CITime, false);
 965   TraceTime t2(nullptr, &_t_stubCompilation, CITime, false);
 966 
 967 #ifndef PRODUCT
 968   set_print_assembly(PrintFrameConverterAssembly);
 969   set_parsed_irreducible_loop(false);
 970 #else
 971   set_print_assembly(false); // Must initialize.
 972 #endif
 973   set_has_irreducible_loop(false); // no loops
 974 
 975   CompileWrapper cw(this);
 976   Init(/*do_aliasing=*/ false);
 977   init_tf((*generator)());
 978 
 979   _igvn_worklist = new (comp_arena()) Unique_Node_List(comp_arena());
 980   _types = new (comp_arena()) Type_Array(comp_arena());
 981   _node_hash = new (comp_arena()) NodeHash(comp_arena(), 255);
 982 
 983   if (StressLCM || StressGCM || StressBailout) {
 984     initialize_stress_seed(directive);
 985   }
 986 
 987   {
 988     PhaseGVN gvn;
 989     set_initial_gvn(&gvn);    // not significant, but GraphKit guys use it pervasively
 990     gvn.transform(top());
 991 
 992     GraphKit kit;
 993     kit.gen_stub(stub_function, stub_name, is_fancy_jump, pass_tls, return_pc);
 994   }
 995 
 996   NOT_PRODUCT( verify_graph_edges(); )
 997 
 998   Code_Gen();
 999 }
1000 
1001 Compile::~Compile() {
1002   delete _first_failure_details;
1003 };
1004 
1005 //------------------------------Init-------------------------------------------
1006 // Prepare for a single compilation
1007 void Compile::Init(bool aliasing) {
1008   _do_aliasing = aliasing;
1009   _unique  = 0;
1010   _regalloc = nullptr;
1011 
1012   _tf      = nullptr;  // filled in later
1013   _top     = nullptr;  // cached later
1014   _matcher = nullptr;  // filled in later
1015   _cfg     = nullptr;  // filled in later
1016 
1017   _node_note_array = nullptr;
1018   _default_node_notes = nullptr;
1019   DEBUG_ONLY( _modified_nodes = nullptr; ) // Used in Optimize()
1020 
1021   _immutable_memory = nullptr; // filled in at first inquiry
1022 
1023 #ifdef ASSERT
1024   _phase_optimize_finished = false;
1025   _phase_verify_ideal_loop = false;
1026   _exception_backedge = false;
1027   _type_verify = nullptr;
1028 #endif
1029 
1030   // Globally visible Nodes
1031   // First set TOP to null to give safe behavior during creation of RootNode
1032   set_cached_top_node(nullptr);
1033   set_root(new RootNode());
1034   // Now that you have a Root to point to, create the real TOP
1035   set_cached_top_node( new ConNode(Type::TOP) );
1036   set_recent_alloc(nullptr, nullptr);
1037 
1038   // Create Debug Information Recorder to record scopes, oopmaps, etc.
1039   env()->set_oop_recorder(new OopRecorder(env()->arena()));
1040   env()->set_debug_info(new DebugInformationRecorder(env()->oop_recorder()));
1041   env()->set_dependencies(new Dependencies(env()));
1042 
1043   _fixed_slots = 0;
1044   set_has_split_ifs(false);
1045   set_has_loops(false); // first approximation
1046   set_has_stringbuilder(false);
1047   set_has_boxed_value(false);
1048   _trap_can_recompile = false;  // no traps emitted yet
1049   _major_progress = true; // start out assuming good things will happen
1050   set_has_unsafe_access(false);
1051   set_max_vector_size(0);
1052   set_clear_upper_avx(false);  //false as default for clear upper bits of ymm registers
1053   Copy::zero_to_bytes(_trap_hist, sizeof(_trap_hist));
1054   set_decompile_count(0);
1055 
1056 #ifndef PRODUCT
1057   Copy::zero_to_bytes(_igv_phase_iter, sizeof(_igv_phase_iter));
1058 #endif
1059 
1060   set_do_freq_based_layout(_directive->BlockLayoutByFrequencyOption);
1061   _loop_opts_cnt = LoopOptsCount;
1062   set_do_inlining(Inline);
1063   set_max_inline_size(MaxInlineSize);
1064   set_freq_inline_size(FreqInlineSize);
1065   set_do_scheduling(OptoScheduling);
1066 
1067   set_do_vector_loop(false);
1068   set_has_monitors(false);
1069   set_has_scoped_access(false);
1070 
1071   if (AllowVectorizeOnDemand) {
1072     if (has_method() && _directive->VectorizeOption) {
1073       set_do_vector_loop(true);
1074       NOT_PRODUCT(if (do_vector_loop() && Verbose) {tty->print("Compile::Init: do vectorized loops (SIMD like) for method %s\n",  method()->name()->as_quoted_ascii());})
1075     } else if (has_method() && method()->name() != nullptr &&
1076                method()->intrinsic_id() == vmIntrinsics::_forEachRemaining) {
1077       set_do_vector_loop(true);
1078     }
1079   }
1080   set_use_cmove(UseCMoveUnconditionally /* || do_vector_loop()*/); //TODO: consider do_vector_loop() mandate use_cmove unconditionally
1081   NOT_PRODUCT(if (use_cmove() && Verbose && has_method()) {tty->print("Compile::Init: use CMove without profitability tests for method %s\n",  method()->name()->as_quoted_ascii());})
1082 
1083   _max_node_limit = _directive->MaxNodeLimitOption;
1084 
1085   if (VM_Version::supports_fast_class_init_checks() && has_method() && !is_osr_compilation() && method()->needs_clinit_barrier()) {
1086     set_clinit_barrier_on_entry(true);
1087   }
1088   if (debug_info()->recording_non_safepoints()) {
1089     set_node_note_array(new(comp_arena()) GrowableArray<Node_Notes*>
1090                         (comp_arena(), 8, 0, nullptr));
1091     set_default_node_notes(Node_Notes::make(this));
1092   }
1093 
1094   const int grow_ats = 16;
1095   _max_alias_types = grow_ats;
1096   _alias_types   = NEW_ARENA_ARRAY(comp_arena(), AliasType*, grow_ats);
1097   AliasType* ats = NEW_ARENA_ARRAY(comp_arena(), AliasType,  grow_ats);
1098   Copy::zero_to_bytes(ats, sizeof(AliasType)*grow_ats);
1099   {
1100     for (int i = 0; i < grow_ats; i++)  _alias_types[i] = &ats[i];
1101   }
1102   // Initialize the first few types.
1103   _alias_types[AliasIdxTop]->Init(AliasIdxTop, nullptr);
1104   _alias_types[AliasIdxBot]->Init(AliasIdxBot, TypePtr::BOTTOM);
1105   _alias_types[AliasIdxRaw]->Init(AliasIdxRaw, TypeRawPtr::BOTTOM);
1106   _num_alias_types = AliasIdxRaw+1;
1107   // Zero out the alias type cache.
1108   Copy::zero_to_bytes(_alias_cache, sizeof(_alias_cache));
1109   // A null adr_type hits in the cache right away.  Preload the right answer.
1110   probe_alias_cache(nullptr)->_index = AliasIdxTop;
1111 }
1112 
1113 #ifdef ASSERT
1114 // Verify that the current StartNode is valid.
1115 void Compile::verify_start(StartNode* s) const {
1116   assert(failing_internal() || s == start(), "should be StartNode");
1117 }
1118 #endif
1119 
1120 /**
1121  * Return the 'StartNode'. We must not have a pending failure, since the ideal graph
1122  * can be in an inconsistent state, i.e., we can get segmentation faults when traversing
1123  * the ideal graph.
1124  */
1125 StartNode* Compile::start() const {
1126   assert (!failing_internal() || C->failure_is_artificial(), "Must not have pending failure. Reason is: %s", failure_reason());
1127   for (DUIterator_Fast imax, i = root()->fast_outs(imax); i < imax; i++) {
1128     Node* start = root()->fast_out(i);
1129     if (start->is_Start()) {
1130       return start->as_Start();
1131     }
1132   }
1133   fatal("Did not find Start node!");
1134   return nullptr;
1135 }
1136 
1137 //-------------------------------immutable_memory-------------------------------------
1138 // Access immutable memory
1139 Node* Compile::immutable_memory() {
1140   if (_immutable_memory != nullptr) {
1141     return _immutable_memory;
1142   }
1143   StartNode* s = start();
1144   for (DUIterator_Fast imax, i = s->fast_outs(imax); true; i++) {
1145     Node *p = s->fast_out(i);
1146     if (p != s && p->as_Proj()->_con == TypeFunc::Memory) {
1147       _immutable_memory = p;
1148       return _immutable_memory;
1149     }
1150   }
1151   ShouldNotReachHere();
1152   return nullptr;
1153 }
1154 
1155 //----------------------set_cached_top_node------------------------------------
1156 // Install the cached top node, and make sure Node::is_top works correctly.
1157 void Compile::set_cached_top_node(Node* tn) {
1158   if (tn != nullptr)  verify_top(tn);
1159   Node* old_top = _top;
1160   _top = tn;
1161   // Calling Node::setup_is_top allows the nodes the chance to adjust
1162   // their _out arrays.
1163   if (_top != nullptr)     _top->setup_is_top();
1164   if (old_top != nullptr)  old_top->setup_is_top();
1165   assert(_top == nullptr || top()->is_top(), "");
1166 }
1167 
1168 #ifdef ASSERT
1169 uint Compile::count_live_nodes_by_graph_walk() {
1170   Unique_Node_List useful(comp_arena());
1171   // Get useful node list by walking the graph.
1172   identify_useful_nodes(useful);
1173   return useful.size();
1174 }
1175 
1176 void Compile::print_missing_nodes() {
1177 
1178   // Return if CompileLog is null and PrintIdealNodeCount is false.
1179   if ((_log == nullptr) && (! PrintIdealNodeCount)) {
1180     return;
1181   }
1182 
1183   // This is an expensive function. It is executed only when the user
1184   // specifies VerifyIdealNodeCount option or otherwise knows the
1185   // additional work that needs to be done to identify reachable nodes
1186   // by walking the flow graph and find the missing ones using
1187   // _dead_node_list.
1188 
1189   Unique_Node_List useful(comp_arena());
1190   // Get useful node list by walking the graph.
1191   identify_useful_nodes(useful);
1192 
1193   uint l_nodes = C->live_nodes();
1194   uint l_nodes_by_walk = useful.size();
1195 
1196   if (l_nodes != l_nodes_by_walk) {
1197     if (_log != nullptr) {
1198       _log->begin_head("mismatched_nodes count='%d'", abs((int) (l_nodes - l_nodes_by_walk)));
1199       _log->stamp();
1200       _log->end_head();
1201     }
1202     VectorSet& useful_member_set = useful.member_set();
1203     int last_idx = l_nodes_by_walk;
1204     for (int i = 0; i < last_idx; i++) {
1205       if (useful_member_set.test(i)) {
1206         if (_dead_node_list.test(i)) {
1207           if (_log != nullptr) {
1208             _log->elem("mismatched_node_info node_idx='%d' type='both live and dead'", i);
1209           }
1210           if (PrintIdealNodeCount) {
1211             // Print the log message to tty
1212               tty->print_cr("mismatched_node idx='%d' both live and dead'", i);
1213               useful.at(i)->dump();
1214           }
1215         }
1216       }
1217       else if (! _dead_node_list.test(i)) {
1218         if (_log != nullptr) {
1219           _log->elem("mismatched_node_info node_idx='%d' type='neither live nor dead'", i);
1220         }
1221         if (PrintIdealNodeCount) {
1222           // Print the log message to tty
1223           tty->print_cr("mismatched_node idx='%d' type='neither live nor dead'", i);
1224         }
1225       }
1226     }
1227     if (_log != nullptr) {
1228       _log->tail("mismatched_nodes");
1229     }
1230   }
1231 }
1232 void Compile::record_modified_node(Node* n) {
1233   if (_modified_nodes != nullptr && !_inlining_incrementally && !n->is_Con()) {
1234     _modified_nodes->push(n);
1235   }
1236 }
1237 
1238 void Compile::remove_modified_node(Node* n) {
1239   if (_modified_nodes != nullptr) {
1240     _modified_nodes->remove(n);
1241   }
1242 }
1243 #endif
1244 
1245 #ifndef PRODUCT
1246 void Compile::verify_top(Node* tn) const {
1247   if (tn != nullptr) {
1248     assert(tn->is_Con(), "top node must be a constant");
1249     assert(((ConNode*)tn)->type() == Type::TOP, "top node must have correct type");
1250     assert(tn->in(0) != nullptr, "must have live top node");
1251   }
1252 }
1253 #endif
1254 
1255 
1256 ///-------------------Managing Per-Node Debug & Profile Info-------------------
1257 
1258 void Compile::grow_node_notes(GrowableArray<Node_Notes*>* arr, int grow_by) {
1259   guarantee(arr != nullptr, "");
1260   int num_blocks = arr->length();
1261   if (grow_by < num_blocks)  grow_by = num_blocks;
1262   int num_notes = grow_by * _node_notes_block_size;
1263   Node_Notes* notes = NEW_ARENA_ARRAY(node_arena(), Node_Notes, num_notes);
1264   Copy::zero_to_bytes(notes, num_notes * sizeof(Node_Notes));
1265   while (num_notes > 0) {
1266     arr->append(notes);
1267     notes     += _node_notes_block_size;
1268     num_notes -= _node_notes_block_size;
1269   }
1270   assert(num_notes == 0, "exact multiple, please");
1271 }
1272 
1273 bool Compile::copy_node_notes_to(Node* dest, Node* source) {
1274   if (source == nullptr || dest == nullptr)  return false;
1275 
1276   if (dest->is_Con())
1277     return false;               // Do not push debug info onto constants.
1278 
1279 #ifdef ASSERT
1280   // Leave a bread crumb trail pointing to the original node:
1281   if (dest != nullptr && dest != source && dest->debug_orig() == nullptr) {
1282     dest->set_debug_orig(source);
1283   }
1284 #endif
1285 
1286   if (node_note_array() == nullptr)
1287     return false;               // Not collecting any notes now.
1288 
1289   // This is a copy onto a pre-existing node, which may already have notes.
1290   // If both nodes have notes, do not overwrite any pre-existing notes.
1291   Node_Notes* source_notes = node_notes_at(source->_idx);
1292   if (source_notes == nullptr || source_notes->is_clear())  return false;
1293   Node_Notes* dest_notes   = node_notes_at(dest->_idx);
1294   if (dest_notes == nullptr || dest_notes->is_clear()) {
1295     return set_node_notes_at(dest->_idx, source_notes);
1296   }
1297 
1298   Node_Notes merged_notes = (*source_notes);
1299   // The order of operations here ensures that dest notes will win...
1300   merged_notes.update_from(dest_notes);
1301   return set_node_notes_at(dest->_idx, &merged_notes);
1302 }
1303 
1304 
1305 //--------------------------allow_range_check_smearing-------------------------
1306 // Gating condition for coalescing similar range checks.
1307 // Sometimes we try 'speculatively' replacing a series of a range checks by a
1308 // single covering check that is at least as strong as any of them.
1309 // If the optimization succeeds, the simplified (strengthened) range check
1310 // will always succeed.  If it fails, we will deopt, and then give up
1311 // on the optimization.
1312 bool Compile::allow_range_check_smearing() const {
1313   // If this method has already thrown a range-check,
1314   // assume it was because we already tried range smearing
1315   // and it failed.
1316   uint already_trapped = trap_count(Deoptimization::Reason_range_check);
1317   return !already_trapped;
1318 }
1319 
1320 
1321 //------------------------------flatten_alias_type-----------------------------
1322 const TypePtr *Compile::flatten_alias_type( const TypePtr *tj ) const {
1323   assert(do_aliasing(), "Aliasing should be enabled");
1324   int offset = tj->offset();
1325   TypePtr::PTR ptr = tj->ptr();
1326 
1327   // Known instance (scalarizable allocation) alias only with itself.
1328   bool is_known_inst = tj->isa_oopptr() != nullptr &&
1329                        tj->is_oopptr()->is_known_instance();
1330 
1331   // Process weird unsafe references.
1332   if (offset == Type::OffsetBot && (tj->isa_instptr() /*|| tj->isa_klassptr()*/)) {
1333     assert(InlineUnsafeOps || StressReflectiveCode, "indeterminate pointers come only from unsafe ops");
1334     assert(!is_known_inst, "scalarizable allocation should not have unsafe references");
1335     tj = TypeOopPtr::BOTTOM;
1336     ptr = tj->ptr();
1337     offset = tj->offset();
1338   }
1339 
1340   // Array pointers need some flattening
1341   const TypeAryPtr* ta = tj->isa_aryptr();
1342   if (ta && ta->is_stable()) {
1343     // Erase stability property for alias analysis.
1344     tj = ta = ta->cast_to_stable(false);
1345   }
1346   if( ta && is_known_inst ) {
1347     if ( offset != Type::OffsetBot &&
1348          offset > arrayOopDesc::length_offset_in_bytes() ) {
1349       offset = Type::OffsetBot; // Flatten constant access into array body only
1350       tj = ta = ta->
1351               remove_speculative()->
1352               cast_to_ptr_type(ptr)->
1353               with_offset(offset);
1354     }
1355   } else if (ta) {
1356     // For arrays indexed by constant indices, we flatten the alias
1357     // space to include all of the array body.  Only the header, klass
1358     // and array length can be accessed un-aliased.
1359     if( offset != Type::OffsetBot ) {
1360       if( ta->const_oop() ) { // MethodData* or Method*
1361         offset = Type::OffsetBot;   // Flatten constant access into array body
1362         tj = ta = ta->
1363                 remove_speculative()->
1364                 cast_to_ptr_type(ptr)->
1365                 cast_to_exactness(false)->
1366                 with_offset(offset);
1367       } else if( offset == arrayOopDesc::length_offset_in_bytes() ) {
1368         // range is OK as-is.
1369         tj = ta = TypeAryPtr::RANGE;
1370       } else if( offset == Type::klass_offset() ) {
1371         tj = TypeInstPtr::KLASS; // all klass loads look alike
1372         ta = TypeAryPtr::RANGE; // generic ignored junk
1373         ptr = TypePtr::BotPTR;
1374       } else if( offset == oopDesc::mark_offset_in_bytes() ) {
1375         tj = TypeInstPtr::MARK;
1376         ta = TypeAryPtr::RANGE; // generic ignored junk
1377         ptr = TypePtr::BotPTR;
1378       } else {                  // Random constant offset into array body
1379         offset = Type::OffsetBot;   // Flatten constant access into array body
1380         tj = ta = ta->
1381                 remove_speculative()->
1382                 cast_to_ptr_type(ptr)->
1383                 cast_to_exactness(false)->
1384                 with_offset(offset);
1385       }
1386     }
1387     // Arrays of fixed size alias with arrays of unknown size.
1388     if (ta->size() != TypeInt::POS) {
1389       const TypeAry *tary = TypeAry::make(ta->elem(), TypeInt::POS);
1390       tj = ta = ta->
1391               remove_speculative()->
1392               cast_to_ptr_type(ptr)->
1393               with_ary(tary)->
1394               cast_to_exactness(false);
1395     }
1396     // Arrays of known objects become arrays of unknown objects.
1397     if (ta->elem()->isa_narrowoop() && ta->elem() != TypeNarrowOop::BOTTOM) {
1398       const TypeAry *tary = TypeAry::make(TypeNarrowOop::BOTTOM, ta->size());
1399       tj = ta = TypeAryPtr::make(ptr,ta->const_oop(),tary,nullptr,false,offset);
1400     }
1401     if (ta->elem()->isa_oopptr() && ta->elem() != TypeInstPtr::BOTTOM) {
1402       const TypeAry *tary = TypeAry::make(TypeInstPtr::BOTTOM, ta->size());
1403       tj = ta = TypeAryPtr::make(ptr,ta->const_oop(),tary,nullptr,false,offset);
1404     }
1405     // Arrays of bytes and of booleans both use 'bastore' and 'baload' so
1406     // cannot be distinguished by bytecode alone.
1407     if (ta->elem() == TypeInt::BOOL) {
1408       const TypeAry *tary = TypeAry::make(TypeInt::BYTE, ta->size());
1409       ciKlass* aklass = ciTypeArrayKlass::make(T_BYTE);
1410       tj = ta = TypeAryPtr::make(ptr,ta->const_oop(),tary,aklass,false,offset);
1411     }
1412     // During the 2nd round of IterGVN, NotNull castings are removed.
1413     // Make sure the Bottom and NotNull variants alias the same.
1414     // Also, make sure exact and non-exact variants alias the same.
1415     if (ptr == TypePtr::NotNull || ta->klass_is_exact() || ta->speculative() != nullptr) {
1416       tj = ta = ta->
1417               remove_speculative()->
1418               cast_to_ptr_type(TypePtr::BotPTR)->
1419               cast_to_exactness(false)->
1420               with_offset(offset);
1421     }
1422   }
1423 
1424   // Oop pointers need some flattening
1425   const TypeInstPtr *to = tj->isa_instptr();
1426   if (to && to != TypeOopPtr::BOTTOM) {
1427     ciInstanceKlass* ik = to->instance_klass();
1428     if( ptr == TypePtr::Constant ) {
1429       if (ik != ciEnv::current()->Class_klass() ||
1430           offset < ik->layout_helper_size_in_bytes()) {
1431         // No constant oop pointers (such as Strings); they alias with
1432         // unknown strings.
1433         assert(!is_known_inst, "not scalarizable allocation");
1434         tj = to = to->
1435                 cast_to_instance_id(TypeOopPtr::InstanceBot)->
1436                 remove_speculative()->
1437                 cast_to_ptr_type(TypePtr::BotPTR)->
1438                 cast_to_exactness(false);
1439       }
1440     } else if( is_known_inst ) {
1441       tj = to; // Keep NotNull and klass_is_exact for instance type
1442     } else if( ptr == TypePtr::NotNull || to->klass_is_exact() ) {
1443       // During the 2nd round of IterGVN, NotNull castings are removed.
1444       // Make sure the Bottom and NotNull variants alias the same.
1445       // Also, make sure exact and non-exact variants alias the same.
1446       tj = to = to->
1447               remove_speculative()->
1448               cast_to_instance_id(TypeOopPtr::InstanceBot)->
1449               cast_to_ptr_type(TypePtr::BotPTR)->
1450               cast_to_exactness(false);
1451     }
1452     if (to->speculative() != nullptr) {
1453       tj = to = to->remove_speculative();
1454     }
1455     // Canonicalize the holder of this field
1456     if (offset >= 0 && offset < instanceOopDesc::base_offset_in_bytes()) {
1457       // First handle header references such as a LoadKlassNode, even if the
1458       // object's klass is unloaded at compile time (4965979).
1459       if (!is_known_inst) { // Do it only for non-instance types
1460         tj = to = TypeInstPtr::make(TypePtr::BotPTR, env()->Object_klass(), false, nullptr, offset);
1461       }
1462     } else if (offset < 0 || offset >= ik->layout_helper_size_in_bytes()) {
1463       // Static fields are in the space above the normal instance
1464       // fields in the java.lang.Class instance.
1465       if (ik != ciEnv::current()->Class_klass()) {
1466         to = nullptr;
1467         tj = TypeOopPtr::BOTTOM;
1468         offset = tj->offset();
1469       }
1470     } else {
1471       ciInstanceKlass *canonical_holder = ik->get_canonical_holder(offset);
1472       assert(offset < canonical_holder->layout_helper_size_in_bytes(), "");
1473       assert(tj->offset() == offset, "no change to offset expected");
1474       bool xk = to->klass_is_exact();
1475       int instance_id = to->instance_id();
1476 
1477       // If the input type's class is the holder: if exact, the type only includes interfaces implemented by the holder
1478       // but if not exact, it may include extra interfaces: build new type from the holder class to make sure only
1479       // its interfaces are included.
1480       if (xk && ik->equals(canonical_holder)) {
1481         assert(tj == TypeInstPtr::make(to->ptr(), canonical_holder, is_known_inst, nullptr, offset, instance_id), "exact type should be canonical type");
1482       } else {
1483         assert(xk || !is_known_inst, "Known instance should be exact type");
1484         tj = to = TypeInstPtr::make(to->ptr(), canonical_holder, is_known_inst, nullptr, offset, instance_id);
1485       }
1486     }
1487   }
1488 
1489   // Klass pointers to object array klasses need some flattening
1490   const TypeKlassPtr *tk = tj->isa_klassptr();
1491   if( tk ) {
1492     // If we are referencing a field within a Klass, we need
1493     // to assume the worst case of an Object.  Both exact and
1494     // inexact types must flatten to the same alias class so
1495     // use NotNull as the PTR.
1496     if ( offset == Type::OffsetBot || (offset >= 0 && (size_t)offset < sizeof(Klass)) ) {
1497       tj = tk = TypeInstKlassPtr::make(TypePtr::NotNull,
1498                                        env()->Object_klass(),
1499                                        offset);
1500     }
1501 
1502     if (tk->isa_aryklassptr() && tk->is_aryklassptr()->elem()->isa_klassptr()) {
1503       ciKlass* k = ciObjArrayKlass::make(env()->Object_klass());
1504       if (!k || !k->is_loaded()) {                  // Only fails for some -Xcomp runs
1505         tj = tk = TypeInstKlassPtr::make(TypePtr::NotNull, env()->Object_klass(), offset);
1506       } else {
1507         tj = tk = TypeAryKlassPtr::make(TypePtr::NotNull, tk->is_aryklassptr()->elem(), k, offset);
1508       }
1509     }
1510 
1511     // Check for precise loads from the primary supertype array and force them
1512     // to the supertype cache alias index.  Check for generic array loads from
1513     // the primary supertype array and also force them to the supertype cache
1514     // alias index.  Since the same load can reach both, we need to merge
1515     // these 2 disparate memories into the same alias class.  Since the
1516     // primary supertype array is read-only, there's no chance of confusion
1517     // where we bypass an array load and an array store.
1518     int primary_supers_offset = in_bytes(Klass::primary_supers_offset());
1519     if (offset == Type::OffsetBot ||
1520         (offset >= primary_supers_offset &&
1521          offset < (int)(primary_supers_offset + Klass::primary_super_limit() * wordSize)) ||
1522         offset == (int)in_bytes(Klass::secondary_super_cache_offset())) {
1523       offset = in_bytes(Klass::secondary_super_cache_offset());
1524       tj = tk = tk->with_offset(offset);
1525     }
1526   }
1527 
1528   // Flatten all Raw pointers together.
1529   if (tj->base() == Type::RawPtr)
1530     tj = TypeRawPtr::BOTTOM;
1531 
1532   if (tj->base() == Type::AnyPtr)
1533     tj = TypePtr::BOTTOM;      // An error, which the caller must check for.
1534 
1535   offset = tj->offset();
1536   assert( offset != Type::OffsetTop, "Offset has fallen from constant" );
1537 
1538   assert( (offset != Type::OffsetBot && tj->base() != Type::AryPtr) ||
1539           (offset == Type::OffsetBot && tj->base() == Type::AryPtr) ||
1540           (offset == Type::OffsetBot && tj == TypeOopPtr::BOTTOM) ||
1541           (offset == Type::OffsetBot && tj == TypePtr::BOTTOM) ||
1542           (offset == oopDesc::mark_offset_in_bytes() && tj->base() == Type::AryPtr) ||
1543           (offset == Type::klass_offset() && tj->base() == Type::AryPtr) ||
1544           (offset == arrayOopDesc::length_offset_in_bytes() && tj->base() == Type::AryPtr),
1545           "For oops, klasses, raw offset must be constant; for arrays the offset is never known" );
1546   assert( tj->ptr() != TypePtr::TopPTR &&
1547           tj->ptr() != TypePtr::AnyNull &&
1548           tj->ptr() != TypePtr::Null, "No imprecise addresses" );
1549 //    assert( tj->ptr() != TypePtr::Constant ||
1550 //            tj->base() == Type::RawPtr ||
1551 //            tj->base() == Type::KlassPtr, "No constant oop addresses" );
1552 
1553   return tj;
1554 }
1555 
1556 void Compile::AliasType::Init(int i, const TypePtr* at) {
1557   assert(AliasIdxTop <= i && i < Compile::current()->_max_alias_types, "Invalid alias index");
1558   _index = i;
1559   _adr_type = at;
1560   _field = nullptr;
1561   _element = nullptr;
1562   _is_rewritable = true; // default
1563   const TypeOopPtr *atoop = (at != nullptr) ? at->isa_oopptr() : nullptr;
1564   if (atoop != nullptr && atoop->is_known_instance()) {
1565     const TypeOopPtr *gt = atoop->cast_to_instance_id(TypeOopPtr::InstanceBot);
1566     _general_index = Compile::current()->get_alias_index(gt);
1567   } else {
1568     _general_index = 0;
1569   }
1570 }
1571 
1572 BasicType Compile::AliasType::basic_type() const {
1573   if (element() != nullptr) {
1574     const Type* element = adr_type()->is_aryptr()->elem();
1575     return element->isa_narrowoop() ? T_OBJECT : element->array_element_basic_type();
1576   } if (field() != nullptr) {
1577     return field()->layout_type();
1578   } else {
1579     return T_ILLEGAL; // unknown
1580   }
1581 }
1582 
1583 //---------------------------------print_on------------------------------------
1584 #ifndef PRODUCT
1585 void Compile::AliasType::print_on(outputStream* st) {
1586   if (index() < 10)
1587         st->print("@ <%d> ", index());
1588   else  st->print("@ <%d>",  index());
1589   st->print(is_rewritable() ? "   " : " RO");
1590   int offset = adr_type()->offset();
1591   if (offset == Type::OffsetBot)
1592         st->print(" +any");
1593   else  st->print(" +%-3d", offset);
1594   st->print(" in ");
1595   adr_type()->dump_on(st);
1596   const TypeOopPtr* tjp = adr_type()->isa_oopptr();
1597   if (field() != nullptr && tjp) {
1598     if (tjp->is_instptr()->instance_klass()  != field()->holder() ||
1599         tjp->offset() != field()->offset_in_bytes()) {
1600       st->print(" != ");
1601       field()->print();
1602       st->print(" ***");
1603     }
1604   }
1605 }
1606 
1607 void print_alias_types() {
1608   Compile* C = Compile::current();
1609   tty->print_cr("--- Alias types, AliasIdxBot .. %d", C->num_alias_types()-1);
1610   for (int idx = Compile::AliasIdxBot; idx < C->num_alias_types(); idx++) {
1611     C->alias_type(idx)->print_on(tty);
1612     tty->cr();
1613   }
1614 }
1615 #endif
1616 
1617 
1618 //----------------------------probe_alias_cache--------------------------------
1619 Compile::AliasCacheEntry* Compile::probe_alias_cache(const TypePtr* adr_type) {
1620   intptr_t key = (intptr_t) adr_type;
1621   key ^= key >> logAliasCacheSize;
1622   return &_alias_cache[key & right_n_bits(logAliasCacheSize)];
1623 }
1624 
1625 
1626 //-----------------------------grow_alias_types--------------------------------
1627 void Compile::grow_alias_types() {
1628   const int old_ats  = _max_alias_types; // how many before?
1629   const int new_ats  = old_ats;          // how many more?
1630   const int grow_ats = old_ats+new_ats;  // how many now?
1631   _max_alias_types = grow_ats;
1632   _alias_types =  REALLOC_ARENA_ARRAY(comp_arena(), AliasType*, _alias_types, old_ats, grow_ats);
1633   AliasType* ats =    NEW_ARENA_ARRAY(comp_arena(), AliasType, new_ats);
1634   Copy::zero_to_bytes(ats, sizeof(AliasType)*new_ats);
1635   for (int i = 0; i < new_ats; i++)  _alias_types[old_ats+i] = &ats[i];
1636 }
1637 
1638 
1639 //--------------------------------find_alias_type------------------------------
1640 Compile::AliasType* Compile::find_alias_type(const TypePtr* adr_type, bool no_create, ciField* original_field) {
1641   if (!do_aliasing()) {
1642     return alias_type(AliasIdxBot);
1643   }
1644 
1645   AliasCacheEntry* ace = probe_alias_cache(adr_type);
1646   if (ace->_adr_type == adr_type) {
1647     return alias_type(ace->_index);
1648   }
1649 
1650   // Handle special cases.
1651   if (adr_type == nullptr)          return alias_type(AliasIdxTop);
1652   if (adr_type == TypePtr::BOTTOM)  return alias_type(AliasIdxBot);
1653 
1654   // Do it the slow way.
1655   const TypePtr* flat = flatten_alias_type(adr_type);
1656 
1657 #ifdef ASSERT
1658   {
1659     ResourceMark rm;
1660     assert(flat == flatten_alias_type(flat), "not idempotent: adr_type = %s; flat = %s => %s",
1661            Type::str(adr_type), Type::str(flat), Type::str(flatten_alias_type(flat)));
1662     assert(flat != TypePtr::BOTTOM, "cannot alias-analyze an untyped ptr: adr_type = %s",
1663            Type::str(adr_type));
1664     if (flat->isa_oopptr() && !flat->isa_klassptr()) {
1665       const TypeOopPtr* foop = flat->is_oopptr();
1666       // Scalarizable allocations have exact klass always.
1667       bool exact = !foop->klass_is_exact() || foop->is_known_instance();
1668       const TypePtr* xoop = foop->cast_to_exactness(exact)->is_ptr();
1669       assert(foop == flatten_alias_type(xoop), "exactness must not affect alias type: foop = %s; xoop = %s",
1670              Type::str(foop), Type::str(xoop));
1671     }
1672   }
1673 #endif
1674 
1675   int idx = AliasIdxTop;
1676   for (int i = 0; i < num_alias_types(); i++) {
1677     if (alias_type(i)->adr_type() == flat) {
1678       idx = i;
1679       break;
1680     }
1681   }
1682 
1683   if (idx == AliasIdxTop) {
1684     if (no_create)  return nullptr;
1685     // Grow the array if necessary.
1686     if (_num_alias_types == _max_alias_types)  grow_alias_types();
1687     // Add a new alias type.
1688     idx = _num_alias_types++;
1689     _alias_types[idx]->Init(idx, flat);
1690     if (flat == TypeInstPtr::KLASS)  alias_type(idx)->set_rewritable(false);
1691     if (flat == TypeAryPtr::RANGE)   alias_type(idx)->set_rewritable(false);
1692     if (flat->isa_instptr()) {
1693       if (flat->offset() == java_lang_Class::klass_offset()
1694           && flat->is_instptr()->instance_klass() == env()->Class_klass())
1695         alias_type(idx)->set_rewritable(false);
1696     }
1697     if (flat->isa_aryptr()) {
1698 #ifdef ASSERT
1699       const int header_size_min  = arrayOopDesc::base_offset_in_bytes(T_BYTE);
1700       // (T_BYTE has the weakest alignment and size restrictions...)
1701       assert(flat->offset() < header_size_min, "array body reference must be OffsetBot");
1702 #endif
1703       if (flat->offset() == TypePtr::OffsetBot) {
1704         alias_type(idx)->set_element(flat->is_aryptr()->elem());
1705       }
1706     }
1707     if (flat->isa_klassptr()) {
1708       if (UseCompactObjectHeaders) {
1709         if (flat->offset() == in_bytes(Klass::prototype_header_offset()))
1710           alias_type(idx)->set_rewritable(false);
1711       }
1712       if (flat->offset() == in_bytes(Klass::super_check_offset_offset()))
1713         alias_type(idx)->set_rewritable(false);
1714       if (flat->offset() == in_bytes(Klass::access_flags_offset()))
1715         alias_type(idx)->set_rewritable(false);
1716       if (flat->offset() == in_bytes(Klass::misc_flags_offset()))
1717         alias_type(idx)->set_rewritable(false);
1718       if (flat->offset() == in_bytes(Klass::java_mirror_offset()))
1719         alias_type(idx)->set_rewritable(false);
1720       if (flat->offset() == in_bytes(Klass::secondary_super_cache_offset()))
1721         alias_type(idx)->set_rewritable(false);
1722     }
1723     // %%% (We would like to finalize JavaThread::threadObj_offset(),
1724     // but the base pointer type is not distinctive enough to identify
1725     // references into JavaThread.)
1726 
1727     // Check for final fields.
1728     const TypeInstPtr* tinst = flat->isa_instptr();
1729     if (tinst && tinst->offset() >= instanceOopDesc::base_offset_in_bytes()) {
1730       ciField* field;
1731       if (tinst->const_oop() != nullptr &&
1732           tinst->instance_klass() == ciEnv::current()->Class_klass() &&
1733           tinst->offset() >= (tinst->instance_klass()->layout_helper_size_in_bytes())) {
1734         // static field
1735         ciInstanceKlass* k = tinst->const_oop()->as_instance()->java_lang_Class_klass()->as_instance_klass();
1736         field = k->get_field_by_offset(tinst->offset(), true);
1737       } else {
1738         ciInstanceKlass *k = tinst->instance_klass();
1739         field = k->get_field_by_offset(tinst->offset(), false);
1740       }
1741       assert(field == nullptr ||
1742              original_field == nullptr ||
1743              (field->holder() == original_field->holder() &&
1744               field->offset_in_bytes() == original_field->offset_in_bytes() &&
1745               field->is_static() == original_field->is_static()), "wrong field?");
1746       // Set field() and is_rewritable() attributes.
1747       if (field != nullptr)  alias_type(idx)->set_field(field);
1748     }
1749   }
1750 
1751   // Fill the cache for next time.
1752   ace->_adr_type = adr_type;
1753   ace->_index    = idx;
1754   assert(alias_type(adr_type) == alias_type(idx),  "type must be installed");
1755 
1756   // Might as well try to fill the cache for the flattened version, too.
1757   AliasCacheEntry* face = probe_alias_cache(flat);
1758   if (face->_adr_type == nullptr) {
1759     face->_adr_type = flat;
1760     face->_index    = idx;
1761     assert(alias_type(flat) == alias_type(idx), "flat type must work too");
1762   }
1763 
1764   return alias_type(idx);
1765 }
1766 
1767 
1768 Compile::AliasType* Compile::alias_type(ciField* field) {
1769   const TypeOopPtr* t;
1770   if (field->is_static())
1771     t = TypeInstPtr::make(field->holder()->java_mirror());
1772   else
1773     t = TypeOopPtr::make_from_klass_raw(field->holder());
1774   AliasType* atp = alias_type(t->add_offset(field->offset_in_bytes()), field);
1775   assert((field->is_final() || field->is_stable()) == !atp->is_rewritable(), "must get the rewritable bits correct");
1776   return atp;
1777 }
1778 
1779 
1780 //------------------------------have_alias_type--------------------------------
1781 bool Compile::have_alias_type(const TypePtr* adr_type) {
1782   AliasCacheEntry* ace = probe_alias_cache(adr_type);
1783   if (ace->_adr_type == adr_type) {
1784     return true;
1785   }
1786 
1787   // Handle special cases.
1788   if (adr_type == nullptr)             return true;
1789   if (adr_type == TypePtr::BOTTOM)  return true;
1790 
1791   return find_alias_type(adr_type, true, nullptr) != nullptr;
1792 }
1793 
1794 //-----------------------------must_alias--------------------------------------
1795 // True if all values of the given address type are in the given alias category.
1796 bool Compile::must_alias(const TypePtr* adr_type, int alias_idx) {
1797   if (alias_idx == AliasIdxBot)         return true;  // the universal category
1798   if (adr_type == nullptr)              return true;  // null serves as TypePtr::TOP
1799   if (alias_idx == AliasIdxTop)         return false; // the empty category
1800   if (adr_type->base() == Type::AnyPtr) return false; // TypePtr::BOTTOM or its twins
1801 
1802   // the only remaining possible overlap is identity
1803   int adr_idx = get_alias_index(adr_type);
1804   assert(adr_idx != AliasIdxBot && adr_idx != AliasIdxTop, "");
1805   assert(adr_idx == alias_idx ||
1806          (alias_type(alias_idx)->adr_type() != TypeOopPtr::BOTTOM
1807           && adr_type                       != TypeOopPtr::BOTTOM),
1808          "should not be testing for overlap with an unsafe pointer");
1809   return adr_idx == alias_idx;
1810 }
1811 
1812 //------------------------------can_alias--------------------------------------
1813 // True if any values of the given address type are in the given alias category.
1814 bool Compile::can_alias(const TypePtr* adr_type, int alias_idx) {
1815   if (alias_idx == AliasIdxTop)         return false; // the empty category
1816   if (adr_type == nullptr)              return false; // null serves as TypePtr::TOP
1817   // Known instance doesn't alias with bottom memory
1818   if (alias_idx == AliasIdxBot)         return !adr_type->is_known_instance();                   // the universal category
1819   if (adr_type->base() == Type::AnyPtr) return !C->get_adr_type(alias_idx)->is_known_instance(); // TypePtr::BOTTOM or its twins
1820 
1821   // the only remaining possible overlap is identity
1822   int adr_idx = get_alias_index(adr_type);
1823   assert(adr_idx != AliasIdxBot && adr_idx != AliasIdxTop, "");
1824   return adr_idx == alias_idx;
1825 }
1826 
1827 // Mark all ParsePredicateNodes as useless. They will later be removed from the graph in IGVN together with their
1828 // uncommon traps if no Runtime Predicates were created from the Parse Predicates.
1829 void Compile::mark_parse_predicate_nodes_useless(PhaseIterGVN& igvn) {
1830   if (parse_predicate_count() == 0) {
1831     return;
1832   }
1833   for (int i = 0; i < parse_predicate_count(); i++) {
1834     ParsePredicateNode* parse_predicate = _parse_predicates.at(i);
1835     parse_predicate->mark_useless(igvn);
1836   }
1837   _parse_predicates.clear();
1838 }
1839 
1840 void Compile::record_for_post_loop_opts_igvn(Node* n) {
1841   if (!n->for_post_loop_opts_igvn()) {
1842     assert(!_for_post_loop_igvn.contains(n), "duplicate");
1843     n->add_flag(Node::NodeFlags::Flag_for_post_loop_opts_igvn);
1844     _for_post_loop_igvn.append(n);
1845   }
1846 }
1847 
1848 void Compile::remove_from_post_loop_opts_igvn(Node* n) {
1849   n->remove_flag(Node::NodeFlags::Flag_for_post_loop_opts_igvn);
1850   _for_post_loop_igvn.remove(n);
1851 }
1852 
1853 void Compile::process_for_post_loop_opts_igvn(PhaseIterGVN& igvn) {
1854   // Verify that all previous optimizations produced a valid graph
1855   // at least to this point, even if no loop optimizations were done.
1856   PhaseIdealLoop::verify(igvn);
1857 
1858   if (has_loops() || _loop_opts_cnt > 0) {
1859     print_method(PHASE_AFTER_LOOP_OPTS, 2);
1860   }
1861   C->set_post_loop_opts_phase(); // no more loop opts allowed
1862 
1863   assert(!C->major_progress(), "not cleared");
1864 
1865   if (_for_post_loop_igvn.length() > 0) {
1866     while (_for_post_loop_igvn.length() > 0) {
1867       Node* n = _for_post_loop_igvn.pop();
1868       n->remove_flag(Node::NodeFlags::Flag_for_post_loop_opts_igvn);
1869       igvn._worklist.push(n);
1870     }
1871     igvn.optimize();
1872     if (failing()) return;
1873     assert(_for_post_loop_igvn.length() == 0, "no more delayed nodes allowed");
1874     assert(C->parse_predicate_count() == 0, "all parse predicates should have been removed now");
1875 
1876     // Sometimes IGVN sets major progress (e.g., when processing loop nodes).
1877     if (C->major_progress()) {
1878       C->clear_major_progress(); // ensure that major progress is now clear
1879     }
1880   }
1881 }
1882 
1883 void Compile::record_for_merge_stores_igvn(Node* n) {
1884   if (!n->for_merge_stores_igvn()) {
1885     assert(!_for_merge_stores_igvn.contains(n), "duplicate");
1886     n->add_flag(Node::NodeFlags::Flag_for_merge_stores_igvn);
1887     _for_merge_stores_igvn.append(n);
1888   }
1889 }
1890 
1891 void Compile::remove_from_merge_stores_igvn(Node* n) {
1892   n->remove_flag(Node::NodeFlags::Flag_for_merge_stores_igvn);
1893   _for_merge_stores_igvn.remove(n);
1894 }
1895 
1896 // We need to wait with merging stores until RangeCheck smearing has removed the RangeChecks during
1897 // the post loops IGVN phase. If we do it earlier, then there may still be some RangeChecks between
1898 // the stores, and we merge the wrong sequence of stores.
1899 // Example:
1900 //   StoreI RangeCheck StoreI StoreI RangeCheck StoreI
1901 // Apply MergeStores:
1902 //   StoreI RangeCheck [   StoreL  ] RangeCheck StoreI
1903 // Remove more RangeChecks:
1904 //   StoreI            [   StoreL  ]            StoreI
1905 // But now it would have been better to do this instead:
1906 //   [         StoreL       ] [       StoreL         ]
1907 //
1908 // Note: we allow stores to merge in this dedicated IGVN round, and any later IGVN round,
1909 //       since we never unset _merge_stores_phase.
1910 void Compile::process_for_merge_stores_igvn(PhaseIterGVN& igvn) {
1911   C->set_merge_stores_phase();
1912 
1913   if (_for_merge_stores_igvn.length() > 0) {
1914     while (_for_merge_stores_igvn.length() > 0) {
1915       Node* n = _for_merge_stores_igvn.pop();
1916       n->remove_flag(Node::NodeFlags::Flag_for_merge_stores_igvn);
1917       igvn._worklist.push(n);
1918     }
1919     igvn.optimize();
1920     if (failing()) return;
1921     assert(_for_merge_stores_igvn.length() == 0, "no more delayed nodes allowed");
1922     print_method(PHASE_AFTER_MERGE_STORES, 3);
1923   }
1924 }
1925 
1926 void Compile::record_unstable_if_trap(UnstableIfTrap* trap) {
1927   if (OptimizeUnstableIf) {
1928     _unstable_if_traps.append(trap);
1929   }
1930 }
1931 
1932 void Compile::remove_useless_unstable_if_traps(Unique_Node_List& useful) {
1933   for (int i = _unstable_if_traps.length() - 1; i >= 0; i--) {
1934     UnstableIfTrap* trap = _unstable_if_traps.at(i);
1935     Node* n = trap->uncommon_trap();
1936     if (!useful.member(n)) {
1937       _unstable_if_traps.delete_at(i); // replaces i-th with last element which is known to be useful (already processed)
1938     }
1939   }
1940 }
1941 
1942 // Remove the unstable if trap associated with 'unc' from candidates. It is either dead
1943 // or fold-compares case. Return true if succeed or not found.
1944 //
1945 // In rare cases, the found trap has been processed. It is too late to delete it. Return
1946 // false and ask fold-compares to yield.
1947 //
1948 // 'fold-compares' may use the uncommon_trap of the dominating IfNode to cover the fused
1949 // IfNode. This breaks the unstable_if trap invariant: control takes the unstable path
1950 // when deoptimization does happen.
1951 bool Compile::remove_unstable_if_trap(CallStaticJavaNode* unc, bool yield) {
1952   for (int i = 0; i < _unstable_if_traps.length(); ++i) {
1953     UnstableIfTrap* trap = _unstable_if_traps.at(i);
1954     if (trap->uncommon_trap() == unc) {
1955       if (yield && trap->modified()) {
1956         return false;
1957       }
1958       _unstable_if_traps.delete_at(i);
1959       break;
1960     }
1961   }
1962   return true;
1963 }
1964 
1965 // Re-calculate unstable_if traps with the liveness of next_bci, which points to the unlikely path.
1966 // It needs to be done after igvn because fold-compares may fuse uncommon_traps and before renumbering.
1967 void Compile::process_for_unstable_if_traps(PhaseIterGVN& igvn) {
1968   for (int i = _unstable_if_traps.length() - 1; i >= 0; --i) {
1969     UnstableIfTrap* trap = _unstable_if_traps.at(i);
1970     CallStaticJavaNode* unc = trap->uncommon_trap();
1971     int next_bci = trap->next_bci();
1972     bool modified = trap->modified();
1973 
1974     if (next_bci != -1 && !modified) {
1975       assert(!_dead_node_list.test(unc->_idx), "changing a dead node!");
1976       JVMState* jvms = unc->jvms();
1977       ciMethod* method = jvms->method();
1978       ciBytecodeStream iter(method);
1979 
1980       iter.force_bci(jvms->bci());
1981       assert(next_bci == iter.next_bci() || next_bci == iter.get_dest(), "wrong next_bci at unstable_if");
1982       Bytecodes::Code c = iter.cur_bc();
1983       Node* lhs = nullptr;
1984       Node* rhs = nullptr;
1985       if (c == Bytecodes::_if_acmpeq || c == Bytecodes::_if_acmpne) {
1986         lhs = unc->peek_operand(0);
1987         rhs = unc->peek_operand(1);
1988       } else if (c == Bytecodes::_ifnull || c == Bytecodes::_ifnonnull) {
1989         lhs = unc->peek_operand(0);
1990       }
1991 
1992       ResourceMark rm;
1993       const MethodLivenessResult& live_locals = method->liveness_at_bci(next_bci);
1994       assert(live_locals.is_valid(), "broken liveness info");
1995       int len = (int)live_locals.size();
1996 
1997       for (int i = 0; i < len; i++) {
1998         Node* local = unc->local(jvms, i);
1999         // kill local using the liveness of next_bci.
2000         // give up when the local looks like an operand to secure reexecution.
2001         if (!live_locals.at(i) && !local->is_top() && local != lhs && local!= rhs) {
2002           uint idx = jvms->locoff() + i;
2003 #ifdef ASSERT
2004           if (PrintOpto && Verbose) {
2005             tty->print("[unstable_if] kill local#%d: ", idx);
2006             local->dump();
2007             tty->cr();
2008           }
2009 #endif
2010           igvn.replace_input_of(unc, idx, top());
2011           modified = true;
2012         }
2013       }
2014     }
2015 
2016     // keep the mondified trap for late query
2017     if (modified) {
2018       trap->set_modified();
2019     } else {
2020       _unstable_if_traps.delete_at(i);
2021     }
2022   }
2023   igvn.optimize();
2024 }
2025 
2026 // StringOpts and late inlining of string methods
2027 void Compile::inline_string_calls(bool parse_time) {
2028   {
2029     // remove useless nodes to make the usage analysis simpler
2030     ResourceMark rm;
2031     PhaseRemoveUseless pru(initial_gvn(), *igvn_worklist());
2032   }
2033 
2034   {
2035     ResourceMark rm;
2036     print_method(PHASE_BEFORE_STRINGOPTS, 3);
2037     PhaseStringOpts pso(initial_gvn());
2038     print_method(PHASE_AFTER_STRINGOPTS, 3);
2039   }
2040 
2041   // now inline anything that we skipped the first time around
2042   if (!parse_time) {
2043     _late_inlines_pos = _late_inlines.length();
2044   }
2045 
2046   while (_string_late_inlines.length() > 0) {
2047     CallGenerator* cg = _string_late_inlines.pop();
2048     cg->do_late_inline();
2049     if (failing())  return;
2050   }
2051   _string_late_inlines.trunc_to(0);
2052 }
2053 
2054 // Late inlining of boxing methods
2055 void Compile::inline_boxing_calls(PhaseIterGVN& igvn) {
2056   if (_boxing_late_inlines.length() > 0) {
2057     assert(has_boxed_value(), "inconsistent");
2058 
2059     set_inlining_incrementally(true);
2060 
2061     igvn_worklist()->ensure_empty(); // should be done with igvn
2062 
2063     _late_inlines_pos = _late_inlines.length();
2064 
2065     while (_boxing_late_inlines.length() > 0) {
2066       CallGenerator* cg = _boxing_late_inlines.pop();
2067       cg->do_late_inline();
2068       if (failing())  return;
2069     }
2070     _boxing_late_inlines.trunc_to(0);
2071 
2072     inline_incrementally_cleanup(igvn);
2073 
2074     set_inlining_incrementally(false);
2075   }
2076 }
2077 
2078 bool Compile::inline_incrementally_one() {
2079   assert(IncrementalInline, "incremental inlining should be on");
2080 
2081   TracePhase tp(_t_incrInline_inline);
2082 
2083   set_inlining_progress(false);
2084   set_do_cleanup(false);
2085 
2086   for (int i = 0; i < _late_inlines.length(); i++) {
2087     _late_inlines_pos = i+1;
2088     CallGenerator* cg = _late_inlines.at(i);
2089     bool is_scheduled_for_igvn_before = C->igvn_worklist()->member(cg->call_node());
2090     bool does_dispatch = cg->is_virtual_late_inline() || cg->is_mh_late_inline();
2091     if (inlining_incrementally() || does_dispatch) { // a call can be either inlined or strength-reduced to a direct call
2092       cg->do_late_inline();
2093       assert(_late_inlines.at(i) == cg, "no insertions before current position allowed");
2094       if (failing()) {
2095         return false;
2096       } else if (inlining_progress()) {
2097         _late_inlines_pos = i+1; // restore the position in case new elements were inserted
2098         print_method(PHASE_INCREMENTAL_INLINE_STEP, 3, cg->call_node());
2099         break; // process one call site at a time
2100       } else {
2101         bool is_scheduled_for_igvn_after = C->igvn_worklist()->member(cg->call_node());
2102         if (!is_scheduled_for_igvn_before && is_scheduled_for_igvn_after) {
2103           // Avoid potential infinite loop if node already in the IGVN list
2104           assert(false, "scheduled for IGVN during inlining attempt");
2105         } else {
2106           // Ensure call node has not disappeared from IGVN worklist during a failed inlining attempt
2107           assert(!is_scheduled_for_igvn_before || is_scheduled_for_igvn_after, "call node removed from IGVN list during inlining pass");
2108           cg->call_node()->set_generator(cg);
2109         }
2110       }
2111     } else {
2112       // Ignore late inline direct calls when inlining is not allowed.
2113       // They are left in the late inline list when node budget is exhausted until the list is fully drained.
2114     }
2115   }
2116   // Remove processed elements.
2117   _late_inlines.remove_till(_late_inlines_pos);
2118   _late_inlines_pos = 0;
2119 
2120   assert(inlining_progress() || _late_inlines.length() == 0, "no progress");
2121 
2122   bool needs_cleanup = do_cleanup() || over_inlining_cutoff();
2123 
2124   set_inlining_progress(false);
2125   set_do_cleanup(false);
2126 
2127   bool force_cleanup = directive()->IncrementalInlineForceCleanupOption;
2128   return (_late_inlines.length() > 0) && !needs_cleanup && !force_cleanup;
2129 }
2130 
2131 void Compile::inline_incrementally_cleanup(PhaseIterGVN& igvn) {
2132   {
2133     TracePhase tp(_t_incrInline_pru);
2134     ResourceMark rm;
2135     PhaseRemoveUseless pru(initial_gvn(), *igvn_worklist());
2136   }
2137   {
2138     TracePhase tp(_t_incrInline_igvn);
2139     igvn.reset_from_gvn(initial_gvn());
2140     igvn.optimize();
2141     if (failing()) return;
2142   }
2143   print_method(PHASE_INCREMENTAL_INLINE_CLEANUP, 3);
2144 }
2145 
2146 // Perform incremental inlining until bound on number of live nodes is reached
2147 void Compile::inline_incrementally(PhaseIterGVN& igvn) {
2148   TracePhase tp(_t_incrInline);
2149 
2150   set_inlining_incrementally(true);
2151   uint low_live_nodes = 0;
2152 
2153   while (_late_inlines.length() > 0) {
2154     if (live_nodes() > (uint)LiveNodeCountInliningCutoff) {
2155       if (low_live_nodes < (uint)LiveNodeCountInliningCutoff * 8 / 10) {
2156         TracePhase tp(_t_incrInline_ideal);
2157         // PhaseIdealLoop is expensive so we only try it once we are
2158         // out of live nodes and we only try it again if the previous
2159         // helped got the number of nodes down significantly
2160         PhaseIdealLoop::optimize(igvn, LoopOptsNone);
2161         if (failing())  return;
2162         low_live_nodes = live_nodes();
2163         _major_progress = true;
2164       }
2165 
2166       if (live_nodes() > (uint)LiveNodeCountInliningCutoff) {
2167         bool do_print_inlining = print_inlining() || print_intrinsics();
2168         if (do_print_inlining || log() != nullptr) {
2169           // Print inlining message for candidates that we couldn't inline for lack of space.
2170           for (int i = 0; i < _late_inlines.length(); i++) {
2171             CallGenerator* cg = _late_inlines.at(i);
2172             const char* msg = "live nodes > LiveNodeCountInliningCutoff";
2173             if (do_print_inlining) {
2174               inline_printer()->record(cg->method(), cg->call_node()->jvms(), InliningResult::FAILURE, msg);
2175             }
2176             log_late_inline_failure(cg, msg);
2177           }
2178         }
2179         break; // finish
2180       }
2181     }
2182 
2183     igvn_worklist()->ensure_empty(); // should be done with igvn
2184 
2185     while (inline_incrementally_one()) {
2186       assert(!failing_internal() || failure_is_artificial(), "inconsistent");
2187     }
2188     if (failing())  return;
2189 
2190     inline_incrementally_cleanup(igvn);
2191 
2192     print_method(PHASE_INCREMENTAL_INLINE_STEP, 3);
2193 
2194     if (failing())  return;
2195 
2196     if (_late_inlines.length() == 0) {
2197       break; // no more progress
2198     }
2199   }
2200 
2201   igvn_worklist()->ensure_empty(); // should be done with igvn
2202 
2203   if (_string_late_inlines.length() > 0) {
2204     assert(has_stringbuilder(), "inconsistent");
2205 
2206     inline_string_calls(false);
2207 
2208     if (failing())  return;
2209 
2210     inline_incrementally_cleanup(igvn);
2211   }
2212 
2213   set_inlining_incrementally(false);
2214 }
2215 
2216 void Compile::process_late_inline_calls_no_inline(PhaseIterGVN& igvn) {
2217   // "inlining_incrementally() == false" is used to signal that no inlining is allowed
2218   // (see LateInlineVirtualCallGenerator::do_late_inline_check() for details).
2219   // Tracking and verification of modified nodes is disabled by setting "_modified_nodes == nullptr"
2220   // as if "inlining_incrementally() == true" were set.
2221   assert(inlining_incrementally() == false, "not allowed");
2222   assert(_modified_nodes == nullptr, "not allowed");
2223   assert(_late_inlines.length() > 0, "sanity");
2224 
2225   while (_late_inlines.length() > 0) {
2226     igvn_worklist()->ensure_empty(); // should be done with igvn
2227 
2228     while (inline_incrementally_one()) {
2229       assert(!failing_internal() || failure_is_artificial(), "inconsistent");
2230     }
2231     if (failing())  return;
2232 
2233     inline_incrementally_cleanup(igvn);
2234   }
2235 }
2236 
2237 bool Compile::optimize_loops(PhaseIterGVN& igvn, LoopOptsMode mode) {
2238   if (_loop_opts_cnt > 0) {
2239     while (major_progress() && (_loop_opts_cnt > 0)) {
2240       TracePhase tp(_t_idealLoop);
2241       PhaseIdealLoop::optimize(igvn, mode);
2242       _loop_opts_cnt--;
2243       if (failing())  return false;
2244       if (major_progress()) print_method(PHASE_PHASEIDEALLOOP_ITERATIONS, 2);
2245     }
2246   }
2247   return true;
2248 }
2249 
2250 // Remove edges from "root" to each SafePoint at a backward branch.
2251 // They were inserted during parsing (see add_safepoint()) to make
2252 // infinite loops without calls or exceptions visible to root, i.e.,
2253 // useful.
2254 void Compile::remove_root_to_sfpts_edges(PhaseIterGVN& igvn) {
2255   Node *r = root();
2256   if (r != nullptr) {
2257     for (uint i = r->req(); i < r->len(); ++i) {
2258       Node *n = r->in(i);
2259       if (n != nullptr && n->is_SafePoint()) {
2260         r->rm_prec(i);
2261         if (n->outcnt() == 0) {
2262           igvn.remove_dead_node(n);
2263         }
2264         --i;
2265       }
2266     }
2267     // Parsing may have added top inputs to the root node (Path
2268     // leading to the Halt node proven dead). Make sure we get a
2269     // chance to clean them up.
2270     igvn._worklist.push(r);
2271     igvn.optimize();
2272   }
2273 }
2274 
2275 //------------------------------Optimize---------------------------------------
2276 // Given a graph, optimize it.
2277 void Compile::Optimize() {
2278   TracePhase tp(_t_optimizer);
2279 
2280 #ifndef PRODUCT
2281   if (env()->break_at_compile()) {
2282     BREAKPOINT;
2283   }
2284 
2285 #endif
2286 
2287   BarrierSetC2* bs = BarrierSet::barrier_set()->barrier_set_c2();
2288 #ifdef ASSERT
2289   bs->verify_gc_barriers(this, BarrierSetC2::BeforeOptimize);
2290 #endif
2291 
2292   ResourceMark rm;
2293 
2294   NOT_PRODUCT( verify_graph_edges(); )
2295 
2296   print_method(PHASE_AFTER_PARSING, 1);
2297 
2298  {
2299   // Iterative Global Value Numbering, including ideal transforms
2300   // Initialize IterGVN with types and values from parse-time GVN
2301   PhaseIterGVN igvn(initial_gvn());
2302 #ifdef ASSERT
2303   _modified_nodes = new (comp_arena()) Unique_Node_List(comp_arena());
2304 #endif
2305   {
2306     TracePhase tp(_t_iterGVN);
2307     igvn.optimize();
2308   }
2309 
2310   if (failing())  return;
2311 
2312   print_method(PHASE_ITER_GVN1, 2);
2313 
2314   process_for_unstable_if_traps(igvn);
2315 
2316   if (failing())  return;
2317 
2318   inline_incrementally(igvn);
2319 
2320   print_method(PHASE_INCREMENTAL_INLINE, 2);
2321 
2322   if (failing())  return;
2323 
2324   if (eliminate_boxing()) {
2325     // Inline valueOf() methods now.
2326     inline_boxing_calls(igvn);
2327 
2328     if (failing())  return;
2329 
2330     if (AlwaysIncrementalInline || StressIncrementalInlining) {
2331       inline_incrementally(igvn);
2332     }
2333 
2334     print_method(PHASE_INCREMENTAL_BOXING_INLINE, 2);
2335 
2336     if (failing())  return;
2337   }
2338 
2339   // Remove the speculative part of types and clean up the graph from
2340   // the extra CastPP nodes whose only purpose is to carry them. Do
2341   // that early so that optimizations are not disrupted by the extra
2342   // CastPP nodes.
2343   remove_speculative_types(igvn);
2344 
2345   if (failing())  return;
2346 
2347   // No more new expensive nodes will be added to the list from here
2348   // so keep only the actual candidates for optimizations.
2349   cleanup_expensive_nodes(igvn);
2350 
2351   if (failing())  return;
2352 
2353   assert(EnableVectorSupport || !has_vbox_nodes(), "sanity");
2354   if (EnableVectorSupport && has_vbox_nodes()) {
2355     TracePhase tp(_t_vector);
2356     PhaseVector pv(igvn);
2357     pv.optimize_vector_boxes();
2358     if (failing())  return;
2359     print_method(PHASE_ITER_GVN_AFTER_VECTOR, 2);
2360   }
2361   assert(!has_vbox_nodes(), "sanity");
2362 
2363   if (!failing() && RenumberLiveNodes && live_nodes() + NodeLimitFudgeFactor < unique()) {
2364     Compile::TracePhase tp(_t_renumberLive);
2365     igvn_worklist()->ensure_empty(); // should be done with igvn
2366     {
2367       ResourceMark rm;
2368       PhaseRenumberLive prl(initial_gvn(), *igvn_worklist());
2369     }
2370     igvn.reset_from_gvn(initial_gvn());
2371     igvn.optimize();
2372     if (failing()) return;
2373   }
2374 
2375   // Now that all inlining is over and no PhaseRemoveUseless will run, cut edge from root to loop
2376   // safepoints
2377   remove_root_to_sfpts_edges(igvn);
2378 
2379   if (failing())  return;
2380 
2381   if (has_loops()) {
2382     print_method(PHASE_BEFORE_LOOP_OPTS, 2);
2383   }
2384 
2385   // Perform escape analysis
2386   if (do_escape_analysis() && ConnectionGraph::has_candidates(this)) {
2387     if (has_loops()) {
2388       // Cleanup graph (remove dead nodes).
2389       TracePhase tp(_t_idealLoop);
2390       PhaseIdealLoop::optimize(igvn, LoopOptsMaxUnroll);
2391       if (failing())  return;
2392     }
2393     bool progress;
2394     print_method(PHASE_PHASEIDEAL_BEFORE_EA, 2);
2395     do {
2396       ConnectionGraph::do_analysis(this, &igvn);
2397 
2398       if (failing())  return;
2399 
2400       int mcount = macro_count(); // Record number of allocations and locks before IGVN
2401 
2402       // Optimize out fields loads from scalar replaceable allocations.
2403       igvn.optimize();
2404       print_method(PHASE_ITER_GVN_AFTER_EA, 2);
2405 
2406       if (failing()) return;
2407 
2408       if (congraph() != nullptr && macro_count() > 0) {
2409         TracePhase tp(_t_macroEliminate);
2410         PhaseMacroExpand mexp(igvn);
2411         mexp.eliminate_macro_nodes();
2412         if (failing()) return;
2413 
2414         igvn.set_delay_transform(false);
2415         igvn.optimize();
2416         if (failing()) return;
2417 
2418         print_method(PHASE_ITER_GVN_AFTER_ELIMINATION, 2);
2419       }
2420 
2421       ConnectionGraph::verify_ram_nodes(this, root());
2422       if (failing())  return;
2423 
2424       progress = do_iterative_escape_analysis() &&
2425                  (macro_count() < mcount) &&
2426                  ConnectionGraph::has_candidates(this);
2427       // Try again if candidates exist and made progress
2428       // by removing some allocations and/or locks.
2429     } while (progress);
2430   }
2431 
2432   // Loop transforms on the ideal graph.  Range Check Elimination,
2433   // peeling, unrolling, etc.
2434 
2435   // Set loop opts counter
2436   if((_loop_opts_cnt > 0) && (has_loops() || has_split_ifs())) {
2437     {
2438       TracePhase tp(_t_idealLoop);
2439       PhaseIdealLoop::optimize(igvn, LoopOptsDefault);
2440       _loop_opts_cnt--;
2441       if (major_progress()) print_method(PHASE_PHASEIDEALLOOP1, 2);
2442       if (failing())  return;
2443     }
2444     // Loop opts pass if partial peeling occurred in previous pass
2445     if(PartialPeelLoop && major_progress() && (_loop_opts_cnt > 0)) {
2446       TracePhase tp(_t_idealLoop);
2447       PhaseIdealLoop::optimize(igvn, LoopOptsSkipSplitIf);
2448       _loop_opts_cnt--;
2449       if (major_progress()) print_method(PHASE_PHASEIDEALLOOP2, 2);
2450       if (failing())  return;
2451     }
2452     // Loop opts pass for loop-unrolling before CCP
2453     if(major_progress() && (_loop_opts_cnt > 0)) {
2454       TracePhase tp(_t_idealLoop);
2455       PhaseIdealLoop::optimize(igvn, LoopOptsSkipSplitIf);
2456       _loop_opts_cnt--;
2457       if (major_progress()) print_method(PHASE_PHASEIDEALLOOP3, 2);
2458     }
2459     if (!failing()) {
2460       // Verify that last round of loop opts produced a valid graph
2461       PhaseIdealLoop::verify(igvn);
2462     }
2463   }
2464   if (failing())  return;
2465 
2466   // Conditional Constant Propagation;
2467   print_method(PHASE_BEFORE_CCP1, 2);
2468   PhaseCCP ccp( &igvn );
2469   assert( true, "Break here to ccp.dump_nodes_and_types(_root,999,1)");
2470   {
2471     TracePhase tp(_t_ccp);
2472     ccp.do_transform();
2473   }
2474   print_method(PHASE_CCP1, 2);
2475 
2476   assert( true, "Break here to ccp.dump_old2new_map()");
2477 
2478   // Iterative Global Value Numbering, including ideal transforms
2479   {
2480     TracePhase tp(_t_iterGVN2);
2481     igvn.reset_from_igvn(&ccp);
2482     igvn.optimize();
2483   }
2484   print_method(PHASE_ITER_GVN2, 2);
2485 
2486   if (failing())  return;
2487 
2488   // Loop transforms on the ideal graph.  Range Check Elimination,
2489   // peeling, unrolling, etc.
2490   if (!optimize_loops(igvn, LoopOptsDefault)) {
2491     return;
2492   }
2493 
2494   if (failing())  return;
2495 
2496   C->clear_major_progress(); // ensure that major progress is now clear
2497 
2498   process_for_post_loop_opts_igvn(igvn);
2499 
2500   process_for_merge_stores_igvn(igvn);
2501 
2502   if (failing())  return;
2503 
2504 #ifdef ASSERT
2505   bs->verify_gc_barriers(this, BarrierSetC2::BeforeMacroExpand);
2506 #endif
2507 
2508   {
2509     TracePhase tp(_t_macroExpand);
2510     print_method(PHASE_BEFORE_MACRO_EXPANSION, 3);
2511     PhaseMacroExpand  mex(igvn);
2512     if (mex.expand_macro_nodes()) {
2513       assert(failing(), "must bail out w/ explicit message");
2514       return;
2515     }
2516     print_method(PHASE_AFTER_MACRO_EXPANSION, 2);
2517   }
2518 
2519   {
2520     TracePhase tp(_t_barrierExpand);
2521     if (bs->expand_barriers(this, igvn)) {
2522       assert(failing(), "must bail out w/ explicit message");
2523       return;
2524     }
2525     print_method(PHASE_BARRIER_EXPANSION, 2);
2526   }
2527 
2528   if (C->max_vector_size() > 0) {
2529     C->optimize_logic_cones(igvn);
2530     igvn.optimize();
2531     if (failing()) return;
2532   }
2533 
2534   DEBUG_ONLY( _modified_nodes = nullptr; )
2535 
2536   assert(igvn._worklist.size() == 0, "not empty");
2537 
2538   assert(_late_inlines.length() == 0 || IncrementalInlineMH || IncrementalInlineVirtual, "not empty");
2539 
2540   if (_late_inlines.length() > 0) {
2541     // More opportunities to optimize virtual and MH calls.
2542     // Though it's maybe too late to perform inlining, strength-reducing them to direct calls is still an option.
2543     process_late_inline_calls_no_inline(igvn);
2544     if (failing())  return;
2545   }
2546  } // (End scope of igvn; run destructor if necessary for asserts.)
2547 
2548  check_no_dead_use();
2549 
2550  // We will never use the NodeHash table any more. Clear it so that final_graph_reshaping does not have
2551  // to remove hashes to unlock nodes for modifications.
2552  C->node_hash()->clear();
2553 
2554  // A method with only infinite loops has no edges entering loops from root
2555  {
2556    TracePhase tp(_t_graphReshaping);
2557    if (final_graph_reshaping()) {
2558      assert(failing(), "must bail out w/ explicit message");
2559      return;
2560    }
2561  }
2562 
2563  print_method(PHASE_OPTIMIZE_FINISHED, 2);
2564  DEBUG_ONLY(set_phase_optimize_finished();)
2565 }
2566 
2567 #ifdef ASSERT
2568 void Compile::check_no_dead_use() const {
2569   ResourceMark rm;
2570   Unique_Node_List wq;
2571   wq.push(root());
2572   for (uint i = 0; i < wq.size(); ++i) {
2573     Node* n = wq.at(i);
2574     for (DUIterator_Fast jmax, j = n->fast_outs(jmax); j < jmax; j++) {
2575       Node* u = n->fast_out(j);
2576       if (u->outcnt() == 0 && !u->is_Con()) {
2577         u->dump();
2578         fatal("no reachable node should have no use");
2579       }
2580       wq.push(u);
2581     }
2582   }
2583 }
2584 #endif
2585 
2586 void Compile::inline_vector_reboxing_calls() {
2587   if (C->_vector_reboxing_late_inlines.length() > 0) {
2588     _late_inlines_pos = C->_late_inlines.length();
2589     while (_vector_reboxing_late_inlines.length() > 0) {
2590       CallGenerator* cg = _vector_reboxing_late_inlines.pop();
2591       cg->do_late_inline();
2592       if (failing())  return;
2593       print_method(PHASE_INLINE_VECTOR_REBOX, 3, cg->call_node());
2594     }
2595     _vector_reboxing_late_inlines.trunc_to(0);
2596   }
2597 }
2598 
2599 bool Compile::has_vbox_nodes() {
2600   if (C->_vector_reboxing_late_inlines.length() > 0) {
2601     return true;
2602   }
2603   for (int macro_idx = C->macro_count() - 1; macro_idx >= 0; macro_idx--) {
2604     Node * n = C->macro_node(macro_idx);
2605     assert(n->is_macro(), "only macro nodes expected here");
2606     if (n->Opcode() == Op_VectorUnbox || n->Opcode() == Op_VectorBox || n->Opcode() == Op_VectorBoxAllocate) {
2607       return true;
2608     }
2609   }
2610   return false;
2611 }
2612 
2613 //---------------------------- Bitwise operation packing optimization ---------------------------
2614 
2615 static bool is_vector_unary_bitwise_op(Node* n) {
2616   return n->Opcode() == Op_XorV &&
2617          VectorNode::is_vector_bitwise_not_pattern(n);
2618 }
2619 
2620 static bool is_vector_binary_bitwise_op(Node* n) {
2621   switch (n->Opcode()) {
2622     case Op_AndV:
2623     case Op_OrV:
2624       return true;
2625 
2626     case Op_XorV:
2627       return !is_vector_unary_bitwise_op(n);
2628 
2629     default:
2630       return false;
2631   }
2632 }
2633 
2634 static bool is_vector_ternary_bitwise_op(Node* n) {
2635   return n->Opcode() == Op_MacroLogicV;
2636 }
2637 
2638 static bool is_vector_bitwise_op(Node* n) {
2639   return is_vector_unary_bitwise_op(n)  ||
2640          is_vector_binary_bitwise_op(n) ||
2641          is_vector_ternary_bitwise_op(n);
2642 }
2643 
2644 static bool is_vector_bitwise_cone_root(Node* n) {
2645   if (n->bottom_type()->isa_vectmask() || !is_vector_bitwise_op(n)) {
2646     return false;
2647   }
2648   for (DUIterator_Fast imax, i = n->fast_outs(imax); i < imax; i++) {
2649     if (is_vector_bitwise_op(n->fast_out(i))) {
2650       return false;
2651     }
2652   }
2653   return true;
2654 }
2655 
2656 static uint collect_unique_inputs(Node* n, Unique_Node_List& inputs) {
2657   uint cnt = 0;
2658   if (is_vector_bitwise_op(n)) {
2659     uint inp_cnt = n->is_predicated_vector() ? n->req()-1 : n->req();
2660     if (VectorNode::is_vector_bitwise_not_pattern(n)) {
2661       for (uint i = 1; i < inp_cnt; i++) {
2662         Node* in = n->in(i);
2663         bool skip = VectorNode::is_all_ones_vector(in);
2664         if (!skip && !inputs.member(in)) {
2665           inputs.push(in);
2666           cnt++;
2667         }
2668       }
2669       assert(cnt <= 1, "not unary");
2670     } else {
2671       uint last_req = inp_cnt;
2672       if (is_vector_ternary_bitwise_op(n)) {
2673         last_req = inp_cnt - 1; // skip last input
2674       }
2675       for (uint i = 1; i < last_req; i++) {
2676         Node* def = n->in(i);
2677         if (!inputs.member(def)) {
2678           inputs.push(def);
2679           cnt++;
2680         }
2681       }
2682     }
2683   } else { // not a bitwise operations
2684     if (!inputs.member(n)) {
2685       inputs.push(n);
2686       cnt++;
2687     }
2688   }
2689   return cnt;
2690 }
2691 
2692 void Compile::collect_logic_cone_roots(Unique_Node_List& list) {
2693   Unique_Node_List useful_nodes;
2694   C->identify_useful_nodes(useful_nodes);
2695 
2696   for (uint i = 0; i < useful_nodes.size(); i++) {
2697     Node* n = useful_nodes.at(i);
2698     if (is_vector_bitwise_cone_root(n)) {
2699       list.push(n);
2700     }
2701   }
2702 }
2703 
2704 Node* Compile::xform_to_MacroLogicV(PhaseIterGVN& igvn,
2705                                     const TypeVect* vt,
2706                                     Unique_Node_List& partition,
2707                                     Unique_Node_List& inputs) {
2708   assert(partition.size() == 2 || partition.size() == 3, "not supported");
2709   assert(inputs.size()    == 2 || inputs.size()    == 3, "not supported");
2710   assert(Matcher::match_rule_supported_vector(Op_MacroLogicV, vt->length(), vt->element_basic_type()), "not supported");
2711 
2712   Node* in1 = inputs.at(0);
2713   Node* in2 = inputs.at(1);
2714   Node* in3 = (inputs.size() == 3 ? inputs.at(2) : in2);
2715 
2716   uint func = compute_truth_table(partition, inputs);
2717 
2718   Node* pn = partition.at(partition.size() - 1);
2719   Node* mask = pn->is_predicated_vector() ? pn->in(pn->req()-1) : nullptr;
2720   return igvn.transform(MacroLogicVNode::make(igvn, in1, in2, in3, mask, func, vt));
2721 }
2722 
2723 static uint extract_bit(uint func, uint pos) {
2724   return (func & (1 << pos)) >> pos;
2725 }
2726 
2727 //
2728 //  A macro logic node represents a truth table. It has 4 inputs,
2729 //  First three inputs corresponds to 3 columns of a truth table
2730 //  and fourth input captures the logic function.
2731 //
2732 //  eg.  fn = (in1 AND in2) OR in3;
2733 //
2734 //      MacroNode(in1,in2,in3,fn)
2735 //
2736 //  -----------------
2737 //  in1 in2 in3  fn
2738 //  -----------------
2739 //  0    0   0    0
2740 //  0    0   1    1
2741 //  0    1   0    0
2742 //  0    1   1    1
2743 //  1    0   0    0
2744 //  1    0   1    1
2745 //  1    1   0    1
2746 //  1    1   1    1
2747 //
2748 
2749 uint Compile::eval_macro_logic_op(uint func, uint in1 , uint in2, uint in3) {
2750   int res = 0;
2751   for (int i = 0; i < 8; i++) {
2752     int bit1 = extract_bit(in1, i);
2753     int bit2 = extract_bit(in2, i);
2754     int bit3 = extract_bit(in3, i);
2755 
2756     int func_bit_pos = (bit1 << 2 | bit2 << 1 | bit3);
2757     int func_bit = extract_bit(func, func_bit_pos);
2758 
2759     res |= func_bit << i;
2760   }
2761   return res;
2762 }
2763 
2764 static uint eval_operand(Node* n, ResourceHashtable<Node*,uint>& eval_map) {
2765   assert(n != nullptr, "");
2766   assert(eval_map.contains(n), "absent");
2767   return *(eval_map.get(n));
2768 }
2769 
2770 static void eval_operands(Node* n,
2771                           uint& func1, uint& func2, uint& func3,
2772                           ResourceHashtable<Node*,uint>& eval_map) {
2773   assert(is_vector_bitwise_op(n), "");
2774 
2775   if (is_vector_unary_bitwise_op(n)) {
2776     Node* opnd = n->in(1);
2777     if (VectorNode::is_vector_bitwise_not_pattern(n) && VectorNode::is_all_ones_vector(opnd)) {
2778       opnd = n->in(2);
2779     }
2780     func1 = eval_operand(opnd, eval_map);
2781   } else if (is_vector_binary_bitwise_op(n)) {
2782     func1 = eval_operand(n->in(1), eval_map);
2783     func2 = eval_operand(n->in(2), eval_map);
2784   } else {
2785     assert(is_vector_ternary_bitwise_op(n), "unknown operation");
2786     func1 = eval_operand(n->in(1), eval_map);
2787     func2 = eval_operand(n->in(2), eval_map);
2788     func3 = eval_operand(n->in(3), eval_map);
2789   }
2790 }
2791 
2792 uint Compile::compute_truth_table(Unique_Node_List& partition, Unique_Node_List& inputs) {
2793   assert(inputs.size() <= 3, "sanity");
2794   ResourceMark rm;
2795   uint res = 0;
2796   ResourceHashtable<Node*,uint> eval_map;
2797 
2798   // Populate precomputed functions for inputs.
2799   // Each input corresponds to one column of 3 input truth-table.
2800   uint input_funcs[] = { 0xAA,   // (_, _, c) -> c
2801                          0xCC,   // (_, b, _) -> b
2802                          0xF0 }; // (a, _, _) -> a
2803   for (uint i = 0; i < inputs.size(); i++) {
2804     eval_map.put(inputs.at(i), input_funcs[2-i]);
2805   }
2806 
2807   for (uint i = 0; i < partition.size(); i++) {
2808     Node* n = partition.at(i);
2809 
2810     uint func1 = 0, func2 = 0, func3 = 0;
2811     eval_operands(n, func1, func2, func3, eval_map);
2812 
2813     switch (n->Opcode()) {
2814       case Op_OrV:
2815         assert(func3 == 0, "not binary");
2816         res = func1 | func2;
2817         break;
2818       case Op_AndV:
2819         assert(func3 == 0, "not binary");
2820         res = func1 & func2;
2821         break;
2822       case Op_XorV:
2823         if (VectorNode::is_vector_bitwise_not_pattern(n)) {
2824           assert(func2 == 0 && func3 == 0, "not unary");
2825           res = (~func1) & 0xFF;
2826         } else {
2827           assert(func3 == 0, "not binary");
2828           res = func1 ^ func2;
2829         }
2830         break;
2831       case Op_MacroLogicV:
2832         // Ordering of inputs may change during evaluation of sub-tree
2833         // containing MacroLogic node as a child node, thus a re-evaluation
2834         // makes sure that function is evaluated in context of current
2835         // inputs.
2836         res = eval_macro_logic_op(n->in(4)->get_int(), func1, func2, func3);
2837         break;
2838 
2839       default: assert(false, "not supported: %s", n->Name());
2840     }
2841     assert(res <= 0xFF, "invalid");
2842     eval_map.put(n, res);
2843   }
2844   return res;
2845 }
2846 
2847 // Criteria under which nodes gets packed into a macro logic node:-
2848 //  1) Parent and both child nodes are all unmasked or masked with
2849 //     same predicates.
2850 //  2) Masked parent can be packed with left child if it is predicated
2851 //     and both have same predicates.
2852 //  3) Masked parent can be packed with right child if its un-predicated
2853 //     or has matching predication condition.
2854 //  4) An unmasked parent can be packed with an unmasked child.
2855 bool Compile::compute_logic_cone(Node* n, Unique_Node_List& partition, Unique_Node_List& inputs) {
2856   assert(partition.size() == 0, "not empty");
2857   assert(inputs.size() == 0, "not empty");
2858   if (is_vector_ternary_bitwise_op(n)) {
2859     return false;
2860   }
2861 
2862   bool is_unary_op = is_vector_unary_bitwise_op(n);
2863   if (is_unary_op) {
2864     assert(collect_unique_inputs(n, inputs) == 1, "not unary");
2865     return false; // too few inputs
2866   }
2867 
2868   bool pack_left_child = true;
2869   bool pack_right_child = true;
2870 
2871   bool left_child_LOP = is_vector_bitwise_op(n->in(1));
2872   bool right_child_LOP = is_vector_bitwise_op(n->in(2));
2873 
2874   int left_child_input_cnt = 0;
2875   int right_child_input_cnt = 0;
2876 
2877   bool parent_is_predicated = n->is_predicated_vector();
2878   bool left_child_predicated = n->in(1)->is_predicated_vector();
2879   bool right_child_predicated = n->in(2)->is_predicated_vector();
2880 
2881   Node* parent_pred = parent_is_predicated ? n->in(n->req()-1) : nullptr;
2882   Node* left_child_pred = left_child_predicated ? n->in(1)->in(n->in(1)->req()-1) : nullptr;
2883   Node* right_child_pred = right_child_predicated ? n->in(1)->in(n->in(1)->req()-1) : nullptr;
2884 
2885   do {
2886     if (pack_left_child && left_child_LOP &&
2887         ((!parent_is_predicated && !left_child_predicated) ||
2888         ((parent_is_predicated && left_child_predicated &&
2889           parent_pred == left_child_pred)))) {
2890        partition.push(n->in(1));
2891        left_child_input_cnt = collect_unique_inputs(n->in(1), inputs);
2892     } else {
2893        inputs.push(n->in(1));
2894        left_child_input_cnt = 1;
2895     }
2896 
2897     if (pack_right_child && right_child_LOP &&
2898         (!right_child_predicated ||
2899          (right_child_predicated && parent_is_predicated &&
2900           parent_pred == right_child_pred))) {
2901        partition.push(n->in(2));
2902        right_child_input_cnt = collect_unique_inputs(n->in(2), inputs);
2903     } else {
2904        inputs.push(n->in(2));
2905        right_child_input_cnt = 1;
2906     }
2907 
2908     if (inputs.size() > 3) {
2909       assert(partition.size() > 0, "");
2910       inputs.clear();
2911       partition.clear();
2912       if (left_child_input_cnt > right_child_input_cnt) {
2913         pack_left_child = false;
2914       } else {
2915         pack_right_child = false;
2916       }
2917     } else {
2918       break;
2919     }
2920   } while(true);
2921 
2922   if(partition.size()) {
2923     partition.push(n);
2924   }
2925 
2926   return (partition.size() == 2 || partition.size() == 3) &&
2927          (inputs.size()    == 2 || inputs.size()    == 3);
2928 }
2929 
2930 void Compile::process_logic_cone_root(PhaseIterGVN &igvn, Node *n, VectorSet &visited) {
2931   assert(is_vector_bitwise_op(n), "not a root");
2932 
2933   visited.set(n->_idx);
2934 
2935   // 1) Do a DFS walk over the logic cone.
2936   for (uint i = 1; i < n->req(); i++) {
2937     Node* in = n->in(i);
2938     if (!visited.test(in->_idx) && is_vector_bitwise_op(in)) {
2939       process_logic_cone_root(igvn, in, visited);
2940     }
2941   }
2942 
2943   // 2) Bottom up traversal: Merge node[s] with
2944   // the parent to form macro logic node.
2945   Unique_Node_List partition;
2946   Unique_Node_List inputs;
2947   if (compute_logic_cone(n, partition, inputs)) {
2948     const TypeVect* vt = n->bottom_type()->is_vect();
2949     Node* pn = partition.at(partition.size() - 1);
2950     Node* mask = pn->is_predicated_vector() ? pn->in(pn->req()-1) : nullptr;
2951     if (mask == nullptr ||
2952         Matcher::match_rule_supported_vector_masked(Op_MacroLogicV, vt->length(), vt->element_basic_type())) {
2953       Node* macro_logic = xform_to_MacroLogicV(igvn, vt, partition, inputs);
2954       VectorNode::trace_new_vector(macro_logic, "MacroLogic");
2955       igvn.replace_node(n, macro_logic);
2956     }
2957   }
2958 }
2959 
2960 void Compile::optimize_logic_cones(PhaseIterGVN &igvn) {
2961   ResourceMark rm;
2962   if (Matcher::match_rule_supported(Op_MacroLogicV)) {
2963     Unique_Node_List list;
2964     collect_logic_cone_roots(list);
2965 
2966     while (list.size() > 0) {
2967       Node* n = list.pop();
2968       const TypeVect* vt = n->bottom_type()->is_vect();
2969       bool supported = Matcher::match_rule_supported_vector(Op_MacroLogicV, vt->length(), vt->element_basic_type());
2970       if (supported) {
2971         VectorSet visited(comp_arena());
2972         process_logic_cone_root(igvn, n, visited);
2973       }
2974     }
2975   }
2976 }
2977 
2978 //------------------------------Code_Gen---------------------------------------
2979 // Given a graph, generate code for it
2980 void Compile::Code_Gen() {
2981   if (failing()) {
2982     return;
2983   }
2984 
2985   // Perform instruction selection.  You might think we could reclaim Matcher
2986   // memory PDQ, but actually the Matcher is used in generating spill code.
2987   // Internals of the Matcher (including some VectorSets) must remain live
2988   // for awhile - thus I cannot reclaim Matcher memory lest a VectorSet usage
2989   // set a bit in reclaimed memory.
2990 
2991   // In debug mode can dump m._nodes.dump() for mapping of ideal to machine
2992   // nodes.  Mapping is only valid at the root of each matched subtree.
2993   NOT_PRODUCT( verify_graph_edges(); )
2994 
2995   Matcher matcher;
2996   _matcher = &matcher;
2997   {
2998     TracePhase tp(_t_matcher);
2999     matcher.match();
3000     if (failing()) {
3001       return;
3002     }
3003   }
3004   // In debug mode can dump m._nodes.dump() for mapping of ideal to machine
3005   // nodes.  Mapping is only valid at the root of each matched subtree.
3006   NOT_PRODUCT( verify_graph_edges(); )
3007 
3008   // If you have too many nodes, or if matching has failed, bail out
3009   check_node_count(0, "out of nodes matching instructions");
3010   if (failing()) {
3011     return;
3012   }
3013 
3014   print_method(PHASE_MATCHING, 2);
3015 
3016   // Build a proper-looking CFG
3017   PhaseCFG cfg(node_arena(), root(), matcher);
3018   if (failing()) {
3019     return;
3020   }
3021   _cfg = &cfg;
3022   {
3023     TracePhase tp(_t_scheduler);
3024     bool success = cfg.do_global_code_motion();
3025     if (!success) {
3026       return;
3027     }
3028 
3029     print_method(PHASE_GLOBAL_CODE_MOTION, 2);
3030     NOT_PRODUCT( verify_graph_edges(); )
3031     cfg.verify();
3032     if (failing()) {
3033       return;
3034     }
3035   }
3036 
3037   PhaseChaitin regalloc(unique(), cfg, matcher, false);
3038   _regalloc = &regalloc;
3039   {
3040     TracePhase tp(_t_registerAllocation);
3041     // Perform register allocation.  After Chaitin, use-def chains are
3042     // no longer accurate (at spill code) and so must be ignored.
3043     // Node->LRG->reg mappings are still accurate.
3044     _regalloc->Register_Allocate();
3045 
3046     // Bail out if the allocator builds too many nodes
3047     if (failing()) {
3048       return;
3049     }
3050 
3051     print_method(PHASE_REGISTER_ALLOCATION, 2);
3052   }
3053 
3054   // Prior to register allocation we kept empty basic blocks in case the
3055   // the allocator needed a place to spill.  After register allocation we
3056   // are not adding any new instructions.  If any basic block is empty, we
3057   // can now safely remove it.
3058   {
3059     TracePhase tp(_t_blockOrdering);
3060     cfg.remove_empty_blocks();
3061     if (do_freq_based_layout()) {
3062       PhaseBlockLayout layout(cfg);
3063     } else {
3064       cfg.set_loop_alignment();
3065     }
3066     cfg.fixup_flow();
3067     cfg.remove_unreachable_blocks();
3068     cfg.verify_dominator_tree();
3069     print_method(PHASE_BLOCK_ORDERING, 3);
3070   }
3071 
3072   // Apply peephole optimizations
3073   if( OptoPeephole ) {
3074     TracePhase tp(_t_peephole);
3075     PhasePeephole peep( _regalloc, cfg);
3076     peep.do_transform();
3077     print_method(PHASE_PEEPHOLE, 3);
3078   }
3079 
3080   // Do late expand if CPU requires this.
3081   if (Matcher::require_postalloc_expand) {
3082     TracePhase tp(_t_postalloc_expand);
3083     cfg.postalloc_expand(_regalloc);
3084     print_method(PHASE_POSTALLOC_EXPAND, 3);
3085   }
3086 
3087 #ifdef ASSERT
3088   {
3089     CompilationMemoryStatistic::do_test_allocations();
3090     if (failing()) return;
3091   }
3092 #endif
3093 
3094   // Convert Nodes to instruction bits in a buffer
3095   {
3096     TracePhase tp(_t_output);
3097     PhaseOutput output;
3098     output.Output();
3099     if (failing())  return;
3100     output.install();
3101     print_method(PHASE_FINAL_CODE, 1); // Compile::_output is not null here
3102   }
3103 
3104   // He's dead, Jim.
3105   _cfg     = (PhaseCFG*)((intptr_t)0xdeadbeef);
3106   _regalloc = (PhaseChaitin*)((intptr_t)0xdeadbeef);
3107 }
3108 
3109 //------------------------------Final_Reshape_Counts---------------------------
3110 // This class defines counters to help identify when a method
3111 // may/must be executed using hardware with only 24-bit precision.
3112 struct Final_Reshape_Counts : public StackObj {
3113   int  _call_count;             // count non-inlined 'common' calls
3114   int  _float_count;            // count float ops requiring 24-bit precision
3115   int  _double_count;           // count double ops requiring more precision
3116   int  _java_call_count;        // count non-inlined 'java' calls
3117   int  _inner_loop_count;       // count loops which need alignment
3118   VectorSet _visited;           // Visitation flags
3119   Node_List _tests;             // Set of IfNodes & PCTableNodes
3120 
3121   Final_Reshape_Counts() :
3122     _call_count(0), _float_count(0), _double_count(0),
3123     _java_call_count(0), _inner_loop_count(0) { }
3124 
3125   void inc_call_count  () { _call_count  ++; }
3126   void inc_float_count () { _float_count ++; }
3127   void inc_double_count() { _double_count++; }
3128   void inc_java_call_count() { _java_call_count++; }
3129   void inc_inner_loop_count() { _inner_loop_count++; }
3130 
3131   int  get_call_count  () const { return _call_count  ; }
3132   int  get_float_count () const { return _float_count ; }
3133   int  get_double_count() const { return _double_count; }
3134   int  get_java_call_count() const { return _java_call_count; }
3135   int  get_inner_loop_count() const { return _inner_loop_count; }
3136 };
3137 
3138 //------------------------------final_graph_reshaping_impl----------------------
3139 // Implement items 1-5 from final_graph_reshaping below.
3140 void Compile::final_graph_reshaping_impl(Node *n, Final_Reshape_Counts& frc, Unique_Node_List& dead_nodes) {
3141 
3142   if ( n->outcnt() == 0 ) return; // dead node
3143   uint nop = n->Opcode();
3144 
3145   // Check for 2-input instruction with "last use" on right input.
3146   // Swap to left input.  Implements item (2).
3147   if( n->req() == 3 &&          // two-input instruction
3148       n->in(1)->outcnt() > 1 && // left use is NOT a last use
3149       (!n->in(1)->is_Phi() || n->in(1)->in(2) != n) && // it is not data loop
3150       n->in(2)->outcnt() == 1 &&// right use IS a last use
3151       !n->in(2)->is_Con() ) {   // right use is not a constant
3152     // Check for commutative opcode
3153     switch( nop ) {
3154     case Op_AddI:  case Op_AddF:  case Op_AddD:  case Op_AddL:
3155     case Op_MaxI:  case Op_MaxL:  case Op_MaxF:  case Op_MaxD:
3156     case Op_MinI:  case Op_MinL:  case Op_MinF:  case Op_MinD:
3157     case Op_MulI:  case Op_MulF:  case Op_MulD:  case Op_MulL:
3158     case Op_AndL:  case Op_XorL:  case Op_OrL:
3159     case Op_AndI:  case Op_XorI:  case Op_OrI: {
3160       // Move "last use" input to left by swapping inputs
3161       n->swap_edges(1, 2);
3162       break;
3163     }
3164     default:
3165       break;
3166     }
3167   }
3168 
3169 #ifdef ASSERT
3170   if( n->is_Mem() ) {
3171     int alias_idx = get_alias_index(n->as_Mem()->adr_type());
3172     assert( n->in(0) != nullptr || alias_idx != Compile::AliasIdxRaw ||
3173             // oop will be recorded in oop map if load crosses safepoint
3174             (n->is_Load() && (n->as_Load()->bottom_type()->isa_oopptr() ||
3175                               LoadNode::is_immutable_value(n->in(MemNode::Address)))),
3176             "raw memory operations should have control edge");
3177   }
3178   if (n->is_MemBar()) {
3179     MemBarNode* mb = n->as_MemBar();
3180     if (mb->trailing_store() || mb->trailing_load_store()) {
3181       assert(mb->leading_membar()->trailing_membar() == mb, "bad membar pair");
3182       Node* mem = BarrierSet::barrier_set()->barrier_set_c2()->step_over_gc_barrier(mb->in(MemBarNode::Precedent));
3183       assert((mb->trailing_store() && mem->is_Store() && mem->as_Store()->is_release()) ||
3184              (mb->trailing_load_store() && mem->is_LoadStore()), "missing mem op");
3185     } else if (mb->leading()) {
3186       assert(mb->trailing_membar()->leading_membar() == mb, "bad membar pair");
3187     }
3188   }
3189 #endif
3190   // Count FPU ops and common calls, implements item (3)
3191   bool gc_handled = BarrierSet::barrier_set()->barrier_set_c2()->final_graph_reshaping(this, n, nop, dead_nodes);
3192   if (!gc_handled) {
3193     final_graph_reshaping_main_switch(n, frc, nop, dead_nodes);
3194   }
3195 
3196   // Collect CFG split points
3197   if (n->is_MultiBranch() && !n->is_RangeCheck()) {
3198     frc._tests.push(n);
3199   }
3200 }
3201 
3202 void Compile::handle_div_mod_op(Node* n, BasicType bt, bool is_unsigned) {
3203   if (!UseDivMod) {
3204     return;
3205   }
3206 
3207   // Check if "a % b" and "a / b" both exist
3208   Node* d = n->find_similar(Op_DivIL(bt, is_unsigned));
3209   if (d == nullptr) {
3210     return;
3211   }
3212 
3213   // Replace them with a fused divmod if supported
3214   if (Matcher::has_match_rule(Op_DivModIL(bt, is_unsigned))) {
3215     DivModNode* divmod = DivModNode::make(n, bt, is_unsigned);
3216     // If the divisor input for a Div (or Mod etc.) is not zero, then the control input of the Div is set to zero.
3217     // It could be that the divisor input is found not zero because its type is narrowed down by a CastII in the
3218     // subgraph for that input. Range check CastIIs are removed during final graph reshape. To preserve the dependency
3219     // carried by a CastII, precedence edges are added to the Div node. We need to transfer the precedence edges to the
3220     // DivMod node so the dependency is not lost.
3221     divmod->add_prec_from(n);
3222     divmod->add_prec_from(d);
3223     d->subsume_by(divmod->div_proj(), this);
3224     n->subsume_by(divmod->mod_proj(), this);
3225   } else {
3226     // Replace "a % b" with "a - ((a / b) * b)"
3227     Node* mult = MulNode::make(d, d->in(2), bt);
3228     Node* sub = SubNode::make(d->in(1), mult, bt);
3229     n->subsume_by(sub, this);
3230   }
3231 }
3232 
3233 void Compile::final_graph_reshaping_main_switch(Node* n, Final_Reshape_Counts& frc, uint nop, Unique_Node_List& dead_nodes) {
3234   switch( nop ) {
3235   // Count all float operations that may use FPU
3236   case Op_AddF:
3237   case Op_SubF:
3238   case Op_MulF:
3239   case Op_DivF:
3240   case Op_NegF:
3241   case Op_ModF:
3242   case Op_ConvI2F:
3243   case Op_ConF:
3244   case Op_CmpF:
3245   case Op_CmpF3:
3246   case Op_StoreF:
3247   case Op_LoadF:
3248   // case Op_ConvL2F: // longs are split into 32-bit halves
3249     frc.inc_float_count();
3250     break;
3251 
3252   case Op_ConvF2D:
3253   case Op_ConvD2F:
3254     frc.inc_float_count();
3255     frc.inc_double_count();
3256     break;
3257 
3258   // Count all double operations that may use FPU
3259   case Op_AddD:
3260   case Op_SubD:
3261   case Op_MulD:
3262   case Op_DivD:
3263   case Op_NegD:
3264   case Op_ModD:
3265   case Op_ConvI2D:
3266   case Op_ConvD2I:
3267   // case Op_ConvL2D: // handled by leaf call
3268   // case Op_ConvD2L: // handled by leaf call
3269   case Op_ConD:
3270   case Op_CmpD:
3271   case Op_CmpD3:
3272   case Op_StoreD:
3273   case Op_LoadD:
3274   case Op_LoadD_unaligned:
3275     frc.inc_double_count();
3276     break;
3277   case Op_Opaque1:              // Remove Opaque Nodes before matching
3278     n->subsume_by(n->in(1), this);
3279     break;
3280   case Op_CallStaticJava:
3281   case Op_CallJava:
3282   case Op_CallDynamicJava:
3283     frc.inc_java_call_count(); // Count java call site;
3284   case Op_CallRuntime:
3285   case Op_CallLeaf:
3286   case Op_CallLeafVector:
3287   case Op_CallLeafNoFP: {
3288     assert (n->is_Call(), "");
3289     CallNode *call = n->as_Call();
3290     // Count call sites where the FP mode bit would have to be flipped.
3291     // Do not count uncommon runtime calls:
3292     // uncommon_trap, _complete_monitor_locking, _complete_monitor_unlocking,
3293     // _new_Java, _new_typeArray, _new_objArray, _rethrow_Java, ...
3294     if (!call->is_CallStaticJava() || !call->as_CallStaticJava()->_name) {
3295       frc.inc_call_count();   // Count the call site
3296     } else {                  // See if uncommon argument is shared
3297       Node *n = call->in(TypeFunc::Parms);
3298       int nop = n->Opcode();
3299       // Clone shared simple arguments to uncommon calls, item (1).
3300       if (n->outcnt() > 1 &&
3301           !n->is_Proj() &&
3302           nop != Op_CreateEx &&
3303           nop != Op_CheckCastPP &&
3304           nop != Op_DecodeN &&
3305           nop != Op_DecodeNKlass &&
3306           !n->is_Mem() &&
3307           !n->is_Phi()) {
3308         Node *x = n->clone();
3309         call->set_req(TypeFunc::Parms, x);
3310       }
3311     }
3312     break;
3313   }
3314   case Op_StoreB:
3315   case Op_StoreC:
3316   case Op_StoreI:
3317   case Op_StoreL:
3318   case Op_CompareAndSwapB:
3319   case Op_CompareAndSwapS:
3320   case Op_CompareAndSwapI:
3321   case Op_CompareAndSwapL:
3322   case Op_CompareAndSwapP:
3323   case Op_CompareAndSwapN:
3324   case Op_WeakCompareAndSwapB:
3325   case Op_WeakCompareAndSwapS:
3326   case Op_WeakCompareAndSwapI:
3327   case Op_WeakCompareAndSwapL:
3328   case Op_WeakCompareAndSwapP:
3329   case Op_WeakCompareAndSwapN:
3330   case Op_CompareAndExchangeB:
3331   case Op_CompareAndExchangeS:
3332   case Op_CompareAndExchangeI:
3333   case Op_CompareAndExchangeL:
3334   case Op_CompareAndExchangeP:
3335   case Op_CompareAndExchangeN:
3336   case Op_GetAndAddS:
3337   case Op_GetAndAddB:
3338   case Op_GetAndAddI:
3339   case Op_GetAndAddL:
3340   case Op_GetAndSetS:
3341   case Op_GetAndSetB:
3342   case Op_GetAndSetI:
3343   case Op_GetAndSetL:
3344   case Op_GetAndSetP:
3345   case Op_GetAndSetN:
3346   case Op_StoreP:
3347   case Op_StoreN:
3348   case Op_StoreNKlass:
3349   case Op_LoadB:
3350   case Op_LoadUB:
3351   case Op_LoadUS:
3352   case Op_LoadI:
3353   case Op_LoadKlass:
3354   case Op_LoadNKlass:
3355   case Op_LoadL:
3356   case Op_LoadL_unaligned:
3357   case Op_LoadP:
3358   case Op_LoadN:
3359   case Op_LoadRange:
3360   case Op_LoadS:
3361     break;
3362 
3363   case Op_AddP: {               // Assert sane base pointers
3364     Node *addp = n->in(AddPNode::Address);
3365     assert( !addp->is_AddP() ||
3366             addp->in(AddPNode::Base)->is_top() || // Top OK for allocation
3367             addp->in(AddPNode::Base) == n->in(AddPNode::Base),
3368             "Base pointers must match (addp %u)", addp->_idx );
3369 #ifdef _LP64
3370     if ((UseCompressedOops || UseCompressedClassPointers) &&
3371         addp->Opcode() == Op_ConP &&
3372         addp == n->in(AddPNode::Base) &&
3373         n->in(AddPNode::Offset)->is_Con()) {
3374       // If the transformation of ConP to ConN+DecodeN is beneficial depends
3375       // on the platform and on the compressed oops mode.
3376       // Use addressing with narrow klass to load with offset on x86.
3377       // Some platforms can use the constant pool to load ConP.
3378       // Do this transformation here since IGVN will convert ConN back to ConP.
3379       const Type* t = addp->bottom_type();
3380       bool is_oop   = t->isa_oopptr() != nullptr;
3381       bool is_klass = t->isa_klassptr() != nullptr;
3382 
3383       if ((is_oop   && UseCompressedOops          && Matcher::const_oop_prefer_decode()  ) ||
3384           (is_klass && UseCompressedClassPointers && Matcher::const_klass_prefer_decode() &&
3385            t->isa_klassptr()->exact_klass()->is_in_encoding_range())) {
3386         Node* nn = nullptr;
3387 
3388         int op = is_oop ? Op_ConN : Op_ConNKlass;
3389 
3390         // Look for existing ConN node of the same exact type.
3391         Node* r  = root();
3392         uint cnt = r->outcnt();
3393         for (uint i = 0; i < cnt; i++) {
3394           Node* m = r->raw_out(i);
3395           if (m!= nullptr && m->Opcode() == op &&
3396               m->bottom_type()->make_ptr() == t) {
3397             nn = m;
3398             break;
3399           }
3400         }
3401         if (nn != nullptr) {
3402           // Decode a narrow oop to match address
3403           // [R12 + narrow_oop_reg<<3 + offset]
3404           if (is_oop) {
3405             nn = new DecodeNNode(nn, t);
3406           } else {
3407             nn = new DecodeNKlassNode(nn, t);
3408           }
3409           // Check for succeeding AddP which uses the same Base.
3410           // Otherwise we will run into the assertion above when visiting that guy.
3411           for (uint i = 0; i < n->outcnt(); ++i) {
3412             Node *out_i = n->raw_out(i);
3413             if (out_i && out_i->is_AddP() && out_i->in(AddPNode::Base) == addp) {
3414               out_i->set_req(AddPNode::Base, nn);
3415 #ifdef ASSERT
3416               for (uint j = 0; j < out_i->outcnt(); ++j) {
3417                 Node *out_j = out_i->raw_out(j);
3418                 assert(out_j == nullptr || !out_j->is_AddP() || out_j->in(AddPNode::Base) != addp,
3419                        "more than 2 AddP nodes in a chain (out_j %u)", out_j->_idx);
3420               }
3421 #endif
3422             }
3423           }
3424           n->set_req(AddPNode::Base, nn);
3425           n->set_req(AddPNode::Address, nn);
3426           if (addp->outcnt() == 0) {
3427             addp->disconnect_inputs(this);
3428           }
3429         }
3430       }
3431     }
3432 #endif
3433     break;
3434   }
3435 
3436   case Op_CastPP: {
3437     // Remove CastPP nodes to gain more freedom during scheduling but
3438     // keep the dependency they encode as control or precedence edges
3439     // (if control is set already) on memory operations. Some CastPP
3440     // nodes don't have a control (don't carry a dependency): skip
3441     // those.
3442     if (n->in(0) != nullptr) {
3443       ResourceMark rm;
3444       Unique_Node_List wq;
3445       wq.push(n);
3446       for (uint next = 0; next < wq.size(); ++next) {
3447         Node *m = wq.at(next);
3448         for (DUIterator_Fast imax, i = m->fast_outs(imax); i < imax; i++) {
3449           Node* use = m->fast_out(i);
3450           if (use->is_Mem() || use->is_EncodeNarrowPtr()) {
3451             use->ensure_control_or_add_prec(n->in(0));
3452           } else {
3453             switch(use->Opcode()) {
3454             case Op_AddP:
3455             case Op_DecodeN:
3456             case Op_DecodeNKlass:
3457             case Op_CheckCastPP:
3458             case Op_CastPP:
3459               wq.push(use);
3460               break;
3461             }
3462           }
3463         }
3464       }
3465     }
3466     const bool is_LP64 = LP64_ONLY(true) NOT_LP64(false);
3467     if (is_LP64 && n->in(1)->is_DecodeN() && Matcher::gen_narrow_oop_implicit_null_checks()) {
3468       Node* in1 = n->in(1);
3469       const Type* t = n->bottom_type();
3470       Node* new_in1 = in1->clone();
3471       new_in1->as_DecodeN()->set_type(t);
3472 
3473       if (!Matcher::narrow_oop_use_complex_address()) {
3474         //
3475         // x86, ARM and friends can handle 2 adds in addressing mode
3476         // and Matcher can fold a DecodeN node into address by using
3477         // a narrow oop directly and do implicit null check in address:
3478         //
3479         // [R12 + narrow_oop_reg<<3 + offset]
3480         // NullCheck narrow_oop_reg
3481         //
3482         // On other platforms (Sparc) we have to keep new DecodeN node and
3483         // use it to do implicit null check in address:
3484         //
3485         // decode_not_null narrow_oop_reg, base_reg
3486         // [base_reg + offset]
3487         // NullCheck base_reg
3488         //
3489         // Pin the new DecodeN node to non-null path on these platform (Sparc)
3490         // to keep the information to which null check the new DecodeN node
3491         // corresponds to use it as value in implicit_null_check().
3492         //
3493         new_in1->set_req(0, n->in(0));
3494       }
3495 
3496       n->subsume_by(new_in1, this);
3497       if (in1->outcnt() == 0) {
3498         in1->disconnect_inputs(this);
3499       }
3500     } else {
3501       n->subsume_by(n->in(1), this);
3502       if (n->outcnt() == 0) {
3503         n->disconnect_inputs(this);
3504       }
3505     }
3506     break;
3507   }
3508   case Op_CastII: {
3509     n->as_CastII()->remove_range_check_cast(this);
3510     break;
3511   }
3512 #ifdef _LP64
3513   case Op_CmpP:
3514     // Do this transformation here to preserve CmpPNode::sub() and
3515     // other TypePtr related Ideal optimizations (for example, ptr nullness).
3516     if (n->in(1)->is_DecodeNarrowPtr() || n->in(2)->is_DecodeNarrowPtr()) {
3517       Node* in1 = n->in(1);
3518       Node* in2 = n->in(2);
3519       if (!in1->is_DecodeNarrowPtr()) {
3520         in2 = in1;
3521         in1 = n->in(2);
3522       }
3523       assert(in1->is_DecodeNarrowPtr(), "sanity");
3524 
3525       Node* new_in2 = nullptr;
3526       if (in2->is_DecodeNarrowPtr()) {
3527         assert(in2->Opcode() == in1->Opcode(), "must be same node type");
3528         new_in2 = in2->in(1);
3529       } else if (in2->Opcode() == Op_ConP) {
3530         const Type* t = in2->bottom_type();
3531         if (t == TypePtr::NULL_PTR) {
3532           assert(in1->is_DecodeN(), "compare klass to null?");
3533           // Don't convert CmpP null check into CmpN if compressed
3534           // oops implicit null check is not generated.
3535           // This will allow to generate normal oop implicit null check.
3536           if (Matcher::gen_narrow_oop_implicit_null_checks())
3537             new_in2 = ConNode::make(TypeNarrowOop::NULL_PTR);
3538           //
3539           // This transformation together with CastPP transformation above
3540           // will generated code for implicit null checks for compressed oops.
3541           //
3542           // The original code after Optimize()
3543           //
3544           //    LoadN memory, narrow_oop_reg
3545           //    decode narrow_oop_reg, base_reg
3546           //    CmpP base_reg, nullptr
3547           //    CastPP base_reg // NotNull
3548           //    Load [base_reg + offset], val_reg
3549           //
3550           // after these transformations will be
3551           //
3552           //    LoadN memory, narrow_oop_reg
3553           //    CmpN narrow_oop_reg, nullptr
3554           //    decode_not_null narrow_oop_reg, base_reg
3555           //    Load [base_reg + offset], val_reg
3556           //
3557           // and the uncommon path (== nullptr) will use narrow_oop_reg directly
3558           // since narrow oops can be used in debug info now (see the code in
3559           // final_graph_reshaping_walk()).
3560           //
3561           // At the end the code will be matched to
3562           // on x86:
3563           //
3564           //    Load_narrow_oop memory, narrow_oop_reg
3565           //    Load [R12 + narrow_oop_reg<<3 + offset], val_reg
3566           //    NullCheck narrow_oop_reg
3567           //
3568           // and on sparc:
3569           //
3570           //    Load_narrow_oop memory, narrow_oop_reg
3571           //    decode_not_null narrow_oop_reg, base_reg
3572           //    Load [base_reg + offset], val_reg
3573           //    NullCheck base_reg
3574           //
3575         } else if (t->isa_oopptr()) {
3576           new_in2 = ConNode::make(t->make_narrowoop());
3577         } else if (t->isa_klassptr()) {
3578           new_in2 = ConNode::make(t->make_narrowklass());
3579         }
3580       }
3581       if (new_in2 != nullptr) {
3582         Node* cmpN = new CmpNNode(in1->in(1), new_in2);
3583         n->subsume_by(cmpN, this);
3584         if (in1->outcnt() == 0) {
3585           in1->disconnect_inputs(this);
3586         }
3587         if (in2->outcnt() == 0) {
3588           in2->disconnect_inputs(this);
3589         }
3590       }
3591     }
3592     break;
3593 
3594   case Op_DecodeN:
3595   case Op_DecodeNKlass:
3596     assert(!n->in(1)->is_EncodeNarrowPtr(), "should be optimized out");
3597     // DecodeN could be pinned when it can't be fold into
3598     // an address expression, see the code for Op_CastPP above.
3599     assert(n->in(0) == nullptr || (UseCompressedOops && !Matcher::narrow_oop_use_complex_address()), "no control");
3600     break;
3601 
3602   case Op_EncodeP:
3603   case Op_EncodePKlass: {
3604     Node* in1 = n->in(1);
3605     if (in1->is_DecodeNarrowPtr()) {
3606       n->subsume_by(in1->in(1), this);
3607     } else if (in1->Opcode() == Op_ConP) {
3608       const Type* t = in1->bottom_type();
3609       if (t == TypePtr::NULL_PTR) {
3610         assert(t->isa_oopptr(), "null klass?");
3611         n->subsume_by(ConNode::make(TypeNarrowOop::NULL_PTR), this);
3612       } else if (t->isa_oopptr()) {
3613         n->subsume_by(ConNode::make(t->make_narrowoop()), this);
3614       } else if (t->isa_klassptr()) {
3615         n->subsume_by(ConNode::make(t->make_narrowklass()), this);
3616       }
3617     }
3618     if (in1->outcnt() == 0) {
3619       in1->disconnect_inputs(this);
3620     }
3621     break;
3622   }
3623 
3624   case Op_Proj: {
3625     if (OptimizeStringConcat || IncrementalInline) {
3626       ProjNode* proj = n->as_Proj();
3627       if (proj->_is_io_use) {
3628         assert(proj->_con == TypeFunc::I_O || proj->_con == TypeFunc::Memory, "");
3629         // Separate projections were used for the exception path which
3630         // are normally removed by a late inline.  If it wasn't inlined
3631         // then they will hang around and should just be replaced with
3632         // the original one. Merge them.
3633         Node* non_io_proj = proj->in(0)->as_Multi()->proj_out_or_null(proj->_con, false /*is_io_use*/);
3634         if (non_io_proj  != nullptr) {
3635           proj->subsume_by(non_io_proj , this);
3636         }
3637       }
3638     }
3639     break;
3640   }
3641 
3642   case Op_Phi:
3643     if (n->as_Phi()->bottom_type()->isa_narrowoop() || n->as_Phi()->bottom_type()->isa_narrowklass()) {
3644       // The EncodeP optimization may create Phi with the same edges
3645       // for all paths. It is not handled well by Register Allocator.
3646       Node* unique_in = n->in(1);
3647       assert(unique_in != nullptr, "");
3648       uint cnt = n->req();
3649       for (uint i = 2; i < cnt; i++) {
3650         Node* m = n->in(i);
3651         assert(m != nullptr, "");
3652         if (unique_in != m)
3653           unique_in = nullptr;
3654       }
3655       if (unique_in != nullptr) {
3656         n->subsume_by(unique_in, this);
3657       }
3658     }
3659     break;
3660 
3661 #endif
3662 
3663   case Op_ModI:
3664     handle_div_mod_op(n, T_INT, false);
3665     break;
3666 
3667   case Op_ModL:
3668     handle_div_mod_op(n, T_LONG, false);
3669     break;
3670 
3671   case Op_UModI:
3672     handle_div_mod_op(n, T_INT, true);
3673     break;
3674 
3675   case Op_UModL:
3676     handle_div_mod_op(n, T_LONG, true);
3677     break;
3678 
3679   case Op_LoadVector:
3680   case Op_StoreVector:
3681 #ifdef ASSERT
3682     // Add VerifyVectorAlignment node between adr and load / store.
3683     if (VerifyAlignVector && Matcher::has_match_rule(Op_VerifyVectorAlignment)) {
3684       bool must_verify_alignment = n->is_LoadVector() ? n->as_LoadVector()->must_verify_alignment() :
3685                                                         n->as_StoreVector()->must_verify_alignment();
3686       if (must_verify_alignment) {
3687         jlong vector_width = n->is_LoadVector() ? n->as_LoadVector()->memory_size() :
3688                                                   n->as_StoreVector()->memory_size();
3689         // The memory access should be aligned to the vector width in bytes.
3690         // However, the underlying array is possibly less well aligned, but at least
3691         // to ObjectAlignmentInBytes. Hence, even if multiple arrays are accessed in
3692         // a loop we can expect at least the following alignment:
3693         jlong guaranteed_alignment = MIN2(vector_width, (jlong)ObjectAlignmentInBytes);
3694         assert(2 <= guaranteed_alignment && guaranteed_alignment <= 64, "alignment must be in range");
3695         assert(is_power_of_2(guaranteed_alignment), "alignment must be power of 2");
3696         // Create mask from alignment. e.g. 0b1000 -> 0b0111
3697         jlong mask = guaranteed_alignment - 1;
3698         Node* mask_con = ConLNode::make(mask);
3699         VerifyVectorAlignmentNode* va = new VerifyVectorAlignmentNode(n->in(MemNode::Address), mask_con);
3700         n->set_req(MemNode::Address, va);
3701       }
3702     }
3703 #endif
3704     break;
3705 
3706   case Op_LoadVectorGather:
3707   case Op_StoreVectorScatter:
3708   case Op_LoadVectorGatherMasked:
3709   case Op_StoreVectorScatterMasked:
3710   case Op_VectorCmpMasked:
3711   case Op_VectorMaskGen:
3712   case Op_LoadVectorMasked:
3713   case Op_StoreVectorMasked:
3714     break;
3715 
3716   case Op_AddReductionVI:
3717   case Op_AddReductionVL:
3718   case Op_AddReductionVF:
3719   case Op_AddReductionVD:
3720   case Op_MulReductionVI:
3721   case Op_MulReductionVL:
3722   case Op_MulReductionVF:
3723   case Op_MulReductionVD:
3724   case Op_MinReductionV:
3725   case Op_MaxReductionV:
3726   case Op_AndReductionV:
3727   case Op_OrReductionV:
3728   case Op_XorReductionV:
3729     break;
3730 
3731   case Op_PackB:
3732   case Op_PackS:
3733   case Op_PackI:
3734   case Op_PackF:
3735   case Op_PackL:
3736   case Op_PackD:
3737     if (n->req()-1 > 2) {
3738       // Replace many operand PackNodes with a binary tree for matching
3739       PackNode* p = (PackNode*) n;
3740       Node* btp = p->binary_tree_pack(1, n->req());
3741       n->subsume_by(btp, this);
3742     }
3743     break;
3744   case Op_Loop:
3745     assert(!n->as_Loop()->is_loop_nest_inner_loop() || _loop_opts_cnt == 0, "should have been turned into a counted loop");
3746   case Op_CountedLoop:
3747   case Op_LongCountedLoop:
3748   case Op_OuterStripMinedLoop:
3749     if (n->as_Loop()->is_inner_loop()) {
3750       frc.inc_inner_loop_count();
3751     }
3752     n->as_Loop()->verify_strip_mined(0);
3753     break;
3754   case Op_LShiftI:
3755   case Op_RShiftI:
3756   case Op_URShiftI:
3757   case Op_LShiftL:
3758   case Op_RShiftL:
3759   case Op_URShiftL:
3760     if (Matcher::need_masked_shift_count) {
3761       // The cpu's shift instructions don't restrict the count to the
3762       // lower 5/6 bits. We need to do the masking ourselves.
3763       Node* in2 = n->in(2);
3764       juint mask = (n->bottom_type() == TypeInt::INT) ? (BitsPerInt - 1) : (BitsPerLong - 1);
3765       const TypeInt* t = in2->find_int_type();
3766       if (t != nullptr && t->is_con()) {
3767         juint shift = t->get_con();
3768         if (shift > mask) { // Unsigned cmp
3769           n->set_req(2, ConNode::make(TypeInt::make(shift & mask)));
3770         }
3771       } else {
3772         if (t == nullptr || t->_lo < 0 || t->_hi > (int)mask) {
3773           Node* shift = new AndINode(in2, ConNode::make(TypeInt::make(mask)));
3774           n->set_req(2, shift);
3775         }
3776       }
3777       if (in2->outcnt() == 0) { // Remove dead node
3778         in2->disconnect_inputs(this);
3779       }
3780     }
3781     break;
3782   case Op_MemBarStoreStore:
3783   case Op_MemBarRelease:
3784     // Break the link with AllocateNode: it is no longer useful and
3785     // confuses register allocation.
3786     if (n->req() > MemBarNode::Precedent) {
3787       n->set_req(MemBarNode::Precedent, top());
3788     }
3789     break;
3790   case Op_MemBarAcquire: {
3791     if (n->as_MemBar()->trailing_load() && n->req() > MemBarNode::Precedent) {
3792       // At parse time, the trailing MemBarAcquire for a volatile load
3793       // is created with an edge to the load. After optimizations,
3794       // that input may be a chain of Phis. If those phis have no
3795       // other use, then the MemBarAcquire keeps them alive and
3796       // register allocation can be confused.
3797       dead_nodes.push(n->in(MemBarNode::Precedent));
3798       n->set_req(MemBarNode::Precedent, top());
3799     }
3800     break;
3801   }
3802   case Op_Blackhole:
3803     break;
3804   case Op_RangeCheck: {
3805     RangeCheckNode* rc = n->as_RangeCheck();
3806     Node* iff = new IfNode(rc->in(0), rc->in(1), rc->_prob, rc->_fcnt);
3807     n->subsume_by(iff, this);
3808     frc._tests.push(iff);
3809     break;
3810   }
3811   case Op_ConvI2L: {
3812     if (!Matcher::convi2l_type_required) {
3813       // Code generation on some platforms doesn't need accurate
3814       // ConvI2L types. Widening the type can help remove redundant
3815       // address computations.
3816       n->as_Type()->set_type(TypeLong::INT);
3817       ResourceMark rm;
3818       Unique_Node_List wq;
3819       wq.push(n);
3820       for (uint next = 0; next < wq.size(); next++) {
3821         Node *m = wq.at(next);
3822 
3823         for(;;) {
3824           // Loop over all nodes with identical inputs edges as m
3825           Node* k = m->find_similar(m->Opcode());
3826           if (k == nullptr) {
3827             break;
3828           }
3829           // Push their uses so we get a chance to remove node made
3830           // redundant
3831           for (DUIterator_Fast imax, i = k->fast_outs(imax); i < imax; i++) {
3832             Node* u = k->fast_out(i);
3833             if (u->Opcode() == Op_LShiftL ||
3834                 u->Opcode() == Op_AddL ||
3835                 u->Opcode() == Op_SubL ||
3836                 u->Opcode() == Op_AddP) {
3837               wq.push(u);
3838             }
3839           }
3840           // Replace all nodes with identical edges as m with m
3841           k->subsume_by(m, this);
3842         }
3843       }
3844     }
3845     break;
3846   }
3847   case Op_CmpUL: {
3848     if (!Matcher::has_match_rule(Op_CmpUL)) {
3849       // No support for unsigned long comparisons
3850       ConINode* sign_pos = new ConINode(TypeInt::make(BitsPerLong - 1));
3851       Node* sign_bit_mask = new RShiftLNode(n->in(1), sign_pos);
3852       Node* orl = new OrLNode(n->in(1), sign_bit_mask);
3853       ConLNode* remove_sign_mask = new ConLNode(TypeLong::make(max_jlong));
3854       Node* andl = new AndLNode(orl, remove_sign_mask);
3855       Node* cmp = new CmpLNode(andl, n->in(2));
3856       n->subsume_by(cmp, this);
3857     }
3858     break;
3859   }
3860 #ifdef ASSERT
3861   case Op_ConNKlass: {
3862     const TypePtr* tp = n->as_Type()->type()->make_ptr();
3863     ciKlass* klass = tp->is_klassptr()->exact_klass();
3864     assert(klass->is_in_encoding_range(), "klass cannot be compressed");
3865     break;
3866   }
3867 #endif
3868   default:
3869     assert(!n->is_Call(), "");
3870     assert(!n->is_Mem(), "");
3871     assert(nop != Op_ProfileBoolean, "should be eliminated during IGVN");
3872     break;
3873   }
3874 }
3875 
3876 //------------------------------final_graph_reshaping_walk---------------------
3877 // Replacing Opaque nodes with their input in final_graph_reshaping_impl(),
3878 // requires that the walk visits a node's inputs before visiting the node.
3879 void Compile::final_graph_reshaping_walk(Node_Stack& nstack, Node* root, Final_Reshape_Counts& frc, Unique_Node_List& dead_nodes) {
3880   Unique_Node_List sfpt;
3881 
3882   frc._visited.set(root->_idx); // first, mark node as visited
3883   uint cnt = root->req();
3884   Node *n = root;
3885   uint  i = 0;
3886   while (true) {
3887     if (i < cnt) {
3888       // Place all non-visited non-null inputs onto stack
3889       Node* m = n->in(i);
3890       ++i;
3891       if (m != nullptr && !frc._visited.test_set(m->_idx)) {
3892         if (m->is_SafePoint() && m->as_SafePoint()->jvms() != nullptr) {
3893           // compute worst case interpreter size in case of a deoptimization
3894           update_interpreter_frame_size(m->as_SafePoint()->jvms()->interpreter_frame_size());
3895 
3896           sfpt.push(m);
3897         }
3898         cnt = m->req();
3899         nstack.push(n, i); // put on stack parent and next input's index
3900         n = m;
3901         i = 0;
3902       }
3903     } else {
3904       // Now do post-visit work
3905       final_graph_reshaping_impl(n, frc, dead_nodes);
3906       if (nstack.is_empty())
3907         break;             // finished
3908       n = nstack.node();   // Get node from stack
3909       cnt = n->req();
3910       i = nstack.index();
3911       nstack.pop();        // Shift to the next node on stack
3912     }
3913   }
3914 
3915   // Skip next transformation if compressed oops are not used.
3916   if ((UseCompressedOops && !Matcher::gen_narrow_oop_implicit_null_checks()) ||
3917       (!UseCompressedOops && !UseCompressedClassPointers))
3918     return;
3919 
3920   // Go over safepoints nodes to skip DecodeN/DecodeNKlass nodes for debug edges.
3921   // It could be done for an uncommon traps or any safepoints/calls
3922   // if the DecodeN/DecodeNKlass node is referenced only in a debug info.
3923   while (sfpt.size() > 0) {
3924     n = sfpt.pop();
3925     JVMState *jvms = n->as_SafePoint()->jvms();
3926     assert(jvms != nullptr, "sanity");
3927     int start = jvms->debug_start();
3928     int end   = n->req();
3929     bool is_uncommon = (n->is_CallStaticJava() &&
3930                         n->as_CallStaticJava()->uncommon_trap_request() != 0);
3931     for (int j = start; j < end; j++) {
3932       Node* in = n->in(j);
3933       if (in->is_DecodeNarrowPtr()) {
3934         bool safe_to_skip = true;
3935         if (!is_uncommon ) {
3936           // Is it safe to skip?
3937           for (uint i = 0; i < in->outcnt(); i++) {
3938             Node* u = in->raw_out(i);
3939             if (!u->is_SafePoint() ||
3940                 (u->is_Call() && u->as_Call()->has_non_debug_use(n))) {
3941               safe_to_skip = false;
3942             }
3943           }
3944         }
3945         if (safe_to_skip) {
3946           n->set_req(j, in->in(1));
3947         }
3948         if (in->outcnt() == 0) {
3949           in->disconnect_inputs(this);
3950         }
3951       }
3952     }
3953   }
3954 }
3955 
3956 //------------------------------final_graph_reshaping--------------------------
3957 // Final Graph Reshaping.
3958 //
3959 // (1) Clone simple inputs to uncommon calls, so they can be scheduled late
3960 //     and not commoned up and forced early.  Must come after regular
3961 //     optimizations to avoid GVN undoing the cloning.  Clone constant
3962 //     inputs to Loop Phis; these will be split by the allocator anyways.
3963 //     Remove Opaque nodes.
3964 // (2) Move last-uses by commutative operations to the left input to encourage
3965 //     Intel update-in-place two-address operations and better register usage
3966 //     on RISCs.  Must come after regular optimizations to avoid GVN Ideal
3967 //     calls canonicalizing them back.
3968 // (3) Count the number of double-precision FP ops, single-precision FP ops
3969 //     and call sites.  On Intel, we can get correct rounding either by
3970 //     forcing singles to memory (requires extra stores and loads after each
3971 //     FP bytecode) or we can set a rounding mode bit (requires setting and
3972 //     clearing the mode bit around call sites).  The mode bit is only used
3973 //     if the relative frequency of single FP ops to calls is low enough.
3974 //     This is a key transform for SPEC mpeg_audio.
3975 // (4) Detect infinite loops; blobs of code reachable from above but not
3976 //     below.  Several of the Code_Gen algorithms fail on such code shapes,
3977 //     so we simply bail out.  Happens a lot in ZKM.jar, but also happens
3978 //     from time to time in other codes (such as -Xcomp finalizer loops, etc).
3979 //     Detection is by looking for IfNodes where only 1 projection is
3980 //     reachable from below or CatchNodes missing some targets.
3981 // (5) Assert for insane oop offsets in debug mode.
3982 
3983 bool Compile::final_graph_reshaping() {
3984   // an infinite loop may have been eliminated by the optimizer,
3985   // in which case the graph will be empty.
3986   if (root()->req() == 1) {
3987     // Do not compile method that is only a trivial infinite loop,
3988     // since the content of the loop may have been eliminated.
3989     record_method_not_compilable("trivial infinite loop");
3990     return true;
3991   }
3992 
3993   // Expensive nodes have their control input set to prevent the GVN
3994   // from freely commoning them. There's no GVN beyond this point so
3995   // no need to keep the control input. We want the expensive nodes to
3996   // be freely moved to the least frequent code path by gcm.
3997   assert(OptimizeExpensiveOps || expensive_count() == 0, "optimization off but list non empty?");
3998   for (int i = 0; i < expensive_count(); i++) {
3999     _expensive_nodes.at(i)->set_req(0, nullptr);
4000   }
4001 
4002   Final_Reshape_Counts frc;
4003 
4004   // Visit everybody reachable!
4005   // Allocate stack of size C->live_nodes()/2 to avoid frequent realloc
4006   Node_Stack nstack(live_nodes() >> 1);
4007   Unique_Node_List dead_nodes;
4008   final_graph_reshaping_walk(nstack, root(), frc, dead_nodes);
4009 
4010   // Check for unreachable (from below) code (i.e., infinite loops).
4011   for( uint i = 0; i < frc._tests.size(); i++ ) {
4012     MultiBranchNode *n = frc._tests[i]->as_MultiBranch();
4013     // Get number of CFG targets.
4014     // Note that PCTables include exception targets after calls.
4015     uint required_outcnt = n->required_outcnt();
4016     if (n->outcnt() != required_outcnt) {
4017       // Check for a few special cases.  Rethrow Nodes never take the
4018       // 'fall-thru' path, so expected kids is 1 less.
4019       if (n->is_PCTable() && n->in(0) && n->in(0)->in(0)) {
4020         if (n->in(0)->in(0)->is_Call()) {
4021           CallNode* call = n->in(0)->in(0)->as_Call();
4022           if (call->entry_point() == OptoRuntime::rethrow_stub()) {
4023             required_outcnt--;      // Rethrow always has 1 less kid
4024           } else if (call->req() > TypeFunc::Parms &&
4025                      call->is_CallDynamicJava()) {
4026             // Check for null receiver. In such case, the optimizer has
4027             // detected that the virtual call will always result in a null
4028             // pointer exception. The fall-through projection of this CatchNode
4029             // will not be populated.
4030             Node* arg0 = call->in(TypeFunc::Parms);
4031             if (arg0->is_Type() &&
4032                 arg0->as_Type()->type()->higher_equal(TypePtr::NULL_PTR)) {
4033               required_outcnt--;
4034             }
4035           } else if (call->entry_point() == OptoRuntime::new_array_Java() ||
4036                      call->entry_point() == OptoRuntime::new_array_nozero_Java()) {
4037             // Check for illegal array length. In such case, the optimizer has
4038             // detected that the allocation attempt will always result in an
4039             // exception. There is no fall-through projection of this CatchNode .
4040             assert(call->is_CallStaticJava(), "static call expected");
4041             assert(call->req() == call->jvms()->endoff() + 1, "missing extra input");
4042             uint valid_length_test_input = call->req() - 1;
4043             Node* valid_length_test = call->in(valid_length_test_input);
4044             call->del_req(valid_length_test_input);
4045             if (valid_length_test->find_int_con(1) == 0) {
4046               required_outcnt--;
4047             }
4048             dead_nodes.push(valid_length_test);
4049             assert(n->outcnt() == required_outcnt, "malformed control flow");
4050             continue;
4051           }
4052         }
4053       }
4054 
4055       // Recheck with a better notion of 'required_outcnt'
4056       if (n->outcnt() != required_outcnt) {
4057         record_method_not_compilable("malformed control flow");
4058         return true;            // Not all targets reachable!
4059       }
4060     } else if (n->is_PCTable() && n->in(0) && n->in(0)->in(0) && n->in(0)->in(0)->is_Call()) {
4061       CallNode* call = n->in(0)->in(0)->as_Call();
4062       if (call->entry_point() == OptoRuntime::new_array_Java() ||
4063           call->entry_point() == OptoRuntime::new_array_nozero_Java()) {
4064         assert(call->is_CallStaticJava(), "static call expected");
4065         assert(call->req() == call->jvms()->endoff() + 1, "missing extra input");
4066         uint valid_length_test_input = call->req() - 1;
4067         dead_nodes.push(call->in(valid_length_test_input));
4068         call->del_req(valid_length_test_input); // valid length test useless now
4069       }
4070     }
4071     // Check that I actually visited all kids.  Unreached kids
4072     // must be infinite loops.
4073     for (DUIterator_Fast jmax, j = n->fast_outs(jmax); j < jmax; j++)
4074       if (!frc._visited.test(n->fast_out(j)->_idx)) {
4075         record_method_not_compilable("infinite loop");
4076         return true;            // Found unvisited kid; must be unreach
4077       }
4078 
4079     // Here so verification code in final_graph_reshaping_walk()
4080     // always see an OuterStripMinedLoopEnd
4081     if (n->is_OuterStripMinedLoopEnd() || n->is_LongCountedLoopEnd()) {
4082       IfNode* init_iff = n->as_If();
4083       Node* iff = new IfNode(init_iff->in(0), init_iff->in(1), init_iff->_prob, init_iff->_fcnt);
4084       n->subsume_by(iff, this);
4085     }
4086   }
4087 
4088   while (dead_nodes.size() > 0) {
4089     Node* m = dead_nodes.pop();
4090     if (m->outcnt() == 0 && m != top()) {
4091       for (uint j = 0; j < m->req(); j++) {
4092         Node* in = m->in(j);
4093         if (in != nullptr) {
4094           dead_nodes.push(in);
4095         }
4096       }
4097       m->disconnect_inputs(this);
4098     }
4099   }
4100 
4101   set_java_calls(frc.get_java_call_count());
4102   set_inner_loops(frc.get_inner_loop_count());
4103 
4104   // No infinite loops, no reason to bail out.
4105   return false;
4106 }
4107 
4108 //-----------------------------too_many_traps----------------------------------
4109 // Report if there are too many traps at the current method and bci.
4110 // Return true if there was a trap, and/or PerMethodTrapLimit is exceeded.
4111 bool Compile::too_many_traps(ciMethod* method,
4112                              int bci,
4113                              Deoptimization::DeoptReason reason) {
4114   ciMethodData* md = method->method_data();
4115   if (md->is_empty()) {
4116     // Assume the trap has not occurred, or that it occurred only
4117     // because of a transient condition during start-up in the interpreter.
4118     return false;
4119   }
4120   ciMethod* m = Deoptimization::reason_is_speculate(reason) ? this->method() : nullptr;
4121   if (md->has_trap_at(bci, m, reason) != 0) {
4122     // Assume PerBytecodeTrapLimit==0, for a more conservative heuristic.
4123     // Also, if there are multiple reasons, or if there is no per-BCI record,
4124     // assume the worst.
4125     if (log())
4126       log()->elem("observe trap='%s' count='%d'",
4127                   Deoptimization::trap_reason_name(reason),
4128                   md->trap_count(reason));
4129     return true;
4130   } else {
4131     // Ignore method/bci and see if there have been too many globally.
4132     return too_many_traps(reason, md);
4133   }
4134 }
4135 
4136 // Less-accurate variant which does not require a method and bci.
4137 bool Compile::too_many_traps(Deoptimization::DeoptReason reason,
4138                              ciMethodData* logmd) {
4139   if (trap_count(reason) >= Deoptimization::per_method_trap_limit(reason)) {
4140     // Too many traps globally.
4141     // Note that we use cumulative trap_count, not just md->trap_count.
4142     if (log()) {
4143       int mcount = (logmd == nullptr)? -1: (int)logmd->trap_count(reason);
4144       log()->elem("observe trap='%s' count='0' mcount='%d' ccount='%d'",
4145                   Deoptimization::trap_reason_name(reason),
4146                   mcount, trap_count(reason));
4147     }
4148     return true;
4149   } else {
4150     // The coast is clear.
4151     return false;
4152   }
4153 }
4154 
4155 //--------------------------too_many_recompiles--------------------------------
4156 // Report if there are too many recompiles at the current method and bci.
4157 // Consults PerBytecodeRecompilationCutoff and PerMethodRecompilationCutoff.
4158 // Is not eager to return true, since this will cause the compiler to use
4159 // Action_none for a trap point, to avoid too many recompilations.
4160 bool Compile::too_many_recompiles(ciMethod* method,
4161                                   int bci,
4162                                   Deoptimization::DeoptReason reason) {
4163   ciMethodData* md = method->method_data();
4164   if (md->is_empty()) {
4165     // Assume the trap has not occurred, or that it occurred only
4166     // because of a transient condition during start-up in the interpreter.
4167     return false;
4168   }
4169   // Pick a cutoff point well within PerBytecodeRecompilationCutoff.
4170   uint bc_cutoff = (uint) PerBytecodeRecompilationCutoff / 8;
4171   uint m_cutoff  = (uint) PerMethodRecompilationCutoff / 2 + 1;  // not zero
4172   Deoptimization::DeoptReason per_bc_reason
4173     = Deoptimization::reason_recorded_per_bytecode_if_any(reason);
4174   ciMethod* m = Deoptimization::reason_is_speculate(reason) ? this->method() : nullptr;
4175   if ((per_bc_reason == Deoptimization::Reason_none
4176        || md->has_trap_at(bci, m, reason) != 0)
4177       // The trap frequency measure we care about is the recompile count:
4178       && md->trap_recompiled_at(bci, m)
4179       && md->overflow_recompile_count() >= bc_cutoff) {
4180     // Do not emit a trap here if it has already caused recompilations.
4181     // Also, if there are multiple reasons, or if there is no per-BCI record,
4182     // assume the worst.
4183     if (log())
4184       log()->elem("observe trap='%s recompiled' count='%d' recompiles2='%d'",
4185                   Deoptimization::trap_reason_name(reason),
4186                   md->trap_count(reason),
4187                   md->overflow_recompile_count());
4188     return true;
4189   } else if (trap_count(reason) != 0
4190              && decompile_count() >= m_cutoff) {
4191     // Too many recompiles globally, and we have seen this sort of trap.
4192     // Use cumulative decompile_count, not just md->decompile_count.
4193     if (log())
4194       log()->elem("observe trap='%s' count='%d' mcount='%d' decompiles='%d' mdecompiles='%d'",
4195                   Deoptimization::trap_reason_name(reason),
4196                   md->trap_count(reason), trap_count(reason),
4197                   md->decompile_count(), decompile_count());
4198     return true;
4199   } else {
4200     // The coast is clear.
4201     return false;
4202   }
4203 }
4204 
4205 // Compute when not to trap. Used by matching trap based nodes and
4206 // NullCheck optimization.
4207 void Compile::set_allowed_deopt_reasons() {
4208   _allowed_reasons = 0;
4209   if (is_method_compilation()) {
4210     for (int rs = (int)Deoptimization::Reason_none+1; rs < Compile::trapHistLength; rs++) {
4211       assert(rs < BitsPerInt, "recode bit map");
4212       if (!too_many_traps((Deoptimization::DeoptReason) rs)) {
4213         _allowed_reasons |= nth_bit(rs);
4214       }
4215     }
4216   }
4217 }
4218 
4219 bool Compile::needs_clinit_barrier(ciMethod* method, ciMethod* accessing_method) {
4220   return method->is_static() && needs_clinit_barrier(method->holder(), accessing_method);
4221 }
4222 
4223 bool Compile::needs_clinit_barrier(ciField* field, ciMethod* accessing_method) {
4224   return field->is_static() && needs_clinit_barrier(field->holder(), accessing_method);
4225 }
4226 
4227 bool Compile::needs_clinit_barrier(ciInstanceKlass* holder, ciMethod* accessing_method) {
4228   if (holder->is_initialized()) {
4229     return false;
4230   }
4231   if (holder->is_being_initialized()) {
4232     if (accessing_method->holder() == holder) {
4233       // Access inside a class. The barrier can be elided when access happens in <clinit>,
4234       // <init>, or a static method. In all those cases, there was an initialization
4235       // barrier on the holder klass passed.
4236       if (accessing_method->is_static_initializer() ||
4237           accessing_method->is_object_initializer() ||
4238           accessing_method->is_static()) {
4239         return false;
4240       }
4241     } else if (accessing_method->holder()->is_subclass_of(holder)) {
4242       // Access from a subclass. The barrier can be elided only when access happens in <clinit>.
4243       // In case of <init> or a static method, the barrier is on the subclass is not enough:
4244       // child class can become fully initialized while its parent class is still being initialized.
4245       if (accessing_method->is_static_initializer()) {
4246         return false;
4247       }
4248     }
4249     ciMethod* root = method(); // the root method of compilation
4250     if (root != accessing_method) {
4251       return needs_clinit_barrier(holder, root); // check access in the context of compilation root
4252     }
4253   }
4254   return true;
4255 }
4256 
4257 #ifndef PRODUCT
4258 //------------------------------verify_bidirectional_edges---------------------
4259 // For each input edge to a node (ie - for each Use-Def edge), verify that
4260 // there is a corresponding Def-Use edge.
4261 void Compile::verify_bidirectional_edges(Unique_Node_List& visited, const Unique_Node_List* root_and_safepoints) const {
4262   // Allocate stack of size C->live_nodes()/16 to avoid frequent realloc
4263   uint stack_size = live_nodes() >> 4;
4264   Node_List nstack(MAX2(stack_size, (uint) OptoNodeListSize));
4265   if (root_and_safepoints != nullptr) {
4266     assert(root_and_safepoints->member(_root), "root is not in root_and_safepoints");
4267     for (uint i = 0, limit = root_and_safepoints->size(); i < limit; i++) {
4268       Node* root_or_safepoint = root_and_safepoints->at(i);
4269       // If the node is a safepoint, let's check if it still has a control input
4270       // Lack of control input signifies that this node was killed by CCP or
4271       // recursively by remove_globally_dead_node and it shouldn't be a starting
4272       // point.
4273       if (!root_or_safepoint->is_SafePoint() || root_or_safepoint->in(0) != nullptr) {
4274         nstack.push(root_or_safepoint);
4275       }
4276     }
4277   } else {
4278     nstack.push(_root);
4279   }
4280 
4281   while (nstack.size() > 0) {
4282     Node* n = nstack.pop();
4283     if (visited.member(n)) {
4284       continue;
4285     }
4286     visited.push(n);
4287 
4288     // Walk over all input edges, checking for correspondence
4289     uint length = n->len();
4290     for (uint i = 0; i < length; i++) {
4291       Node* in = n->in(i);
4292       if (in != nullptr && !visited.member(in)) {
4293         nstack.push(in); // Put it on stack
4294       }
4295       if (in != nullptr && !in->is_top()) {
4296         // Count instances of `next`
4297         int cnt = 0;
4298         for (uint idx = 0; idx < in->_outcnt; idx++) {
4299           if (in->_out[idx] == n) {
4300             cnt++;
4301           }
4302         }
4303         assert(cnt > 0, "Failed to find Def-Use edge.");
4304         // Check for duplicate edges
4305         // walk the input array downcounting the input edges to n
4306         for (uint j = 0; j < length; j++) {
4307           if (n->in(j) == in) {
4308             cnt--;
4309           }
4310         }
4311         assert(cnt == 0, "Mismatched edge count.");
4312       } else if (in == nullptr) {
4313         assert(i == 0 || i >= n->req() ||
4314                n->is_Region() || n->is_Phi() || n->is_ArrayCopy() ||
4315                (n->is_Unlock() && i == (n->req() - 1)) ||
4316                (n->is_MemBar() && i == 5), // the precedence edge to a membar can be removed during macro node expansion
4317               "only region, phi, arraycopy, unlock or membar nodes have null data edges");
4318       } else {
4319         assert(in->is_top(), "sanity");
4320         // Nothing to check.
4321       }
4322     }
4323   }
4324 }
4325 
4326 //------------------------------verify_graph_edges---------------------------
4327 // Walk the Graph and verify that there is a one-to-one correspondence
4328 // between Use-Def edges and Def-Use edges in the graph.
4329 void Compile::verify_graph_edges(bool no_dead_code, const Unique_Node_List* root_and_safepoints) const {
4330   if (VerifyGraphEdges) {
4331     Unique_Node_List visited;
4332 
4333     // Call graph walk to check edges
4334     verify_bidirectional_edges(visited, root_and_safepoints);
4335     if (no_dead_code) {
4336       // Now make sure that no visited node is used by an unvisited node.
4337       bool dead_nodes = false;
4338       Unique_Node_List checked;
4339       while (visited.size() > 0) {
4340         Node* n = visited.pop();
4341         checked.push(n);
4342         for (uint i = 0; i < n->outcnt(); i++) {
4343           Node* use = n->raw_out(i);
4344           if (checked.member(use))  continue;  // already checked
4345           if (visited.member(use))  continue;  // already in the graph
4346           if (use->is_Con())        continue;  // a dead ConNode is OK
4347           // At this point, we have found a dead node which is DU-reachable.
4348           if (!dead_nodes) {
4349             tty->print_cr("*** Dead nodes reachable via DU edges:");
4350             dead_nodes = true;
4351           }
4352           use->dump(2);
4353           tty->print_cr("---");
4354           checked.push(use);  // No repeats; pretend it is now checked.
4355         }
4356       }
4357       assert(!dead_nodes, "using nodes must be reachable from root");
4358     }
4359   }
4360 }
4361 #endif
4362 
4363 // The Compile object keeps track of failure reasons separately from the ciEnv.
4364 // This is required because there is not quite a 1-1 relation between the
4365 // ciEnv and its compilation task and the Compile object.  Note that one
4366 // ciEnv might use two Compile objects, if C2Compiler::compile_method decides
4367 // to backtrack and retry without subsuming loads.  Other than this backtracking
4368 // behavior, the Compile's failure reason is quietly copied up to the ciEnv
4369 // by the logic in C2Compiler.
4370 void Compile::record_failure(const char* reason DEBUG_ONLY(COMMA bool allow_multiple_failures)) {
4371   if (log() != nullptr) {
4372     log()->elem("failure reason='%s' phase='compile'", reason);
4373   }
4374   if (_failure_reason.get() == nullptr) {
4375     // Record the first failure reason.
4376     _failure_reason.set(reason);
4377     if (CaptureBailoutInformation) {
4378       _first_failure_details = new CompilationFailureInfo(reason);
4379     }
4380   } else {
4381     assert(!StressBailout || allow_multiple_failures, "should have handled previous failure.");
4382   }
4383 
4384   if (!C->failure_reason_is(C2Compiler::retry_no_subsuming_loads())) {
4385     C->print_method(PHASE_FAILURE, 1);
4386   }
4387   _root = nullptr;  // flush the graph, too
4388 }
4389 
4390 Compile::TracePhase::TracePhase(const char* name, PhaseTraceId id)
4391   : TraceTime(name, &Phase::timers[id], CITime, CITimeVerbose),
4392     _compile(Compile::current()),
4393     _log(nullptr),
4394     _dolog(CITimeVerbose)
4395 {
4396   assert(_compile != nullptr, "sanity check");
4397   assert(id != PhaseTraceId::_t_none, "Don't use none");
4398   if (_dolog) {
4399     _log = _compile->log();
4400   }
4401   if (_log != nullptr) {
4402     _log->begin_head("phase name='%s' nodes='%d' live='%d'", phase_name(), _compile->unique(), _compile->live_nodes());
4403     _log->stamp();
4404     _log->end_head();
4405   }
4406 
4407   // Inform memory statistic, if enabled
4408   if (CompilationMemoryStatistic::enabled()) {
4409     CompilationMemoryStatistic::on_phase_start((int)id, name);
4410   }
4411 }
4412 
4413 Compile::TracePhase::TracePhase(PhaseTraceId id)
4414   : TracePhase(Phase::get_phase_trace_id_text(id), id) {}
4415 
4416 Compile::TracePhase::~TracePhase() {
4417 
4418   // Inform memory statistic, if enabled
4419   if (CompilationMemoryStatistic::enabled()) {
4420     CompilationMemoryStatistic::on_phase_end();
4421   }
4422 
4423   if (_compile->failing_internal()) {
4424     if (_log != nullptr) {
4425       _log->done("phase");
4426     }
4427     return; // timing code, not stressing bailouts.
4428   }
4429 #ifdef ASSERT
4430   if (PrintIdealNodeCount) {
4431     tty->print_cr("phase name='%s' nodes='%d' live='%d' live_graph_walk='%d'",
4432                   phase_name(), _compile->unique(), _compile->live_nodes(), _compile->count_live_nodes_by_graph_walk());
4433   }
4434 
4435   if (VerifyIdealNodeCount) {
4436     _compile->print_missing_nodes();
4437   }
4438 #endif
4439 
4440   if (_log != nullptr) {
4441     _log->done("phase name='%s' nodes='%d' live='%d'", phase_name(), _compile->unique(), _compile->live_nodes());
4442   }
4443 }
4444 
4445 //----------------------------static_subtype_check-----------------------------
4446 // Shortcut important common cases when superklass is exact:
4447 // (0) superklass is java.lang.Object (can occur in reflective code)
4448 // (1) subklass is already limited to a subtype of superklass => always ok
4449 // (2) subklass does not overlap with superklass => always fail
4450 // (3) superklass has NO subtypes and we can check with a simple compare.
4451 Compile::SubTypeCheckResult Compile::static_subtype_check(const TypeKlassPtr* superk, const TypeKlassPtr* subk, bool skip) {
4452   if (skip) {
4453     return SSC_full_test;       // Let caller generate the general case.
4454   }
4455 
4456   if (subk->is_java_subtype_of(superk)) {
4457     return SSC_always_true; // (0) and (1)  this test cannot fail
4458   }
4459 
4460   if (!subk->maybe_java_subtype_of(superk)) {
4461     return SSC_always_false; // (2) true path dead; no dynamic test needed
4462   }
4463 
4464   const Type* superelem = superk;
4465   if (superk->isa_aryklassptr()) {
4466     int ignored;
4467     superelem = superk->is_aryklassptr()->base_element_type(ignored);
4468   }
4469 
4470   if (superelem->isa_instklassptr()) {
4471     ciInstanceKlass* ik = superelem->is_instklassptr()->instance_klass();
4472     if (!ik->has_subklass()) {
4473       if (!ik->is_final()) {
4474         // Add a dependency if there is a chance of a later subclass.
4475         dependencies()->assert_leaf_type(ik);
4476       }
4477       if (!superk->maybe_java_subtype_of(subk)) {
4478         return SSC_always_false;
4479       }
4480       return SSC_easy_test;     // (3) caller can do a simple ptr comparison
4481     }
4482   } else {
4483     // A primitive array type has no subtypes.
4484     return SSC_easy_test;       // (3) caller can do a simple ptr comparison
4485   }
4486 
4487   return SSC_full_test;
4488 }
4489 
4490 Node* Compile::conv_I2X_index(PhaseGVN* phase, Node* idx, const TypeInt* sizetype, Node* ctrl) {
4491 #ifdef _LP64
4492   // The scaled index operand to AddP must be a clean 64-bit value.
4493   // Java allows a 32-bit int to be incremented to a negative
4494   // value, which appears in a 64-bit register as a large
4495   // positive number.  Using that large positive number as an
4496   // operand in pointer arithmetic has bad consequences.
4497   // On the other hand, 32-bit overflow is rare, and the possibility
4498   // can often be excluded, if we annotate the ConvI2L node with
4499   // a type assertion that its value is known to be a small positive
4500   // number.  (The prior range check has ensured this.)
4501   // This assertion is used by ConvI2LNode::Ideal.
4502   int index_max = max_jint - 1;  // array size is max_jint, index is one less
4503   if (sizetype != nullptr) index_max = sizetype->_hi - 1;
4504   const TypeInt* iidxtype = TypeInt::make(0, index_max, Type::WidenMax);
4505   idx = constrained_convI2L(phase, idx, iidxtype, ctrl);
4506 #endif
4507   return idx;
4508 }
4509 
4510 // Convert integer value to a narrowed long type dependent on ctrl (for example, a range check)
4511 Node* Compile::constrained_convI2L(PhaseGVN* phase, Node* value, const TypeInt* itype, Node* ctrl, bool carry_dependency) {
4512   if (ctrl != nullptr) {
4513     // Express control dependency by a CastII node with a narrow type.
4514     // Make the CastII node dependent on the control input to prevent the narrowed ConvI2L
4515     // node from floating above the range check during loop optimizations. Otherwise, the
4516     // ConvI2L node may be eliminated independently of the range check, causing the data path
4517     // to become TOP while the control path is still there (although it's unreachable).
4518     value = new CastIINode(ctrl, value, itype, carry_dependency ? ConstraintCastNode::StrongDependency : ConstraintCastNode::RegularDependency, true /* range check dependency */);
4519     value = phase->transform(value);
4520   }
4521   const TypeLong* ltype = TypeLong::make(itype->_lo, itype->_hi, itype->_widen);
4522   return phase->transform(new ConvI2LNode(value, ltype));
4523 }
4524 
4525 void Compile::dump_print_inlining() {
4526   inline_printer()->print_on(tty);
4527 }
4528 
4529 void Compile::log_late_inline(CallGenerator* cg) {
4530   if (log() != nullptr) {
4531     log()->head("late_inline method='%d'  inline_id='" JLONG_FORMAT "'", log()->identify(cg->method()),
4532                 cg->unique_id());
4533     JVMState* p = cg->call_node()->jvms();
4534     while (p != nullptr) {
4535       log()->elem("jvms bci='%d' method='%d'", p->bci(), log()->identify(p->method()));
4536       p = p->caller();
4537     }
4538     log()->tail("late_inline");
4539   }
4540 }
4541 
4542 void Compile::log_late_inline_failure(CallGenerator* cg, const char* msg) {
4543   log_late_inline(cg);
4544   if (log() != nullptr) {
4545     log()->inline_fail(msg);
4546   }
4547 }
4548 
4549 void Compile::log_inline_id(CallGenerator* cg) {
4550   if (log() != nullptr) {
4551     // The LogCompilation tool needs a unique way to identify late
4552     // inline call sites. This id must be unique for this call site in
4553     // this compilation. Try to have it unique across compilations as
4554     // well because it can be convenient when grepping through the log
4555     // file.
4556     // Distinguish OSR compilations from others in case CICountOSR is
4557     // on.
4558     jlong id = ((jlong)unique()) + (((jlong)compile_id()) << 33) + (CICountOSR && is_osr_compilation() ? ((jlong)1) << 32 : 0);
4559     cg->set_unique_id(id);
4560     log()->elem("inline_id id='" JLONG_FORMAT "'", id);
4561   }
4562 }
4563 
4564 void Compile::log_inline_failure(const char* msg) {
4565   if (C->log() != nullptr) {
4566     C->log()->inline_fail(msg);
4567   }
4568 }
4569 
4570 
4571 // Dump inlining replay data to the stream.
4572 // Don't change thread state and acquire any locks.
4573 void Compile::dump_inline_data(outputStream* out) {
4574   InlineTree* inl_tree = ilt();
4575   if (inl_tree != nullptr) {
4576     out->print(" inline %d", inl_tree->count());
4577     inl_tree->dump_replay_data(out);
4578   }
4579 }
4580 
4581 void Compile::dump_inline_data_reduced(outputStream* out) {
4582   assert(ReplayReduce, "");
4583 
4584   InlineTree* inl_tree = ilt();
4585   if (inl_tree == nullptr) {
4586     return;
4587   }
4588   // Enable iterative replay file reduction
4589   // Output "compile" lines for depth 1 subtrees,
4590   // simulating that those trees were compiled
4591   // instead of inlined.
4592   for (int i = 0; i < inl_tree->subtrees().length(); ++i) {
4593     InlineTree* sub = inl_tree->subtrees().at(i);
4594     if (sub->inline_level() != 1) {
4595       continue;
4596     }
4597 
4598     ciMethod* method = sub->method();
4599     int entry_bci = -1;
4600     int comp_level = env()->task()->comp_level();
4601     out->print("compile ");
4602     method->dump_name_as_ascii(out);
4603     out->print(" %d %d", entry_bci, comp_level);
4604     out->print(" inline %d", sub->count());
4605     sub->dump_replay_data(out, -1);
4606     out->cr();
4607   }
4608 }
4609 
4610 int Compile::cmp_expensive_nodes(Node* n1, Node* n2) {
4611   if (n1->Opcode() < n2->Opcode())      return -1;
4612   else if (n1->Opcode() > n2->Opcode()) return 1;
4613 
4614   assert(n1->req() == n2->req(), "can't compare %s nodes: n1->req() = %d, n2->req() = %d", NodeClassNames[n1->Opcode()], n1->req(), n2->req());
4615   for (uint i = 1; i < n1->req(); i++) {
4616     if (n1->in(i) < n2->in(i))      return -1;
4617     else if (n1->in(i) > n2->in(i)) return 1;
4618   }
4619 
4620   return 0;
4621 }
4622 
4623 int Compile::cmp_expensive_nodes(Node** n1p, Node** n2p) {
4624   Node* n1 = *n1p;
4625   Node* n2 = *n2p;
4626 
4627   return cmp_expensive_nodes(n1, n2);
4628 }
4629 
4630 void Compile::sort_expensive_nodes() {
4631   if (!expensive_nodes_sorted()) {
4632     _expensive_nodes.sort(cmp_expensive_nodes);
4633   }
4634 }
4635 
4636 bool Compile::expensive_nodes_sorted() const {
4637   for (int i = 1; i < _expensive_nodes.length(); i++) {
4638     if (cmp_expensive_nodes(_expensive_nodes.adr_at(i), _expensive_nodes.adr_at(i-1)) < 0) {
4639       return false;
4640     }
4641   }
4642   return true;
4643 }
4644 
4645 bool Compile::should_optimize_expensive_nodes(PhaseIterGVN &igvn) {
4646   if (_expensive_nodes.length() == 0) {
4647     return false;
4648   }
4649 
4650   assert(OptimizeExpensiveOps, "optimization off?");
4651 
4652   // Take this opportunity to remove dead nodes from the list
4653   int j = 0;
4654   for (int i = 0; i < _expensive_nodes.length(); i++) {
4655     Node* n = _expensive_nodes.at(i);
4656     if (!n->is_unreachable(igvn)) {
4657       assert(n->is_expensive(), "should be expensive");
4658       _expensive_nodes.at_put(j, n);
4659       j++;
4660     }
4661   }
4662   _expensive_nodes.trunc_to(j);
4663 
4664   // Then sort the list so that similar nodes are next to each other
4665   // and check for at least two nodes of identical kind with same data
4666   // inputs.
4667   sort_expensive_nodes();
4668 
4669   for (int i = 0; i < _expensive_nodes.length()-1; i++) {
4670     if (cmp_expensive_nodes(_expensive_nodes.adr_at(i), _expensive_nodes.adr_at(i+1)) == 0) {
4671       return true;
4672     }
4673   }
4674 
4675   return false;
4676 }
4677 
4678 void Compile::cleanup_expensive_nodes(PhaseIterGVN &igvn) {
4679   if (_expensive_nodes.length() == 0) {
4680     return;
4681   }
4682 
4683   assert(OptimizeExpensiveOps, "optimization off?");
4684 
4685   // Sort to bring similar nodes next to each other and clear the
4686   // control input of nodes for which there's only a single copy.
4687   sort_expensive_nodes();
4688 
4689   int j = 0;
4690   int identical = 0;
4691   int i = 0;
4692   bool modified = false;
4693   for (; i < _expensive_nodes.length()-1; i++) {
4694     assert(j <= i, "can't write beyond current index");
4695     if (_expensive_nodes.at(i)->Opcode() == _expensive_nodes.at(i+1)->Opcode()) {
4696       identical++;
4697       _expensive_nodes.at_put(j++, _expensive_nodes.at(i));
4698       continue;
4699     }
4700     if (identical > 0) {
4701       _expensive_nodes.at_put(j++, _expensive_nodes.at(i));
4702       identical = 0;
4703     } else {
4704       Node* n = _expensive_nodes.at(i);
4705       igvn.replace_input_of(n, 0, nullptr);
4706       igvn.hash_insert(n);
4707       modified = true;
4708     }
4709   }
4710   if (identical > 0) {
4711     _expensive_nodes.at_put(j++, _expensive_nodes.at(i));
4712   } else if (_expensive_nodes.length() >= 1) {
4713     Node* n = _expensive_nodes.at(i);
4714     igvn.replace_input_of(n, 0, nullptr);
4715     igvn.hash_insert(n);
4716     modified = true;
4717   }
4718   _expensive_nodes.trunc_to(j);
4719   if (modified) {
4720     igvn.optimize();
4721   }
4722 }
4723 
4724 void Compile::add_expensive_node(Node * n) {
4725   assert(!_expensive_nodes.contains(n), "duplicate entry in expensive list");
4726   assert(n->is_expensive(), "expensive nodes with non-null control here only");
4727   assert(!n->is_CFG() && !n->is_Mem(), "no cfg or memory nodes here");
4728   if (OptimizeExpensiveOps) {
4729     _expensive_nodes.append(n);
4730   } else {
4731     // Clear control input and let IGVN optimize expensive nodes if
4732     // OptimizeExpensiveOps is off.
4733     n->set_req(0, nullptr);
4734   }
4735 }
4736 
4737 /**
4738  * Track coarsened Lock and Unlock nodes.
4739  */
4740 
4741 class Lock_List : public Node_List {
4742   uint _origin_cnt;
4743 public:
4744   Lock_List(Arena *a, uint cnt) : Node_List(a), _origin_cnt(cnt) {}
4745   uint origin_cnt() const { return _origin_cnt; }
4746 };
4747 
4748 void Compile::add_coarsened_locks(GrowableArray<AbstractLockNode*>& locks) {
4749   int length = locks.length();
4750   if (length > 0) {
4751     // Have to keep this list until locks elimination during Macro nodes elimination.
4752     Lock_List* locks_list = new (comp_arena()) Lock_List(comp_arena(), length);
4753     AbstractLockNode* alock = locks.at(0);
4754     BoxLockNode* box = alock->box_node()->as_BoxLock();
4755     for (int i = 0; i < length; i++) {
4756       AbstractLockNode* lock = locks.at(i);
4757       assert(lock->is_coarsened(), "expecting only coarsened AbstractLock nodes, but got '%s'[%d] node", lock->Name(), lock->_idx);
4758       locks_list->push(lock);
4759       BoxLockNode* this_box = lock->box_node()->as_BoxLock();
4760       if (this_box != box) {
4761         // Locking regions (BoxLock) could be Unbalanced here:
4762         //  - its coarsened locks were eliminated in earlier
4763         //    macro nodes elimination followed by loop unroll
4764         //  - it is OSR locking region (no Lock node)
4765         // Preserve Unbalanced status in such cases.
4766         if (!this_box->is_unbalanced()) {
4767           this_box->set_coarsened();
4768         }
4769         if (!box->is_unbalanced()) {
4770           box->set_coarsened();
4771         }
4772       }
4773     }
4774     _coarsened_locks.append(locks_list);
4775   }
4776 }
4777 
4778 void Compile::remove_useless_coarsened_locks(Unique_Node_List& useful) {
4779   int count = coarsened_count();
4780   for (int i = 0; i < count; i++) {
4781     Node_List* locks_list = _coarsened_locks.at(i);
4782     for (uint j = 0; j < locks_list->size(); j++) {
4783       Node* lock = locks_list->at(j);
4784       assert(lock->is_AbstractLock(), "sanity");
4785       if (!useful.member(lock)) {
4786         locks_list->yank(lock);
4787       }
4788     }
4789   }
4790 }
4791 
4792 void Compile::remove_coarsened_lock(Node* n) {
4793   if (n->is_AbstractLock()) {
4794     int count = coarsened_count();
4795     for (int i = 0; i < count; i++) {
4796       Node_List* locks_list = _coarsened_locks.at(i);
4797       locks_list->yank(n);
4798     }
4799   }
4800 }
4801 
4802 bool Compile::coarsened_locks_consistent() {
4803   int count = coarsened_count();
4804   for (int i = 0; i < count; i++) {
4805     bool unbalanced = false;
4806     bool modified = false; // track locks kind modifications
4807     Lock_List* locks_list = (Lock_List*)_coarsened_locks.at(i);
4808     uint size = locks_list->size();
4809     if (size == 0) {
4810       unbalanced = false; // All locks were eliminated - good
4811     } else if (size != locks_list->origin_cnt()) {
4812       unbalanced = true; // Some locks were removed from list
4813     } else {
4814       for (uint j = 0; j < size; j++) {
4815         Node* lock = locks_list->at(j);
4816         // All nodes in group should have the same state (modified or not)
4817         if (!lock->as_AbstractLock()->is_coarsened()) {
4818           if (j == 0) {
4819             // first on list was modified, the rest should be too for consistency
4820             modified = true;
4821           } else if (!modified) {
4822             // this lock was modified but previous locks on the list were not
4823             unbalanced = true;
4824             break;
4825           }
4826         } else if (modified) {
4827           // previous locks on list were modified but not this lock
4828           unbalanced = true;
4829           break;
4830         }
4831       }
4832     }
4833     if (unbalanced) {
4834       // unbalanced monitor enter/exit - only some [un]lock nodes were removed or modified
4835 #ifdef ASSERT
4836       if (PrintEliminateLocks) {
4837         tty->print_cr("=== unbalanced coarsened locks ===");
4838         for (uint l = 0; l < size; l++) {
4839           locks_list->at(l)->dump();
4840         }
4841       }
4842 #endif
4843       record_failure(C2Compiler::retry_no_locks_coarsening());
4844       return false;
4845     }
4846   }
4847   return true;
4848 }
4849 
4850 // Mark locking regions (identified by BoxLockNode) as unbalanced if
4851 // locks coarsening optimization removed Lock/Unlock nodes from them.
4852 // Such regions become unbalanced because coarsening only removes part
4853 // of Lock/Unlock nodes in region. As result we can't execute other
4854 // locks elimination optimizations which assume all code paths have
4855 // corresponding pair of Lock/Unlock nodes - they are balanced.
4856 void Compile::mark_unbalanced_boxes() const {
4857   int count = coarsened_count();
4858   for (int i = 0; i < count; i++) {
4859     Node_List* locks_list = _coarsened_locks.at(i);
4860     uint size = locks_list->size();
4861     if (size > 0) {
4862       AbstractLockNode* alock = locks_list->at(0)->as_AbstractLock();
4863       BoxLockNode* box = alock->box_node()->as_BoxLock();
4864       if (alock->is_coarsened()) {
4865         // coarsened_locks_consistent(), which is called before this method, verifies
4866         // that the rest of Lock/Unlock nodes on locks_list are also coarsened.
4867         assert(!box->is_eliminated(), "regions with coarsened locks should not be marked as eliminated");
4868         for (uint j = 1; j < size; j++) {
4869           assert(locks_list->at(j)->as_AbstractLock()->is_coarsened(), "only coarsened locks are expected here");
4870           BoxLockNode* this_box = locks_list->at(j)->as_AbstractLock()->box_node()->as_BoxLock();
4871           if (box != this_box) {
4872             assert(!this_box->is_eliminated(), "regions with coarsened locks should not be marked as eliminated");
4873             box->set_unbalanced();
4874             this_box->set_unbalanced();
4875           }
4876         }
4877       }
4878     }
4879   }
4880 }
4881 
4882 /**
4883  * Remove the speculative part of types and clean up the graph
4884  */
4885 void Compile::remove_speculative_types(PhaseIterGVN &igvn) {
4886   if (UseTypeSpeculation) {
4887     Unique_Node_List worklist;
4888     worklist.push(root());
4889     int modified = 0;
4890     // Go over all type nodes that carry a speculative type, drop the
4891     // speculative part of the type and enqueue the node for an igvn
4892     // which may optimize it out.
4893     for (uint next = 0; next < worklist.size(); ++next) {
4894       Node *n  = worklist.at(next);
4895       if (n->is_Type()) {
4896         TypeNode* tn = n->as_Type();
4897         const Type* t = tn->type();
4898         const Type* t_no_spec = t->remove_speculative();
4899         if (t_no_spec != t) {
4900           bool in_hash = igvn.hash_delete(n);
4901           assert(in_hash || n->hash() == Node::NO_HASH, "node should be in igvn hash table");
4902           tn->set_type(t_no_spec);
4903           igvn.hash_insert(n);
4904           igvn._worklist.push(n); // give it a chance to go away
4905           modified++;
4906         }
4907       }
4908       // Iterate over outs - endless loops is unreachable from below
4909       for (DUIterator_Fast imax, i = n->fast_outs(imax); i < imax; i++) {
4910         Node *m = n->fast_out(i);
4911         if (not_a_node(m)) {
4912           continue;
4913         }
4914         worklist.push(m);
4915       }
4916     }
4917     // Drop the speculative part of all types in the igvn's type table
4918     igvn.remove_speculative_types();
4919     if (modified > 0) {
4920       igvn.optimize();
4921       if (failing())  return;
4922     }
4923 #ifdef ASSERT
4924     // Verify that after the IGVN is over no speculative type has resurfaced
4925     worklist.clear();
4926     worklist.push(root());
4927     for (uint next = 0; next < worklist.size(); ++next) {
4928       Node *n  = worklist.at(next);
4929       const Type* t = igvn.type_or_null(n);
4930       assert((t == nullptr) || (t == t->remove_speculative()), "no more speculative types");
4931       if (n->is_Type()) {
4932         t = n->as_Type()->type();
4933         assert(t == t->remove_speculative(), "no more speculative types");
4934       }
4935       // Iterate over outs - endless loops is unreachable from below
4936       for (DUIterator_Fast imax, i = n->fast_outs(imax); i < imax; i++) {
4937         Node *m = n->fast_out(i);
4938         if (not_a_node(m)) {
4939           continue;
4940         }
4941         worklist.push(m);
4942       }
4943     }
4944     igvn.check_no_speculative_types();
4945 #endif
4946   }
4947 }
4948 
4949 // Auxiliary methods to support randomized stressing/fuzzing.
4950 
4951 void Compile::initialize_stress_seed(const DirectiveSet* directive) {
4952   if (FLAG_IS_DEFAULT(StressSeed) || (FLAG_IS_ERGO(StressSeed) && directive->RepeatCompilationOption)) {
4953     _stress_seed = static_cast<uint>(Ticks::now().nanoseconds());
4954     FLAG_SET_ERGO(StressSeed, _stress_seed);
4955   } else {
4956     _stress_seed = StressSeed;
4957   }
4958   if (_log != nullptr) {
4959     _log->elem("stress_test seed='%u'", _stress_seed);
4960   }
4961 }
4962 
4963 int Compile::random() {
4964   _stress_seed = os::next_random(_stress_seed);
4965   return static_cast<int>(_stress_seed);
4966 }
4967 
4968 // This method can be called the arbitrary number of times, with current count
4969 // as the argument. The logic allows selecting a single candidate from the
4970 // running list of candidates as follows:
4971 //    int count = 0;
4972 //    Cand* selected = null;
4973 //    while(cand = cand->next()) {
4974 //      if (randomized_select(++count)) {
4975 //        selected = cand;
4976 //      }
4977 //    }
4978 //
4979 // Including count equalizes the chances any candidate is "selected".
4980 // This is useful when we don't have the complete list of candidates to choose
4981 // from uniformly. In this case, we need to adjust the randomicity of the
4982 // selection, or else we will end up biasing the selection towards the latter
4983 // candidates.
4984 //
4985 // Quick back-envelope calculation shows that for the list of n candidates
4986 // the equal probability for the candidate to persist as "best" can be
4987 // achieved by replacing it with "next" k-th candidate with the probability
4988 // of 1/k. It can be easily shown that by the end of the run, the
4989 // probability for any candidate is converged to 1/n, thus giving the
4990 // uniform distribution among all the candidates.
4991 //
4992 // We don't care about the domain size as long as (RANDOMIZED_DOMAIN / count) is large.
4993 #define RANDOMIZED_DOMAIN_POW 29
4994 #define RANDOMIZED_DOMAIN (1 << RANDOMIZED_DOMAIN_POW)
4995 #define RANDOMIZED_DOMAIN_MASK ((1 << (RANDOMIZED_DOMAIN_POW + 1)) - 1)
4996 bool Compile::randomized_select(int count) {
4997   assert(count > 0, "only positive");
4998   return (random() & RANDOMIZED_DOMAIN_MASK) < (RANDOMIZED_DOMAIN / count);
4999 }
5000 
5001 #ifdef ASSERT
5002 // Failures are geometrically distributed with probability 1/StressBailoutMean.
5003 bool Compile::fail_randomly() {
5004   if ((random() % StressBailoutMean) != 0) {
5005     return false;
5006   }
5007   record_failure("StressBailout");
5008   return true;
5009 }
5010 
5011 bool Compile::failure_is_artificial() {
5012   return C->failure_reason_is("StressBailout");
5013 }
5014 #endif
5015 
5016 CloneMap&     Compile::clone_map()                 { return _clone_map; }
5017 void          Compile::set_clone_map(Dict* d)      { _clone_map._dict = d; }
5018 
5019 void NodeCloneInfo::dump_on(outputStream* st) const {
5020   st->print(" {%d:%d} ", idx(), gen());
5021 }
5022 
5023 void CloneMap::clone(Node* old, Node* nnn, int gen) {
5024   uint64_t val = value(old->_idx);
5025   NodeCloneInfo cio(val);
5026   assert(val != 0, "old node should be in the map");
5027   NodeCloneInfo cin(cio.idx(), gen + cio.gen());
5028   insert(nnn->_idx, cin.get());
5029 #ifndef PRODUCT
5030   if (is_debug()) {
5031     tty->print_cr("CloneMap::clone inserted node %d info {%d:%d} into CloneMap", nnn->_idx, cin.idx(), cin.gen());
5032   }
5033 #endif
5034 }
5035 
5036 void CloneMap::verify_insert_and_clone(Node* old, Node* nnn, int gen) {
5037   NodeCloneInfo cio(value(old->_idx));
5038   if (cio.get() == 0) {
5039     cio.set(old->_idx, 0);
5040     insert(old->_idx, cio.get());
5041 #ifndef PRODUCT
5042     if (is_debug()) {
5043       tty->print_cr("CloneMap::verify_insert_and_clone inserted node %d info {%d:%d} into CloneMap", old->_idx, cio.idx(), cio.gen());
5044     }
5045 #endif
5046   }
5047   clone(old, nnn, gen);
5048 }
5049 
5050 int CloneMap::max_gen() const {
5051   int g = 0;
5052   DictI di(_dict);
5053   for(; di.test(); ++di) {
5054     int t = gen(di._key);
5055     if (g < t) {
5056       g = t;
5057 #ifndef PRODUCT
5058       if (is_debug()) {
5059         tty->print_cr("CloneMap::max_gen() update max=%d from %d", g, _2_node_idx_t(di._key));
5060       }
5061 #endif
5062     }
5063   }
5064   return g;
5065 }
5066 
5067 void CloneMap::dump(node_idx_t key, outputStream* st) const {
5068   uint64_t val = value(key);
5069   if (val != 0) {
5070     NodeCloneInfo ni(val);
5071     ni.dump_on(st);
5072   }
5073 }
5074 
5075 void Compile::shuffle_macro_nodes() {
5076   if (_macro_nodes.length() < 2) {
5077     return;
5078   }
5079   for (uint i = _macro_nodes.length() - 1; i >= 1; i--) {
5080     uint j = C->random() % (i + 1);
5081     swap(_macro_nodes.at(i), _macro_nodes.at(j));
5082   }
5083 }
5084 
5085 // Move Allocate nodes to the start of the list
5086 void Compile::sort_macro_nodes() {
5087   int count = macro_count();
5088   int allocates = 0;
5089   for (int i = 0; i < count; i++) {
5090     Node* n = macro_node(i);
5091     if (n->is_Allocate()) {
5092       if (i != allocates) {
5093         Node* tmp = macro_node(allocates);
5094         _macro_nodes.at_put(allocates, n);
5095         _macro_nodes.at_put(i, tmp);
5096       }
5097       allocates++;
5098     }
5099   }
5100 }
5101 
5102 void Compile::print_method(CompilerPhaseType cpt, int level, Node* n) {
5103   if (failing_internal()) { return; } // failing_internal to not stress bailouts from printing code.
5104   EventCompilerPhase event(UNTIMED);
5105   if (event.should_commit()) {
5106     CompilerEvent::PhaseEvent::post(event, C->_latest_stage_start_counter, cpt, C->_compile_id, level);
5107   }
5108 #ifndef PRODUCT
5109   ResourceMark rm;
5110   stringStream ss;
5111   ss.print_raw(CompilerPhaseTypeHelper::to_description(cpt));
5112   int iter = ++_igv_phase_iter[cpt];
5113   if (iter > 1) {
5114     ss.print(" %d", iter);
5115   }
5116   if (n != nullptr) {
5117     ss.print(": %d %s", n->_idx, NodeClassNames[n->Opcode()]);
5118     if (n->is_Call()) {
5119       CallNode* call = n->as_Call();
5120       if (call->_name != nullptr) {
5121         // E.g. uncommon traps etc.
5122         ss.print(" - %s", call->_name);
5123       } else if (call->is_CallJava()) {
5124         CallJavaNode* call_java = call->as_CallJava();
5125         if (call_java->method() != nullptr) {
5126           ss.print(" -");
5127           call_java->method()->print_short_name(&ss);
5128         }
5129       }
5130     }
5131   }
5132 
5133   const char* name = ss.as_string();
5134   if (should_print_igv(level)) {
5135     _igv_printer->print_graph(name);
5136   }
5137   if (should_print_phase(cpt)) {
5138     print_ideal_ir(CompilerPhaseTypeHelper::to_name(cpt));
5139   }
5140 #endif
5141   C->_latest_stage_start_counter.stamp();
5142 }
5143 
5144 // Only used from CompileWrapper
5145 void Compile::begin_method() {
5146 #ifndef PRODUCT
5147   if (_method != nullptr && should_print_igv(1)) {
5148     _igv_printer->begin_method();
5149   }
5150 #endif
5151   C->_latest_stage_start_counter.stamp();
5152 }
5153 
5154 // Only used from CompileWrapper
5155 void Compile::end_method() {
5156   EventCompilerPhase event(UNTIMED);
5157   if (event.should_commit()) {
5158     CompilerEvent::PhaseEvent::post(event, C->_latest_stage_start_counter, PHASE_END, C->_compile_id, 1);
5159   }
5160 
5161 #ifndef PRODUCT
5162   if (_method != nullptr && should_print_igv(1)) {
5163     _igv_printer->end_method();
5164   }
5165 #endif
5166 }
5167 
5168 bool Compile::should_print_phase(CompilerPhaseType cpt) {
5169 #ifndef PRODUCT
5170   if (_directive->should_print_phase(cpt)) {
5171     return true;
5172   }
5173 #endif
5174   return false;
5175 }
5176 
5177 #ifndef PRODUCT
5178 void Compile::init_igv() {
5179   if (_igv_printer == nullptr) {
5180     _igv_printer = IdealGraphPrinter::printer();
5181     _igv_printer->set_compile(this);
5182   }
5183 }
5184 #endif
5185 
5186 bool Compile::should_print_igv(const int level) {
5187 #ifndef PRODUCT
5188   if (PrintIdealGraphLevel < 0) { // disabled by the user
5189     return false;
5190   }
5191 
5192   bool need = directive()->IGVPrintLevelOption >= level;
5193   if (need) {
5194     Compile::init_igv();
5195   }
5196   return need;
5197 #else
5198   return false;
5199 #endif
5200 }
5201 
5202 #ifndef PRODUCT
5203 IdealGraphPrinter* Compile::_debug_file_printer = nullptr;
5204 IdealGraphPrinter* Compile::_debug_network_printer = nullptr;
5205 
5206 // Called from debugger. Prints method to the default file with the default phase name.
5207 // This works regardless of any Ideal Graph Visualizer flags set or not.
5208 void igv_print() {
5209   Compile::current()->igv_print_method_to_file();
5210 }
5211 
5212 // Same as igv_print() above but with a specified phase name.
5213 void igv_print(const char* phase_name) {
5214   Compile::current()->igv_print_method_to_file(phase_name);
5215 }
5216 
5217 // Called from debugger. Prints method with the default phase name to the default network or the one specified with
5218 // the network flags for the Ideal Graph Visualizer, or to the default file depending on the 'network' argument.
5219 // This works regardless of any Ideal Graph Visualizer flags set or not.
5220 void igv_print(bool network) {
5221   if (network) {
5222     Compile::current()->igv_print_method_to_network();
5223   } else {
5224     Compile::current()->igv_print_method_to_file();
5225   }
5226 }
5227 
5228 // Same as igv_print(bool network) above but with a specified phase name.
5229 void igv_print(bool network, const char* phase_name) {
5230   if (network) {
5231     Compile::current()->igv_print_method_to_network(phase_name);
5232   } else {
5233     Compile::current()->igv_print_method_to_file(phase_name);
5234   }
5235 }
5236 
5237 // Called from debugger. Normal write to the default _printer. Only works if Ideal Graph Visualizer printing flags are set.
5238 void igv_print_default() {
5239   Compile::current()->print_method(PHASE_DEBUG, 0);
5240 }
5241 
5242 // Called from debugger, especially when replaying a trace in which the program state cannot be altered like with rr replay.
5243 // A method is appended to an existing default file with the default phase name. This means that igv_append() must follow
5244 // an earlier igv_print(*) call which sets up the file. This works regardless of any Ideal Graph Visualizer flags set or not.
5245 void igv_append() {
5246   Compile::current()->igv_print_method_to_file("Debug", true);
5247 }
5248 
5249 // Same as igv_append() above but with a specified phase name.
5250 void igv_append(const char* phase_name) {
5251   Compile::current()->igv_print_method_to_file(phase_name, true);
5252 }
5253 
5254 void Compile::igv_print_method_to_file(const char* phase_name, bool append) {
5255   const char* file_name = "custom_debug.xml";
5256   if (_debug_file_printer == nullptr) {
5257     _debug_file_printer = new IdealGraphPrinter(C, file_name, append);
5258   } else {
5259     _debug_file_printer->update_compiled_method(C->method());
5260   }
5261   tty->print_cr("Method %s to %s", append ? "appended" : "printed", file_name);
5262   _debug_file_printer->print_graph(phase_name);
5263 }
5264 
5265 void Compile::igv_print_method_to_network(const char* phase_name) {
5266   ResourceMark rm;
5267   GrowableArray<const Node*> empty_list;
5268   igv_print_graph_to_network(phase_name, empty_list);
5269 }
5270 
5271 void Compile::igv_print_graph_to_network(const char* name, GrowableArray<const Node*>& visible_nodes) {
5272   if (_debug_network_printer == nullptr) {
5273     _debug_network_printer = new IdealGraphPrinter(C);
5274   } else {
5275     _debug_network_printer->update_compiled_method(C->method());
5276   }
5277   tty->print_cr("Method printed over network stream to IGV");
5278   _debug_network_printer->print(name, C->root(), visible_nodes);
5279 }
5280 #endif
5281 
5282 Node* Compile::narrow_value(BasicType bt, Node* value, const Type* type, PhaseGVN* phase, bool transform_res) {
5283   if (type != nullptr && phase->type(value)->higher_equal(type)) {
5284     return value;
5285   }
5286   Node* result = nullptr;
5287   if (bt == T_BYTE) {
5288     result = phase->transform(new LShiftINode(value, phase->intcon(24)));
5289     result = new RShiftINode(result, phase->intcon(24));
5290   } else if (bt == T_BOOLEAN) {
5291     result = new AndINode(value, phase->intcon(0xFF));
5292   } else if (bt == T_CHAR) {
5293     result = new AndINode(value,phase->intcon(0xFFFF));
5294   } else {
5295     assert(bt == T_SHORT, "unexpected narrow type");
5296     result = phase->transform(new LShiftINode(value, phase->intcon(16)));
5297     result = new RShiftINode(result, phase->intcon(16));
5298   }
5299   if (transform_res) {
5300     result = phase->transform(result);
5301   }
5302   return result;
5303 }
5304 
5305 void Compile::record_method_not_compilable_oom() {
5306   record_method_not_compilable(CompilationMemoryStatistic::failure_reason_memlimit());
5307 }