1 /*
   2  * Copyright (c) 2010, 2025, Oracle and/or its affiliates. All rights reserved.
   3  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
   4  *
   5  * This code is free software; you can redistribute it and/or modify it
   6  * under the terms of the GNU General Public License version 2 only, as
   7  * published by the Free Software Foundation.
   8  *
   9  * This code is distributed in the hope that it will be useful, but WITHOUT
  10  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  11  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  12  * version 2 for more details (a copy is included in the LICENSE file that
  13  * accompanied this code).
  14  *
  15  * You should have received a copy of the GNU General Public License version
  16  * 2 along with this work; if not, write to the Free Software Foundation,
  17  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
  18  *
  19  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
  20  * or visit www.oracle.com if you need additional information or have any
  21  * questions.
  22  *
  23  */
  24 
  25 #include "cds/aotLinkedClassBulkLoader.hpp"
  26 #include "code/aotCodeCache.hpp"
  27 #include "code/scopeDesc.hpp"
  28 #include "compiler/compilationPolicy.hpp"
  29 #include "compiler/compileBroker.hpp"
  30 #include "compiler/compilerDefinitions.inline.hpp"
  31 #include "compiler/compilerOracle.hpp"
  32 #include "compiler/recompilationPolicy.hpp"
  33 #include "memory/resourceArea.hpp"
  34 #include "oops/method.inline.hpp"
  35 #include "oops/methodData.hpp"
  36 #include "oops/oop.inline.hpp"
  37 #include "oops/trainingData.hpp"
  38 #include "prims/jvmtiExport.hpp"
  39 #include "runtime/arguments.hpp"
  40 #include "runtime/deoptimization.hpp"
  41 #include "runtime/frame.hpp"
  42 #include "runtime/frame.inline.hpp"
  43 #include "runtime/globals_extension.hpp"
  44 #include "runtime/handles.inline.hpp"
  45 #include "runtime/safepoint.hpp"
  46 #include "runtime/safepointVerifiers.hpp"
  47 #ifdef COMPILER1
  48 #include "c1/c1_Compiler.hpp"
  49 #endif
  50 #ifdef COMPILER2
  51 #include "opto/c2compiler.hpp"
  52 #endif
  53 #if INCLUDE_JVMCI
  54 #include "jvmci/jvmci.hpp"
  55 #endif
  56 
  57 int64_t CompilationPolicy::_start_time = 0;
  58 int CompilationPolicy::_c1_count = 0;
  59 int CompilationPolicy::_c2_count = 0;
  60 int CompilationPolicy::_ac_count = 0;
  61 double CompilationPolicy::_increase_threshold_at_ratio = 0;
  62 
  63 CompilationPolicy::TrainingReplayQueue CompilationPolicy::_training_replay_queue;
  64 
  65 void compilationPolicy_init() {
  66   CompilationPolicy::initialize();
  67 }
  68 
  69 int CompilationPolicy::compiler_count(CompLevel comp_level) {
  70   if (is_c1_compile(comp_level)) {
  71     return c1_count();
  72   } else if (is_c2_compile(comp_level)) {
  73     return c2_count();
  74   }
  75   return 0;
  76 }
  77 
  78 // Returns true if m must be compiled before executing it
  79 // This is intended to force compiles for methods (usually for
  80 // debugging) that would otherwise be interpreted for some reason.
  81 bool CompilationPolicy::must_be_compiled(const methodHandle& m, int comp_level) {
  82   // Don't allow Xcomp to cause compiles in replay mode
  83   if (ReplayCompiles) return false;
  84 
  85   if (m->has_compiled_code()) return false;       // already compiled
  86   if (!can_be_compiled(m, comp_level)) return false;
  87 
  88   return !UseInterpreter ||                                              // must compile all methods
  89          (AlwaysCompileLoopMethods && m->has_loops() && CompileBroker::should_compile_new_jobs()); // eagerly compile loop methods
  90 }
  91 
  92 void CompilationPolicy::maybe_compile_early(const methodHandle& m, TRAPS) {
  93   if (m->method_holder()->is_not_initialized()) {
  94     // 'is_not_initialized' means not only '!is_initialized', but also that
  95     // initialization has not been started yet ('!being_initialized')
  96     // Do not force compilation of methods in uninitialized classes.
  97     return;
  98   }
  99   if (!m->is_native() && MethodTrainingData::have_data()) {
 100     MethodTrainingData* mtd = MethodTrainingData::find_fast(m);
 101     if (mtd == nullptr) {
 102       return;              // there is no training data recorded for m
 103     }
 104     // AOT Preload code with class init barriers is used,
 105     // consider replacing it with normal (faster) AOT code
 106     bool recompile = m->code_has_clinit_barriers();
 107     CompLevel cur_level = static_cast<CompLevel>(m->highest_comp_level());
 108     CompLevel next_level = trained_transition(m, cur_level, mtd, THREAD);
 109     if ((next_level != cur_level || recompile) && can_be_compiled(m, next_level) && !CompileBroker::compilation_is_in_queue(m)) {
 110       bool requires_online_compilation = false;
 111       CompileTrainingData* ctd = mtd->last_toplevel_compile(next_level);
 112       if (ctd != nullptr) {
 113         // Can't load normal AOT code - not all dependancies are ready,
 114         // request normal compilation
 115         requires_online_compilation = (ctd->init_deps_left_acquire() > 0);
 116       }
 117       if (requires_online_compilation && recompile) {
 118         // Wait when dependencies are ready to load normal AOT code
 119         // if AOT Preload code is used now.
 120         //
 121         // FIXME. We may never (or it take long time) get all dependencies
 122         // be ready to replace AOT Preload code. Consider using time and how many
 123         // dependencies left to allow normal JIT compilation for replacement.
 124         return;
 125       }
 126       if (PrintTieredEvents) {
 127         print_event(FORCE_COMPILE, m(), m(), InvocationEntryBci, next_level);
 128       }
 129       CompileBroker::compile_method(m, InvocationEntryBci, next_level, 0, requires_online_compilation, CompileTask::Reason_MustBeCompiled, THREAD);
 130       if (HAS_PENDING_EXCEPTION) {
 131         CLEAR_PENDING_EXCEPTION;
 132       }
 133     }
 134   }
 135 }
 136 
 137 void CompilationPolicy::compile_if_required(const methodHandle& m, TRAPS) {
 138   if (!THREAD->can_call_java() || THREAD->is_Compiler_thread()) {
 139     // don't force compilation, resolve was on behalf of compiler
 140     return;
 141   }
 142   if (m->method_holder()->is_not_initialized()) {
 143     // 'is_not_initialized' means not only '!is_initialized', but also that
 144     // initialization has not been started yet ('!being_initialized')
 145     // Do not force compilation of methods in uninitialized classes.
 146     // Note that doing this would throw an assert later,
 147     // in CompileBroker::compile_method.
 148     // We sometimes use the link resolver to do reflective lookups
 149     // even before classes are initialized.
 150     return;
 151   }
 152 
 153   if (must_be_compiled(m)) {
 154     // This path is unusual, mostly used by the '-Xcomp' stress test mode.
 155     CompLevel level = initial_compile_level(m);
 156     if (PrintTieredEvents) {
 157       print_event(FORCE_COMPILE, m(), m(), InvocationEntryBci, level);
 158     }
 159     // Test AOT code too
 160     bool requires_online_compilation = false;
 161     if (TrainingData::have_data()) {
 162       MethodTrainingData* mtd = MethodTrainingData::find_fast(m);
 163       if (mtd != nullptr) {
 164         CompileTrainingData* ctd = mtd->last_toplevel_compile(level);
 165         if (ctd != nullptr) {
 166           requires_online_compilation = (ctd->init_deps_left_acquire() > 0);
 167         }
 168       }
 169     }
 170     CompileBroker::compile_method(m, InvocationEntryBci, level, 0, requires_online_compilation, CompileTask::Reason_MustBeCompiled, THREAD);
 171   }
 172 }
 173 
 174 void CompilationPolicy::replay_training_at_init_impl(InstanceKlass* klass, JavaThread* current) {
 175   if (!klass->has_init_deps_processed()) {
 176     ResourceMark rm;
 177     log_debug(training)("Replay training: %s", klass->external_name());
 178 
 179     KlassTrainingData* ktd = KlassTrainingData::find(klass);
 180     if (ktd != nullptr) {
 181       guarantee(ktd->has_holder(), "");
 182       ktd->notice_fully_initialized(); // sets klass->has_init_deps_processed bit
 183       assert(klass->has_init_deps_processed(), "");
 184 
 185       if (AOTCompileEagerly) {
 186         ktd->iterate_comp_deps([&](CompileTrainingData* ctd) {
 187           if (ctd->init_deps_left_acquire() == 0) {
 188             MethodTrainingData* mtd = ctd->method();
 189             if (mtd->has_holder()) {
 190               const methodHandle mh(current, const_cast<Method*>(mtd->holder()));
 191               CompilationPolicy::maybe_compile_early(mh, current);
 192             }
 193           }
 194         });
 195       }
 196     }
 197   }
 198 }
 199 
 200 void CompilationPolicy::replay_training_at_init(InstanceKlass* klass, JavaThread* current) {
 201   assert(klass->is_initialized(), "");
 202   if (TrainingData::have_data() && klass->in_aot_cache()) {
 203     _training_replay_queue.push(klass, TrainingReplayQueue_lock, current);
 204   }
 205 }
 206 
 207 // For TrainingReplayQueue
 208 template<>
 209 void CompilationPolicyUtils::Queue<InstanceKlass>::print_on(outputStream* st) {
 210   int pos = 0;
 211   for (QueueNode* cur = _head; cur != nullptr; cur = cur->next()) {
 212     ResourceMark rm;
 213     InstanceKlass* ik = cur->value();
 214     st->print_cr("%3d: " INTPTR_FORMAT " %s", ++pos, p2i(ik), ik->external_name());
 215   }
 216 }
 217 
 218 void CompilationPolicy::replay_training_at_init_loop(JavaThread* current) {
 219   while (!CompileBroker::is_compilation_disabled_forever()) {
 220     InstanceKlass* ik = _training_replay_queue.pop(TrainingReplayQueue_lock, current);
 221     if (ik != nullptr) {
 222       replay_training_at_init_impl(ik, current);
 223     }
 224   }
 225 }
 226 
 227 static inline CompLevel adjust_level_for_compilability_query(CompLevel comp_level) {
 228   if (comp_level == CompLevel_any) {
 229      if (CompilerConfig::is_c1_only()) {
 230        comp_level = CompLevel_simple;
 231      } else if (CompilerConfig::is_c2_or_jvmci_compiler_only()) {
 232        comp_level = CompLevel_full_optimization;
 233      }
 234   }
 235   return comp_level;
 236 }
 237 
 238 // Returns true if m is allowed to be compiled
 239 bool CompilationPolicy::can_be_compiled(const methodHandle& m, int comp_level) {
 240   // allow any levels for WhiteBox
 241   assert(WhiteBoxAPI || comp_level == CompLevel_any || is_compile(comp_level), "illegal compilation level %d", comp_level);
 242 
 243   if (m->is_abstract()) return false;
 244   if (DontCompileHugeMethods && m->code_size() > HugeMethodLimit) return false;
 245 
 246   // Math intrinsics should never be compiled as this can lead to
 247   // monotonicity problems because the interpreter will prefer the
 248   // compiled code to the intrinsic version.  This can't happen in
 249   // production because the invocation counter can't be incremented
 250   // but we shouldn't expose the system to this problem in testing
 251   // modes.
 252   if (!AbstractInterpreter::can_be_compiled(m)) {
 253     return false;
 254   }
 255   comp_level = adjust_level_for_compilability_query((CompLevel) comp_level);
 256   if (comp_level == CompLevel_any || is_compile(comp_level)) {
 257     return !m->is_not_compilable(comp_level);
 258   }
 259   return false;
 260 }
 261 
 262 // Returns true if m is allowed to be osr compiled
 263 bool CompilationPolicy::can_be_osr_compiled(const methodHandle& m, int comp_level) {
 264   bool result = false;
 265   comp_level = adjust_level_for_compilability_query((CompLevel) comp_level);
 266   if (comp_level == CompLevel_any || is_compile(comp_level)) {
 267     result = !m->is_not_osr_compilable(comp_level);
 268   }
 269   return (result && can_be_compiled(m, comp_level));
 270 }
 271 
 272 bool CompilationPolicy::is_compilation_enabled() {
 273   // NOTE: CompileBroker::should_compile_new_jobs() checks for UseCompiler
 274   return CompileBroker::should_compile_new_jobs();
 275 }
 276 
 277 CompileTask* CompilationPolicy::select_task_helper(CompileQueue* compile_queue) {
 278   // Remove unloaded methods from the queue
 279   for (CompileTask* task = compile_queue->first(); task != nullptr; ) {
 280     CompileTask* next = task->next();
 281     if (task->is_unloaded()) {
 282       compile_queue->remove_and_mark_stale(task);
 283     }
 284     task = next;
 285   }
 286 #if INCLUDE_JVMCI
 287   if (UseJVMCICompiler && !BackgroundCompilation) {
 288     /*
 289      * In blocking compilation mode, the CompileBroker will make
 290      * compilations submitted by a JVMCI compiler thread non-blocking. These
 291      * compilations should be scheduled after all blocking compilations
 292      * to service non-compiler related compilations sooner and reduce the
 293      * chance of such compilations timing out.
 294      */
 295     for (CompileTask* task = compile_queue->first(); task != nullptr; task = task->next()) {
 296       if (task->is_blocking()) {
 297         return task;
 298       }
 299     }
 300   }
 301 #endif
 302   return compile_queue->first();
 303 }
 304 
 305 // Simple methods are as good being compiled with C1 as C2.
 306 // Determine if a given method is such a case.
 307 bool CompilationPolicy::is_trivial(const methodHandle& method) {
 308   if (method->is_accessor() ||
 309       method->is_constant_getter()) {
 310     return true;
 311   }
 312   return false;
 313 }
 314 
 315 bool CompilationPolicy::force_comp_at_level_simple(const methodHandle& method) {
 316   if (CompilationModeFlag::quick_internal()) {
 317 #if INCLUDE_JVMCI
 318     if (UseJVMCICompiler) {
 319       AbstractCompiler* comp = CompileBroker::compiler(CompLevel_full_optimization);
 320       if (comp != nullptr && comp->is_jvmci() && ((JVMCICompiler*) comp)->force_comp_at_level_simple(method)) {
 321         return true;
 322       }
 323     }
 324 #endif
 325   }
 326   return false;
 327 }
 328 
 329 CompLevel CompilationPolicy::comp_level(Method* method) {
 330   nmethod *nm = method->code();
 331   if (nm != nullptr && nm->is_in_use()) {
 332     return (CompLevel)nm->comp_level();
 333   }
 334   return CompLevel_none;
 335 }
 336 
 337 // Call and loop predicates determine whether a transition to a higher
 338 // compilation level should be performed (pointers to predicate functions
 339 // are passed to common()).
 340 // Tier?LoadFeedback is basically a coefficient that determines of
 341 // how many methods per compiler thread can be in the queue before
 342 // the threshold values double.
 343 class LoopPredicate : AllStatic {
 344 public:
 345   static bool apply_scaled(const methodHandle& method, CompLevel cur_level, int i, int b, double scale) {
 346     double threshold_scaling;
 347     if (CompilerOracle::has_option_value(method, CompileCommandEnum::CompileThresholdScaling, threshold_scaling)) {
 348       scale *= threshold_scaling;
 349     }
 350     switch(cur_level) {
 351     case CompLevel_none:
 352     case CompLevel_limited_profile:
 353       return b >= Tier3BackEdgeThreshold * scale;
 354     case CompLevel_full_profile:
 355       return b >= Tier4BackEdgeThreshold * scale;
 356     default:
 357       return true;
 358     }
 359   }
 360 
 361   static bool apply(const methodHandle& method, CompLevel cur_level, int i, int b) {
 362     double k = 1;
 363     switch(cur_level) {
 364     case CompLevel_none:
 365     // Fall through
 366     case CompLevel_limited_profile: {
 367       k = CompilationPolicy::threshold_scale(CompLevel_full_profile, Tier3LoadFeedback);
 368       break;
 369     }
 370     case CompLevel_full_profile: {
 371       k = CompilationPolicy::threshold_scale(CompLevel_full_optimization, Tier4LoadFeedback);
 372       break;
 373     }
 374     default:
 375       return true;
 376     }
 377     return apply_scaled(method, cur_level, i, b, k);
 378   }
 379 };
 380 
 381 class CallPredicate : AllStatic {
 382 public:
 383   static bool apply_scaled(const methodHandle& method, CompLevel cur_level, int i, int b, double scale) {
 384     double threshold_scaling;
 385     if (CompilerOracle::has_option_value(method, CompileCommandEnum::CompileThresholdScaling, threshold_scaling)) {
 386       scale *= threshold_scaling;
 387     }
 388     switch(cur_level) {
 389     case CompLevel_none:
 390     case CompLevel_limited_profile:
 391       return (i >= Tier3InvocationThreshold * scale) ||
 392              (i >= Tier3MinInvocationThreshold * scale && i + b >= Tier3CompileThreshold * scale);
 393     case CompLevel_full_profile:
 394       return (i >= Tier4InvocationThreshold * scale) ||
 395              (i >= Tier4MinInvocationThreshold * scale && i + b >= Tier4CompileThreshold * scale);
 396     default:
 397      return true;
 398     }
 399   }
 400 
 401   static bool apply(const methodHandle& method, CompLevel cur_level, int i, int b) {
 402     double k = 1;
 403     switch(cur_level) {
 404     case CompLevel_none:
 405     case CompLevel_limited_profile: {
 406       k = CompilationPolicy::threshold_scale(CompLevel_full_profile, Tier3LoadFeedback);
 407       break;
 408     }
 409     case CompLevel_full_profile: {
 410       k = CompilationPolicy::threshold_scale(CompLevel_full_optimization, Tier4LoadFeedback);
 411       break;
 412     }
 413     default:
 414       return true;
 415     }
 416     return apply_scaled(method, cur_level, i, b, k);
 417   }
 418 };
 419 
 420 double CompilationPolicy::threshold_scale(CompLevel level, int feedback_k) {
 421   int comp_count = compiler_count(level);
 422   if (comp_count > 0 && feedback_k > 0) {
 423     double queue_size = CompileBroker::queue_size(level);
 424     double k = (double)queue_size / ((double)feedback_k * (double)comp_count) + 1;
 425 
 426     // Increase C1 compile threshold when the code cache is filled more
 427     // than specified by IncreaseFirstTierCompileThresholdAt percentage.
 428     // The main intention is to keep enough free space for C2 compiled code
 429     // to achieve peak performance if the code cache is under stress.
 430     if (CompilerConfig::is_tiered() && !CompilationModeFlag::disable_intermediate() && is_c1_compile(level))  {
 431       double current_reverse_free_ratio = CodeCache::reverse_free_ratio();
 432       if (current_reverse_free_ratio > _increase_threshold_at_ratio) {
 433         k *= exp(current_reverse_free_ratio - _increase_threshold_at_ratio);
 434       }
 435     }
 436     return k;
 437   }
 438   return 1;
 439 }
 440 
 441 void CompilationPolicy::print_counters_on(outputStream* st, const char* prefix, Method* m) {
 442   int invocation_count = m->invocation_count();
 443   int backedge_count = m->backedge_count();
 444   MethodData* mdh = m->method_data();
 445   int mdo_invocations = 0, mdo_backedges = 0;
 446   int mdo_invocations_start = 0, mdo_backedges_start = 0;
 447   if (mdh != nullptr) {
 448     mdo_invocations = mdh->invocation_count();
 449     mdo_backedges = mdh->backedge_count();
 450     mdo_invocations_start = mdh->invocation_count_start();
 451     mdo_backedges_start = mdh->backedge_count_start();
 452   }
 453   st->print(" %stotal=%d,%d %smdo=%d(%d),%d(%d)", prefix,
 454     invocation_count, backedge_count, prefix,
 455     mdo_invocations, mdo_invocations_start,
 456     mdo_backedges, mdo_backedges_start);
 457   st->print(" %smax levels=%d,%d", prefix, m->highest_comp_level(), m->highest_osr_comp_level());
 458 }
 459 
 460 void CompilationPolicy::print_training_data_on(outputStream* st,  const char* prefix, Method* method) {
 461   methodHandle m(Thread::current(), method);
 462   st->print(" %smtd: ", prefix);
 463   MethodTrainingData* mtd = MethodTrainingData::find(m);
 464   if (mtd == nullptr) {
 465     st->print("null");
 466   } else {
 467     MethodData* md = mtd->final_profile();
 468     st->print("mdo=");
 469     if (md == nullptr) {
 470       st->print("null");
 471     } else {
 472       int mdo_invocations = md->invocation_count();
 473       int mdo_backedges = md->backedge_count();
 474       int mdo_invocations_start = md->invocation_count_start();
 475       int mdo_backedges_start = md->backedge_count_start();
 476       st->print("%d(%d), %d(%d)", mdo_invocations, mdo_invocations_start, mdo_backedges, mdo_backedges_start);
 477     }
 478     CompileTrainingData* ctd = mtd->last_toplevel_compile(CompLevel_full_optimization);
 479     st->print(", deps=");
 480     if (ctd == nullptr) {
 481       st->print("null");
 482     } else {
 483       st->print("%d", ctd->init_deps_left_acquire());
 484     }
 485   }
 486 }
 487 
 488 // Print an event.
 489 void CompilationPolicy::print_event_on(outputStream *st, EventType type, Method* m, Method* im, int bci, CompLevel level) {
 490   bool inlinee_event = m != im;
 491 
 492   st->print("%lf: [", os::elapsedTime());
 493 
 494   switch(type) {
 495   case CALL:
 496     st->print("call");
 497     break;
 498   case LOOP:
 499     st->print("loop");
 500     break;
 501   case COMPILE:
 502     st->print("compile");
 503     break;
 504   case FORCE_COMPILE:
 505     st->print("force-compile");
 506     break;
 507   case FORCE_RECOMPILE:
 508     st->print("force-recompile");
 509     break;
 510   case REMOVE_FROM_QUEUE:
 511     st->print("remove-from-queue");
 512     break;
 513   case UPDATE_IN_QUEUE:
 514     st->print("update-in-queue");
 515     break;
 516   case REPROFILE:
 517     st->print("reprofile");
 518     break;
 519   case MAKE_NOT_ENTRANT:
 520     st->print("make-not-entrant");
 521     break;
 522   default:
 523     st->print("unknown");
 524   }
 525 
 526   st->print(" level=%d ", level);
 527 
 528   ResourceMark rm;
 529   char *method_name = m->name_and_sig_as_C_string();
 530   st->print("[%s", method_name);
 531   if (inlinee_event) {
 532     char *inlinee_name = im->name_and_sig_as_C_string();
 533     st->print(" [%s]] ", inlinee_name);
 534   }
 535   else st->print("] ");
 536   st->print("@%d queues=%d,%d", bci, CompileBroker::queue_size(CompLevel_full_profile),
 537                                      CompileBroker::queue_size(CompLevel_full_optimization));
 538 
 539   st->print(" rate=");
 540   if (m->prev_time() == 0) st->print("n/a");
 541   else st->print("%f", m->rate());
 542 
 543   RecompilationPolicy::print_load_average(st);
 544 
 545   st->print(" k=%.2lf,%.2lf", threshold_scale(CompLevel_full_profile, Tier3LoadFeedback),
 546                               threshold_scale(CompLevel_full_optimization, Tier4LoadFeedback));
 547 
 548   if (type != COMPILE) {
 549     print_counters_on(st, "", m);
 550     if (inlinee_event) {
 551       print_counters_on(st, "inlinee ", im);
 552     }
 553     st->print(" compilable=");
 554     bool need_comma = false;
 555     if (!m->is_not_compilable(CompLevel_full_profile)) {
 556       st->print("c1");
 557       need_comma = true;
 558     }
 559     if (!m->is_not_osr_compilable(CompLevel_full_profile)) {
 560       if (need_comma) st->print(",");
 561       st->print("c1-osr");
 562       need_comma = true;
 563     }
 564     if (!m->is_not_compilable(CompLevel_full_optimization)) {
 565       if (need_comma) st->print(",");
 566       st->print("c2");
 567       need_comma = true;
 568     }
 569     if (!m->is_not_osr_compilable(CompLevel_full_optimization)) {
 570       if (need_comma) st->print(",");
 571       st->print("c2-osr");
 572     }
 573     st->print(" status=");
 574     if (m->queued_for_compilation()) {
 575       st->print("in-queue");
 576     } else st->print("idle");
 577 
 578     print_training_data_on(st, "", m);
 579     if (inlinee_event) {
 580       print_training_data_on(st, "inlinee ", im);
 581     }
 582   }
 583   st->print_cr("]");
 584 
 585 }
 586 
 587 void CompilationPolicy::print_event(EventType type, Method* m, Method* im, int bci, CompLevel level) {
 588   stringStream s;
 589   print_event_on(&s, type, m, im, bci, level);
 590   ResourceMark rm;
 591   tty->print("%s", s.as_string());
 592 }
 593 
 594 void CompilationPolicy::initialize() {
 595   if (!CompilerConfig::is_interpreter_only()) {
 596     int count = CICompilerCount;
 597     bool c1_only = CompilerConfig::is_c1_only();
 598     bool c2_only = CompilerConfig::is_c2_or_jvmci_compiler_only();
 599     int min_count = (c1_only || c2_only) ? 1 : 2;
 600 
 601 #ifdef _LP64
 602     // Turn on ergonomic compiler count selection
 603     if (AOTCodeCache::maybe_dumping_code()) {
 604       // Assembly phase runs C1 and C2 compilation in separate phases,
 605       // and can use all the CPU threads it can reach. Adjust the common
 606       // options before policy starts overwriting them.
 607       FLAG_SET_ERGO_IF_DEFAULT(UseDynamicNumberOfCompilerThreads, false);
 608       FLAG_SET_ERGO_IF_DEFAULT(CICompilerCountPerCPU, false);
 609       if (FLAG_IS_DEFAULT(CICompilerCount)) {
 610         count =  MAX2(count, os::active_processor_count());
 611       }
 612     }
 613     if (FLAG_IS_DEFAULT(CICompilerCountPerCPU) && FLAG_IS_DEFAULT(CICompilerCount)) {
 614       FLAG_SET_DEFAULT(CICompilerCountPerCPU, true);
 615     }
 616     if (CICompilerCountPerCPU) {
 617       // Simple log n seems to grow too slowly for tiered, try something faster: log n * log log n
 618       int log_cpu = log2i(os::active_processor_count());
 619       int loglog_cpu = log2i(MAX2(log_cpu, 1));
 620       count = MAX2(log_cpu * loglog_cpu * 3 / 2, min_count);
 621     }
 622     if (FLAG_IS_DEFAULT(CICompilerCount)) {
 623       // Make sure there is enough space in the code cache to hold all the compiler buffers
 624       size_t c1_size = 0;
 625 #ifdef COMPILER1
 626       c1_size = Compiler::code_buffer_size();
 627 #endif
 628       size_t c2_size = 0;
 629 #ifdef COMPILER2
 630       c2_size = C2Compiler::initial_code_buffer_size();
 631 #endif
 632       size_t buffer_size = 0;
 633       if (c1_only) {
 634         buffer_size = c1_size;
 635       } else if (c2_only) {
 636         buffer_size = c2_size;
 637       } else {
 638         buffer_size = c1_size / 3 + 2 * c2_size / 3;
 639       }
 640       size_t max_count = (NonNMethodCodeHeapSize - (CodeCacheMinimumUseSpace DEBUG_ONLY(* 3))) / buffer_size;
 641       if ((size_t)count > max_count) {
 642         // Lower the compiler count such that all buffers fit into the code cache
 643         count = MAX2((int)max_count, min_count);
 644       }
 645       FLAG_SET_ERGO(CICompilerCount, count);
 646     }
 647 #else
 648     // On 32-bit systems, the number of compiler threads is limited to 3.
 649     // On these systems, the virtual address space available to the JVM
 650     // is usually limited to 2-4 GB (the exact value depends on the platform).
 651     // As the compilers (especially C2) can consume a large amount of
 652     // memory, scaling the number of compiler threads with the number of
 653     // available cores can result in the exhaustion of the address space
 654     /// available to the VM and thus cause the VM to crash.
 655     if (FLAG_IS_DEFAULT(CICompilerCount)) {
 656       count = 3;
 657       FLAG_SET_ERGO(CICompilerCount, count);
 658     }
 659 #endif // _LP64
 660 
 661     if (c1_only) {
 662       // No C2 compiler threads are needed
 663       set_c1_count(count);
 664     } else if (c2_only) {
 665       // No C1 compiler threads are needed
 666       set_c2_count(count);
 667     } else {
 668 #if INCLUDE_JVMCI
 669       if (UseJVMCICompiler && UseJVMCINativeLibrary) {
 670         int libjvmci_count = MAX2((int) (count * JVMCINativeLibraryThreadFraction), 1);
 671         int c1_count = MAX2(count - libjvmci_count, 1);
 672         set_c2_count(libjvmci_count);
 673         set_c1_count(c1_count);
 674       } else
 675 #endif
 676       {
 677         set_c1_count(MAX2(count / 3, 1));
 678         set_c2_count(MAX2(count - c1_count(), 1));
 679       }
 680     }
 681     if (AOTCodeCache::is_code_load_thread_on()) {
 682       set_ac_count((c1_only || c2_only) ? 1 : 2); // At minimum we need 2 threads to load C1 and C2 AOT code in parallel
 683     }
 684     assert(count == c1_count() + c2_count(), "inconsistent compiler thread count");
 685     set_increase_threshold_at_ratio();
 686   } else {
 687     // Interpreter mode creates no compilers
 688     FLAG_SET_ERGO(CICompilerCount, 0);
 689   }
 690   set_start_time(nanos_to_millis(os::javaTimeNanos()));
 691 }
 692 
 693 
 694 #ifdef ASSERT
 695 bool CompilationPolicy::verify_level(CompLevel level) {
 696   if (TieredCompilation && level > TieredStopAtLevel) {
 697     return false;
 698   }
 699   // Check if there is a compiler to process the requested level
 700   if (!CompilerConfig::is_c1_enabled() && is_c1_compile(level)) {
 701     return false;
 702   }
 703   if (!CompilerConfig::is_c2_or_jvmci_compiler_enabled() && is_c2_compile(level)) {
 704     return false;
 705   }
 706 
 707   // Interpreter level is always valid.
 708   if (level == CompLevel_none) {
 709     return true;
 710   }
 711   if (CompilationModeFlag::normal()) {
 712     return true;
 713   } else if (CompilationModeFlag::quick_only()) {
 714     return level == CompLevel_simple;
 715   } else if (CompilationModeFlag::high_only()) {
 716     return level == CompLevel_full_optimization;
 717   } else if (CompilationModeFlag::high_only_quick_internal()) {
 718     return level == CompLevel_full_optimization || level == CompLevel_simple;
 719   }
 720   return false;
 721 }
 722 #endif
 723 
 724 
 725 CompLevel CompilationPolicy::highest_compile_level() {
 726   CompLevel level = CompLevel_none;
 727   // Setup the maximum level available for the current compiler configuration.
 728   if (!CompilerConfig::is_interpreter_only()) {
 729     if (CompilerConfig::is_c2_or_jvmci_compiler_enabled()) {
 730       level = CompLevel_full_optimization;
 731     } else if (CompilerConfig::is_c1_enabled()) {
 732       if (CompilerConfig::is_c1_simple_only()) {
 733         level = CompLevel_simple;
 734       } else {
 735         level = CompLevel_full_profile;
 736       }
 737     }
 738   }
 739   // Clamp the maximum level with TieredStopAtLevel.
 740   if (TieredCompilation) {
 741     level = MIN2(level, (CompLevel) TieredStopAtLevel);
 742   }
 743 
 744   // Fix it up if after the clamping it has become invalid.
 745   // Bring it monotonically down depending on the next available level for
 746   // the compilation mode.
 747   if (!CompilationModeFlag::normal()) {
 748     // a) quick_only - levels 2,3,4 are invalid; levels -1,0,1 are valid;
 749     // b) high_only - levels 1,2,3 are invalid; levels -1,0,4 are valid;
 750     // c) high_only_quick_internal - levels 2,3 are invalid; levels -1,0,1,4 are valid.
 751     if (CompilationModeFlag::quick_only()) {
 752       if (level == CompLevel_limited_profile || level == CompLevel_full_profile || level == CompLevel_full_optimization) {
 753         level = CompLevel_simple;
 754       }
 755     } else if (CompilationModeFlag::high_only()) {
 756       if (level == CompLevel_simple || level == CompLevel_limited_profile || level == CompLevel_full_profile) {
 757         level = CompLevel_none;
 758       }
 759     } else if (CompilationModeFlag::high_only_quick_internal()) {
 760       if (level == CompLevel_limited_profile || level == CompLevel_full_profile) {
 761         level = CompLevel_simple;
 762       }
 763     }
 764   }
 765 
 766   assert(verify_level(level), "Invalid highest compilation level: %d", level);
 767   return level;
 768 }
 769 
 770 CompLevel CompilationPolicy::limit_level(CompLevel level) {
 771   level = MIN2(level, highest_compile_level());
 772   assert(verify_level(level), "Invalid compilation level: %d", level);
 773   return level;
 774 }
 775 
 776 CompLevel CompilationPolicy::initial_compile_level(const methodHandle& method) {
 777   CompLevel level = CompLevel_any;
 778   if (CompilationModeFlag::normal()) {
 779     level = CompLevel_full_profile;
 780   } else if (CompilationModeFlag::quick_only()) {
 781     level = CompLevel_simple;
 782   } else if (CompilationModeFlag::high_only()) {
 783     level = CompLevel_full_optimization;
 784   } else if (CompilationModeFlag::high_only_quick_internal()) {
 785     if (force_comp_at_level_simple(method)) {
 786       level = CompLevel_simple;
 787     } else {
 788       level = CompLevel_full_optimization;
 789     }
 790   }
 791   assert(level != CompLevel_any, "Unhandled compilation mode");
 792   return limit_level(level);
 793 }
 794 
 795 // Set carry flags on the counters if necessary
 796 void CompilationPolicy::handle_counter_overflow(const methodHandle& method) {
 797   MethodCounters *mcs = method->method_counters();
 798   if (mcs != nullptr) {
 799     mcs->invocation_counter()->set_carry_on_overflow();
 800     mcs->backedge_counter()->set_carry_on_overflow();
 801   }
 802   MethodData* mdo = method->method_data();
 803   if (mdo != nullptr) {
 804     mdo->invocation_counter()->set_carry_on_overflow();
 805     mdo->backedge_counter()->set_carry_on_overflow();
 806   }
 807 }
 808 
 809 // Called with the queue locked and with at least one element
 810 CompileTask* CompilationPolicy::select_task(CompileQueue* compile_queue, JavaThread* THREAD) {
 811   CompileTask *max_blocking_task = nullptr;
 812   CompileTask *max_task = nullptr;
 813   Method* max_method = nullptr;
 814 
 815   int64_t t = nanos_to_millis(os::javaTimeNanos());
 816   // Iterate through the queue and find a method with a maximum rate.
 817   for (CompileTask* task = compile_queue->first(); task != nullptr;) {
 818     CompileTask* next_task = task->next();
 819     // If a method was unloaded or has been stale for some time, remove it from the queue.
 820     // Blocking tasks and tasks submitted from whitebox API don't become stale
 821     if (task->is_unloaded()) {
 822       compile_queue->remove_and_mark_stale(task);
 823       task = next_task;
 824       continue;
 825     }
 826     if (task->is_aot_load()) {
 827       // AOTCodeCache tasks are on separate queue, and they should load fast. There is no need to walk
 828       // the rest of the queue, just take the task and go.
 829       return task;
 830     }
 831     if (task->is_blocking() && task->compile_reason() == CompileTask::Reason_Whitebox) {
 832       // CTW tasks, submitted as blocking Whitebox requests, do not participate in rate
 833       // selection and/or any level adjustments. Just return them in order.
 834       return task;
 835     }
 836     Method* method = task->method();
 837     methodHandle mh(THREAD, method);
 838     if (task->can_become_stale() && is_stale(t, TieredCompileTaskTimeout, mh) && !is_old(mh)) {
 839       if (PrintTieredEvents) {
 840         print_event(REMOVE_FROM_QUEUE, method, method, task->osr_bci(), (CompLevel) task->comp_level());
 841       }
 842       method->clear_queued_for_compilation();
 843       method->set_pending_queue_processed(false);
 844       compile_queue->remove_and_mark_stale(task);
 845       task = next_task;
 846       continue;
 847     }
 848     update_rate(t, mh);
 849     if (max_task == nullptr || compare_methods(method, max_method) || compare_tasks(task, max_task)) {
 850       // Select a method with the highest rate
 851       max_task = task;
 852       max_method = method;
 853     }
 854 
 855     if (task->is_blocking()) {
 856       if (max_blocking_task == nullptr || compare_methods(method, max_blocking_task->method())) {
 857         max_blocking_task = task;
 858       }
 859     }
 860 
 861     task = next_task;
 862   }
 863 
 864   if (max_blocking_task != nullptr) {
 865     // In blocking compilation mode, the CompileBroker will make
 866     // compilations submitted by a JVMCI compiler thread non-blocking. These
 867     // compilations should be scheduled after all blocking compilations
 868     // to service non-compiler related compilations sooner and reduce the
 869     // chance of such compilations timing out.
 870     max_task = max_blocking_task;
 871     max_method = max_task->method();
 872   }
 873 
 874   methodHandle max_method_h(THREAD, max_method);
 875 
 876   if (max_task != nullptr && max_task->comp_level() == CompLevel_full_profile && TieredStopAtLevel > CompLevel_full_profile &&
 877       max_method != nullptr && is_method_profiled(max_method_h) && !Arguments::is_compiler_only()) {
 878     max_task->set_comp_level(CompLevel_limited_profile);
 879 
 880     if (CompileBroker::compilation_is_complete(max_method_h(), max_task->osr_bci(), CompLevel_limited_profile,
 881                                                false /* requires_online_compilation */,
 882                                                CompileTask::Reason_None)) {
 883       if (PrintTieredEvents) {
 884         print_event(REMOVE_FROM_QUEUE, max_method, max_method, max_task->osr_bci(), (CompLevel)max_task->comp_level());
 885       }
 886       compile_queue->remove_and_mark_stale(max_task);
 887       max_method->clear_queued_for_compilation();
 888       return nullptr;
 889     }
 890 
 891     if (PrintTieredEvents) {
 892       print_event(UPDATE_IN_QUEUE, max_method, max_method, max_task->osr_bci(), (CompLevel)max_task->comp_level());
 893     }
 894   }
 895 
 896   return max_task;
 897 }
 898 
 899 void CompilationPolicy::reprofile(ScopeDesc* trap_scope, bool is_osr) {
 900   for (ScopeDesc* sd = trap_scope;; sd = sd->sender()) {
 901     if (PrintTieredEvents) {
 902       print_event(REPROFILE, sd->method(), sd->method(), InvocationEntryBci, CompLevel_none);
 903     }
 904     MethodData* mdo = sd->method()->method_data();
 905     if (mdo != nullptr) {
 906       mdo->reset_start_counters();
 907     }
 908     if (sd->is_top()) break;
 909   }
 910 }
 911 
 912 nmethod* CompilationPolicy::event(const methodHandle& method, const methodHandle& inlinee,
 913                                       int branch_bci, int bci, CompLevel comp_level, nmethod* nm, TRAPS) {
 914   if (PrintTieredEvents) {
 915     print_event(bci == InvocationEntryBci ? CALL : LOOP, method(), inlinee(), bci, comp_level);
 916   }
 917 
 918   if (comp_level == CompLevel_none &&
 919       JvmtiExport::can_post_interpreter_events() &&
 920       THREAD->is_interp_only_mode()) {
 921     return nullptr;
 922   }
 923   if (ReplayCompiles) {
 924     // Don't trigger other compiles in testing mode
 925     return nullptr;
 926   }
 927 
 928   handle_counter_overflow(method);
 929   if (method() != inlinee()) {
 930     handle_counter_overflow(inlinee);
 931   }
 932 
 933   if (bci == InvocationEntryBci) {
 934     method_invocation_event(method, inlinee, comp_level, nm, THREAD);
 935   } else {
 936     // method == inlinee if the event originated in the main method
 937     method_back_branch_event(method, inlinee, bci, comp_level, nm, THREAD);
 938     // Check if event led to a higher level OSR compilation
 939     CompLevel expected_comp_level = MIN2(CompLevel_full_optimization, static_cast<CompLevel>(comp_level + 1));
 940     if (!CompilationModeFlag::disable_intermediate() && inlinee->is_not_osr_compilable(expected_comp_level)) {
 941       // It's not possible to reach the expected level so fall back to simple.
 942       expected_comp_level = CompLevel_simple;
 943     }
 944     CompLevel max_osr_level = static_cast<CompLevel>(inlinee->highest_osr_comp_level());
 945     if (max_osr_level >= expected_comp_level) { // fast check to avoid locking in a typical scenario
 946       nmethod* osr_nm = inlinee->lookup_osr_nmethod_for(bci, expected_comp_level, false);
 947       assert(osr_nm == nullptr || osr_nm->comp_level() >= expected_comp_level, "lookup_osr_nmethod_for is broken");
 948       if (osr_nm != nullptr && osr_nm->comp_level() != comp_level) {
 949         // Perform OSR with new nmethod
 950         return osr_nm;
 951       }
 952     }
 953   }
 954   return nullptr;
 955 }
 956 
 957 // Check if the method can be compiled, change level if necessary
 958 void CompilationPolicy::compile(const methodHandle& mh, int bci, CompLevel level, TRAPS) {
 959   assert(verify_level(level), "Invalid compilation level requested: %d", level);
 960 
 961   if (level == CompLevel_none) {
 962     if (mh->has_compiled_code()) {
 963       // Happens when we switch to interpreter to profile.
 964       MutexLocker ml(Compile_lock);
 965       NoSafepointVerifier nsv;
 966       if (mh->has_compiled_code()) {
 967         mh->code()->make_not_used();
 968       }
 969       // Deoptimize immediately (we don't have to wait for a compile).
 970       JavaThread* jt = THREAD;
 971       RegisterMap map(jt,
 972                       RegisterMap::UpdateMap::skip,
 973                       RegisterMap::ProcessFrames::include,
 974                       RegisterMap::WalkContinuation::skip);
 975       frame fr = jt->last_frame().sender(&map);
 976       Deoptimization::deoptimize_frame(jt, fr.id());
 977     }
 978     return;
 979   }
 980 
 981   // Check if the method can be compiled. Additional logic for TieredCompilation:
 982   // If it cannot be compiled with C1, continue profiling in the interpreter
 983   // and then compile with C2 (the transition function will request that,
 984   // see common() ). If the method cannot be compiled with C2 but still can with C1, compile it with
 985   // pure C1.
 986   if ((bci == InvocationEntryBci && !can_be_compiled(mh, level))) {
 987     if (!CompilationModeFlag::disable_intermediate() &&
 988         level == CompLevel_full_optimization && can_be_compiled(mh, CompLevel_simple)) {
 989       compile(mh, bci, CompLevel_simple, THREAD);
 990     }
 991     return;
 992   }
 993   if ((bci != InvocationEntryBci && !can_be_osr_compiled(mh, level))) {
 994     if (!CompilationModeFlag::disable_intermediate() &&
 995         level == CompLevel_full_optimization && can_be_osr_compiled(mh, CompLevel_simple)) {
 996       nmethod* osr_nm = mh->lookup_osr_nmethod_for(bci, CompLevel_simple, false);
 997       if (osr_nm != nullptr && osr_nm->comp_level() > CompLevel_simple) {
 998         // Invalidate the existing OSR nmethod so that a compile at CompLevel_simple is permitted.
 999         osr_nm->make_not_entrant(nmethod::InvalidationReason::OSR_INVALIDATION_FOR_COMPILING_WITH_C1);
1000       }
1001       compile(mh, bci, CompLevel_simple, THREAD);
1002     }
1003     return;
1004   }
1005   if (bci != InvocationEntryBci && mh->is_not_osr_compilable(level)) {
1006     return;
1007   }
1008   if (!CompileBroker::compilation_is_in_queue(mh)) {
1009     if (PrintTieredEvents) {
1010       print_event(COMPILE, mh(), mh(), bci, level);
1011     }
1012     int hot_count = (bci == InvocationEntryBci) ? mh->invocation_count() : mh->backedge_count();
1013     update_rate(nanos_to_millis(os::javaTimeNanos()), mh);
1014     bool requires_online_compilation = false;
1015     if (TrainingData::have_data()) {
1016       MethodTrainingData* mtd = MethodTrainingData::find_fast(mh);
1017       if (mtd != nullptr) {
1018         CompileTrainingData* ctd = mtd->last_toplevel_compile(level);
1019         if (ctd != nullptr) {
1020           requires_online_compilation = (ctd->init_deps_left_acquire() > 0);
1021         }
1022       }
1023     }
1024     CompileBroker::compile_method(mh, bci, level, hot_count, requires_online_compilation, CompileTask::Reason_Tiered, THREAD);
1025   }
1026 }
1027 
1028 // update_rate() is called from select_task() while holding a compile queue lock.
1029 void CompilationPolicy::update_rate(int64_t t, const methodHandle& method) {
1030   // Skip update if counters are absent.
1031   // Can't allocate them since we are holding compile queue lock.
1032   if (method->method_counters() == nullptr)  return;
1033 
1034   if (is_old(method)) {
1035     // We don't remove old methods from the queue,
1036     // so we can just zero the rate.
1037     method->set_rate(0);
1038     return;
1039   }
1040 
1041   // We don't update the rate if we've just came out of a safepoint.
1042   // delta_s is the time since last safepoint in milliseconds.
1043   int64_t delta_s = t - SafepointTracing::end_of_last_safepoint_ms();
1044   int64_t delta_t = t - (method->prev_time() != 0 ? method->prev_time() : start_time()); // milliseconds since the last measurement
1045   // How many events were there since the last time?
1046   int event_count = method->invocation_count() + method->backedge_count();
1047   int delta_e = event_count - method->prev_event_count();
1048 
1049   // We should be running for at least 1ms.
1050   if (delta_s >= TieredRateUpdateMinTime) {
1051     // And we must've taken the previous point at least 1ms before.
1052     if (delta_t >= TieredRateUpdateMinTime && delta_e > 0) {
1053       method->set_prev_time(t);
1054       method->set_prev_event_count(event_count);
1055       method->set_rate((float)delta_e / (float)delta_t); // Rate is events per millisecond
1056     } else {
1057       if (delta_t > TieredRateUpdateMaxTime && delta_e == 0) {
1058         // If nothing happened for 25ms, zero the rate. Don't modify prev values.
1059         method->set_rate(0);
1060       }
1061     }
1062   }
1063 }
1064 
1065 // Check if this method has been stale for a given number of milliseconds.
1066 // See select_task().
1067 bool CompilationPolicy::is_stale(int64_t t, int64_t timeout, const methodHandle& method) {
1068   int64_t delta_s = t - SafepointTracing::end_of_last_safepoint_ms();
1069   int64_t delta_t = t - method->prev_time();
1070   if (delta_t > timeout && delta_s > timeout) {
1071     int event_count = method->invocation_count() + method->backedge_count();
1072     int delta_e = event_count - method->prev_event_count();
1073     // Return true if there were no events.
1074     return delta_e == 0;
1075   }
1076   return false;
1077 }
1078 
1079 // We don't remove old methods from the compile queue even if they have
1080 // very low activity. See select_task().
1081 bool CompilationPolicy::is_old(const methodHandle& method) {
1082   int i = method->invocation_count();
1083   int b = method->backedge_count();
1084   double k = TieredOldPercentage / 100.0;
1085 
1086   return CallPredicate::apply_scaled(method, CompLevel_none, i, b, k) || LoopPredicate::apply_scaled(method, CompLevel_none, i, b, k);
1087 }
1088 
1089 double CompilationPolicy::weight(Method* method) {
1090   return (double)(method->rate() + 1) * (method->invocation_count() + 1) * (method->backedge_count() + 1);
1091 }
1092 
1093 // Apply heuristics and return true if x should be compiled before y
1094 bool CompilationPolicy::compare_methods(Method* x, Method* y) {
1095   if (x->highest_comp_level() > y->highest_comp_level()) {
1096     // recompilation after deopt
1097     return true;
1098   } else
1099     if (x->highest_comp_level() == y->highest_comp_level()) {
1100       if (weight(x) > weight(y)) {
1101         return true;
1102       }
1103     }
1104   return false;
1105 }
1106 
1107 bool CompilationPolicy::compare_tasks(CompileTask* x, CompileTask* y) {
1108   assert(!x->is_aot_load() && !y->is_aot_load(), "AOT code caching tasks are not expected here");
1109   if (x->compile_reason() != y->compile_reason() && y->compile_reason() == CompileTask::Reason_MustBeCompiled) {
1110     return true;
1111   }
1112   return false;
1113 }
1114 
1115 // Is method profiled enough?
1116 bool CompilationPolicy::is_method_profiled(const methodHandle& method) {
1117   MethodData* mdo = method->method_data();
1118   if (mdo != nullptr) {
1119     int i = mdo->invocation_count_delta();
1120     int b = mdo->backedge_count_delta();
1121     return CallPredicate::apply_scaled(method, CompLevel_full_profile, i, b, 1);
1122   }
1123   return false;
1124 }
1125 
1126 
1127 // Determine is a method is mature.
1128 bool CompilationPolicy::is_mature(MethodData* mdo) {
1129   if (Arguments::is_compiler_only()) {
1130     // Always report profiles as immature with -Xcomp
1131     return false;
1132   }
1133   methodHandle mh(Thread::current(), mdo->method());
1134   if (mdo != nullptr) {
1135     int i = mdo->invocation_count();
1136     int b = mdo->backedge_count();
1137     double k = ProfileMaturityPercentage / 100.0;
1138     return CallPredicate::apply_scaled(mh, CompLevel_full_profile, i, b, k) || LoopPredicate::apply_scaled(mh, CompLevel_full_profile, i, b, k);
1139   }
1140   return false;
1141 }
1142 
1143 // If a method is old enough and is still in the interpreter we would want to
1144 // start profiling without waiting for the compiled method to arrive.
1145 // We also take the load on compilers into the account.
1146 bool CompilationPolicy::should_create_mdo(const methodHandle& method, CompLevel cur_level) {
1147   if (cur_level != CompLevel_none || force_comp_at_level_simple(method) || CompilationModeFlag::quick_only() || !ProfileInterpreter) {
1148     return false;
1149   }
1150 
1151   if (TrainingData::have_data()) {
1152     MethodTrainingData* mtd = MethodTrainingData::find_fast(method);
1153     if (mtd != nullptr && mtd->saw_level(CompLevel_full_optimization)) {
1154       return true;
1155     }
1156   }
1157 
1158   if (is_old(method)) {
1159     return true;
1160   }
1161   int i = method->invocation_count();
1162   int b = method->backedge_count();
1163   double k = Tier0ProfilingStartPercentage / 100.0;
1164 
1165   // If the top level compiler is not keeping up, delay profiling.
1166   if (CompileBroker::queue_size(CompLevel_full_optimization) <= Tier0Delay * compiler_count(CompLevel_full_optimization)) {
1167     return CallPredicate::apply_scaled(method, CompLevel_none, i, b, k) || LoopPredicate::apply_scaled(method, CompLevel_none, i, b, k);
1168   }
1169   return false;
1170 }
1171 
1172 // Inlining control: if we're compiling a profiled method with C1 and the callee
1173 // is known to have OSRed in a C2 version, don't inline it.
1174 bool CompilationPolicy::should_not_inline(ciEnv* env, ciMethod* callee) {
1175   CompLevel comp_level = (CompLevel)env->comp_level();
1176   if (comp_level == CompLevel_full_profile ||
1177       comp_level == CompLevel_limited_profile) {
1178     return callee->highest_osr_comp_level() == CompLevel_full_optimization;
1179   }
1180   return false;
1181 }
1182 
1183 // Create MDO if necessary.
1184 void CompilationPolicy::create_mdo(const methodHandle& mh, JavaThread* THREAD) {
1185   if (mh->is_native() ||
1186       mh->is_abstract() ||
1187       mh->is_accessor() ||
1188       mh->is_constant_getter()) {
1189     return;
1190   }
1191   if (mh->method_data() == nullptr) {
1192     Method::build_profiling_method_data(mh, CHECK_AND_CLEAR);
1193   }
1194   if (ProfileInterpreter && THREAD->has_last_Java_frame()) {
1195     MethodData* mdo = mh->method_data();
1196     if (mdo != nullptr) {
1197       frame last_frame = THREAD->last_frame();
1198       if (last_frame.is_interpreted_frame() && mh == last_frame.interpreter_frame_method()) {
1199         int bci = last_frame.interpreter_frame_bci();
1200         address dp = mdo->bci_to_dp(bci);
1201         last_frame.interpreter_frame_set_mdp(dp);
1202       }
1203     }
1204   }
1205 }
1206 
1207 CompLevel CompilationPolicy::trained_transition_from_none(const methodHandle& method, CompLevel cur_level, MethodTrainingData* mtd, JavaThread* THREAD) {
1208   precond(mtd != nullptr);
1209   precond(cur_level == CompLevel_none);
1210 
1211   if (mtd->only_inlined() && !mtd->saw_level(CompLevel_full_optimization)) {
1212     return CompLevel_none;
1213   }
1214 
1215   bool training_has_profile = (mtd->final_profile() != nullptr);
1216   if (mtd->saw_level(CompLevel_full_optimization) && !training_has_profile) {
1217     return CompLevel_full_profile;
1218   }
1219 
1220   CompLevel highest_training_level = static_cast<CompLevel>(mtd->highest_top_level());
1221   switch (highest_training_level) {
1222     case CompLevel_limited_profile:
1223     case CompLevel_full_profile:
1224       return CompLevel_limited_profile;
1225     case CompLevel_simple:
1226       return CompLevel_simple;
1227     case CompLevel_none:
1228       return CompLevel_none;
1229     default:
1230       break;
1231   }
1232 
1233   // Now handle the case of level 4.
1234   assert(highest_training_level == CompLevel_full_optimization, "Unexpected compilation level: %d", highest_training_level);
1235   if (!training_has_profile) {
1236     // The method was a part of a level 4 compile, but don't have a stored profile,
1237     // we need to profile it.
1238     return CompLevel_full_profile;
1239   }
1240   const bool deopt = (static_cast<CompLevel>(method->highest_comp_level()) == CompLevel_full_optimization);
1241   // If we deopted, then we reprofile
1242   if (deopt && !is_method_profiled(method)) {
1243     return CompLevel_full_profile;
1244   }
1245 
1246   CompileTrainingData* ctd = mtd->last_toplevel_compile(CompLevel_full_optimization);
1247   assert(ctd != nullptr, "Should have CTD for CompLevel_full_optimization");
1248   // With SkipTier2IfPossible and all deps satisfied, go to level 4 immediately
1249   if (SkipTier2IfPossible && ctd->init_deps_left_acquire() == 0) {
1250     if (method->method_data() == nullptr) {
1251       create_mdo(method, THREAD);
1252     }
1253     return CompLevel_full_optimization;
1254   }
1255 
1256   // Otherwise go to level 2
1257   return CompLevel_limited_profile;
1258 }
1259 
1260 
1261 CompLevel CompilationPolicy::trained_transition_from_limited_profile(const methodHandle& method, CompLevel cur_level, MethodTrainingData* mtd, JavaThread* THREAD) {
1262   precond(mtd != nullptr);
1263   precond(cur_level == CompLevel_limited_profile);
1264 
1265   // One of the main reasons that we can get here is that we're waiting for the stored C2 code to become ready.
1266 
1267   // But first, check if we have a saved profile
1268   bool training_has_profile = (mtd->final_profile() != nullptr);
1269   if (!training_has_profile) {
1270     return CompLevel_full_profile;
1271   }
1272 
1273 
1274   assert(training_has_profile, "Have to have a profile to be here");
1275   // Check if the method is ready
1276   CompileTrainingData* ctd = mtd->last_toplevel_compile(CompLevel_full_optimization);
1277   if (ctd != nullptr && ctd->init_deps_left_acquire() == 0) {
1278     if (method->method_data() == nullptr) {
1279       create_mdo(method, THREAD);
1280     }
1281     return CompLevel_full_optimization;
1282   }
1283 
1284   // Otherwise stay at the current level
1285   return CompLevel_limited_profile;
1286 }
1287 
1288 
1289 CompLevel CompilationPolicy::trained_transition_from_full_profile(const methodHandle& method, CompLevel cur_level, MethodTrainingData* mtd, JavaThread* THREAD) {
1290   precond(mtd != nullptr);
1291   precond(cur_level == CompLevel_full_profile);
1292 
1293   CompLevel highest_training_level = static_cast<CompLevel>(mtd->highest_top_level());
1294   // We have method at the full profile level and we also know that it's possibly an important method.
1295   if (highest_training_level == CompLevel_full_optimization && !mtd->only_inlined()) {
1296     // Check if it is adequately profiled
1297     if (is_method_profiled(method)) {
1298       return CompLevel_full_optimization;
1299     }
1300   }
1301 
1302   // Otherwise stay at the current level
1303   return CompLevel_full_profile;
1304 }
1305 
1306 CompLevel CompilationPolicy::trained_transition(const methodHandle& method, CompLevel cur_level, MethodTrainingData* mtd, JavaThread* THREAD) {
1307   precond(MethodTrainingData::have_data());
1308 
1309   // If there is no training data recorded for this method, bail out.
1310   if (mtd == nullptr) {
1311     return cur_level;
1312   }
1313 
1314   CompLevel next_level = cur_level;
1315   switch(cur_level) {
1316     default: break;
1317     case CompLevel_none:
1318       next_level = trained_transition_from_none(method, cur_level, mtd, THREAD);
1319       break;
1320     case CompLevel_limited_profile:
1321       next_level = trained_transition_from_limited_profile(method, cur_level, mtd, THREAD);
1322       break;
1323     case CompLevel_full_profile:
1324       next_level = trained_transition_from_full_profile(method, cur_level, mtd, THREAD);
1325       break;
1326   }
1327 
1328   // We don't have any special strategies for the C2-only compilation modes, so just fix up the levels for now.
1329   if (CompilationModeFlag::high_only_quick_internal() && CompLevel_simple < next_level && next_level < CompLevel_full_optimization) {
1330     return CompLevel_none;
1331   }
1332   if (CompilationModeFlag::high_only() && next_level < CompLevel_full_optimization) {
1333     return CompLevel_none;
1334   }
1335   return (cur_level != next_level) ? limit_level(next_level) : cur_level;
1336 }
1337 
1338 /*
1339  * Method states:
1340  *   0 - interpreter (CompLevel_none)
1341  *   1 - pure C1 (CompLevel_simple)
1342  *   2 - C1 with invocation and backedge counting (CompLevel_limited_profile)
1343  *   3 - C1 with full profiling (CompLevel_full_profile)
1344  *   4 - C2 or Graal (CompLevel_full_optimization)
1345  *
1346  * Common state transition patterns:
1347  * a. 0 -> 3 -> 4.
1348  *    The most common path. But note that even in this straightforward case
1349  *    profiling can start at level 0 and finish at level 3.
1350  *
1351  * b. 0 -> 2 -> 3 -> 4.
1352  *    This case occurs when the load on C2 is deemed too high. So, instead of transitioning
1353  *    into state 3 directly and over-profiling while a method is in the C2 queue we transition to
1354  *    level 2 and wait until the load on C2 decreases. This path is disabled for OSRs.
1355  *
1356  * c. 0 -> (3->2) -> 4.
1357  *    In this case we enqueue a method for compilation at level 3, but the C1 queue is long enough
1358  *    to enable the profiling to fully occur at level 0. In this case we change the compilation level
1359  *    of the method to 2 while the request is still in-queue, because it'll allow it to run much faster
1360  *    without full profiling while c2 is compiling.
1361  *
1362  * d. 0 -> 3 -> 1 or 0 -> 2 -> 1.
1363  *    After a method was once compiled with C1 it can be identified as trivial and be compiled to
1364  *    level 1. These transition can also occur if a method can't be compiled with C2 but can with C1.
1365  *
1366  * e. 0 -> 4.
1367  *    This can happen if a method fails C1 compilation (it will still be profiled in the interpreter)
1368  *    or because of a deopt that didn't require reprofiling (compilation won't happen in this case because
1369  *    the compiled version already exists).
1370  *
1371  * Note that since state 0 can be reached from any other state via deoptimization different loops
1372  * are possible.
1373  *
1374  */
1375 
1376 // Common transition function. Given a predicate determines if a method should transition to another level.
1377 template<typename Predicate>
1378 CompLevel CompilationPolicy::common(const methodHandle& method, CompLevel cur_level, JavaThread* THREAD, bool disable_feedback) {
1379   CompLevel next_level = cur_level;
1380 
1381   if (force_comp_at_level_simple(method)) {
1382     next_level = CompLevel_simple;
1383   } else if (is_trivial(method) || method->is_native()) {
1384     // We do not care if there is profiling data for these methods, throw them to compiler.
1385     next_level = CompilationModeFlag::disable_intermediate() ? CompLevel_full_optimization : CompLevel_simple;
1386   } else if (MethodTrainingData::have_data()) {
1387     MethodTrainingData* mtd = MethodTrainingData::find_fast(method);
1388     if (mtd == nullptr) {
1389       // We haven't see compilations of this method in training. It's either very cold or the behavior changed.
1390       // Feed it to the standard TF with no profiling delay.
1391       next_level = standard_transition<Predicate>(method, cur_level, false /*delay_profiling*/, disable_feedback);
1392     } else {
1393       next_level = trained_transition(method, cur_level, mtd, THREAD);
1394       if (cur_level == next_level) {
1395         // trained_transtion() is going to return the same level if no startup/warmup optimizations apply.
1396         // In order to catch possible pathologies due to behavior change we feed the event to the regular
1397         // TF but with profiling delay.
1398         next_level = standard_transition<Predicate>(method, cur_level, true /*delay_profiling*/, disable_feedback);
1399       }
1400     }
1401   } else {
1402     next_level = standard_transition<Predicate>(method, cur_level, false /*delay_profiling*/, disable_feedback);
1403   }
1404   return (next_level != cur_level) ? limit_level(next_level) : next_level;
1405 }
1406 
1407 
1408 template<typename Predicate>
1409 CompLevel CompilationPolicy::standard_transition(const methodHandle& method, CompLevel cur_level, bool delay_profiling, bool disable_feedback) {
1410   CompLevel next_level = cur_level;
1411   switch(cur_level) {
1412   default: break;
1413   case CompLevel_none:
1414     next_level = transition_from_none<Predicate>(method, cur_level, delay_profiling, disable_feedback);
1415     break;
1416   case CompLevel_limited_profile:
1417     next_level = transition_from_limited_profile<Predicate>(method, cur_level, delay_profiling, disable_feedback);
1418     break;
1419   case CompLevel_full_profile:
1420     next_level = transition_from_full_profile<Predicate>(method, cur_level);
1421     break;
1422   }
1423   return next_level;
1424 }
1425 
1426 template<typename Predicate> static inline bool apply_predicate(const methodHandle& method, CompLevel cur_level, int i, int b, bool delay_profiling, double delay_profiling_scale) {
1427   if (delay_profiling) {
1428     return Predicate::apply_scaled(method, cur_level, i, b, delay_profiling_scale);
1429   } else {
1430     return Predicate::apply(method, cur_level, i, b);
1431   }
1432 }
1433 
1434 template<typename Predicate>
1435 CompLevel CompilationPolicy::transition_from_none(const methodHandle& method, CompLevel cur_level, bool delay_profiling, bool disable_feedback) {
1436   precond(cur_level == CompLevel_none);
1437   CompLevel next_level = cur_level;
1438   int i = method->invocation_count();
1439   int b = method->backedge_count();
1440   // If we were at full profile level, would we switch to full opt?
1441   if (transition_from_full_profile<Predicate>(method, CompLevel_full_profile) == CompLevel_full_optimization) {
1442     next_level = CompLevel_full_optimization;
1443   } else if (!CompilationModeFlag::disable_intermediate() && apply_predicate<Predicate>(method, cur_level, i, b, delay_profiling, Tier0ProfileDelayFactor)) {
1444     // C1-generated fully profiled code is about 30% slower than the limited profile
1445     // code that has only invocation and backedge counters. The observation is that
1446     // if C2 queue is large enough we can spend too much time in the fully profiled code
1447     // while waiting for C2 to pick the method from the queue. To alleviate this problem
1448     // we introduce a feedback on the C2 queue size. If the C2 queue is sufficiently long
1449     // we choose to compile a limited profiled version and then recompile with full profiling
1450     // when the load on C2 goes down.
1451     if (delay_profiling || (!disable_feedback && CompileBroker::queue_size(CompLevel_full_optimization) > Tier3DelayOn * compiler_count(CompLevel_full_optimization))) {
1452       next_level = CompLevel_limited_profile;
1453     } else {
1454       next_level = CompLevel_full_profile;
1455     }
1456   }
1457   return next_level;
1458 }
1459 
1460 template<typename Predicate>
1461 CompLevel CompilationPolicy::transition_from_full_profile(const methodHandle& method, CompLevel cur_level) {
1462   precond(cur_level == CompLevel_full_profile);
1463   CompLevel next_level = cur_level;
1464   MethodData* mdo = method->method_data();
1465   if (mdo != nullptr) {
1466     if (mdo->would_profile() || CompilationModeFlag::disable_intermediate()) {
1467       int mdo_i = mdo->invocation_count_delta();
1468       int mdo_b = mdo->backedge_count_delta();
1469       if (Predicate::apply(method, cur_level, mdo_i, mdo_b)) {
1470         next_level = CompLevel_full_optimization;
1471       }
1472     } else {
1473       next_level = CompLevel_full_optimization;
1474     }
1475   }
1476   return next_level;
1477 }
1478 
1479 template<typename Predicate>
1480 CompLevel CompilationPolicy::transition_from_limited_profile(const methodHandle& method, CompLevel cur_level, bool delay_profiling, bool disable_feedback) {
1481   precond(cur_level == CompLevel_limited_profile);
1482   CompLevel next_level = cur_level;
1483   int i = method->invocation_count();
1484   int b = method->backedge_count();
1485   MethodData* mdo = method->method_data();
1486   if (mdo != nullptr) {
1487     if (mdo->would_profile()) {
1488       if (disable_feedback || (CompileBroker::queue_size(CompLevel_full_optimization) <=
1489                               Tier3DelayOff * compiler_count(CompLevel_full_optimization) &&
1490                               apply_predicate<Predicate>(method, cur_level, i, b, delay_profiling, Tier2ProfileDelayFactor))) {
1491         next_level = CompLevel_full_profile;
1492       }
1493     } else {
1494       next_level = CompLevel_full_optimization;
1495     }
1496   } else {
1497     // If there is no MDO we need to profile
1498     if (disable_feedback || (CompileBroker::queue_size(CompLevel_full_optimization) <=
1499                             Tier3DelayOff * compiler_count(CompLevel_full_optimization) &&
1500                             apply_predicate<Predicate>(method, cur_level, i, b, delay_profiling, Tier2ProfileDelayFactor))) {
1501       next_level = CompLevel_full_profile;
1502     }
1503   }
1504   if (next_level == CompLevel_full_profile && is_method_profiled(method)) {
1505     next_level = CompLevel_full_optimization;
1506   }
1507   return next_level;
1508 }
1509 
1510 
1511 // Determine if a method should be compiled with a normal entry point at a different level.
1512 CompLevel CompilationPolicy::call_event(const methodHandle& method, CompLevel cur_level, JavaThread* THREAD) {
1513   CompLevel osr_level = MIN2((CompLevel) method->highest_osr_comp_level(), common<LoopPredicate>(method, cur_level, THREAD, true));
1514   CompLevel next_level = common<CallPredicate>(method, cur_level, THREAD, !TrainingData::have_data() && is_old(method));
1515 
1516   // If OSR method level is greater than the regular method level, the levels should be
1517   // equalized by raising the regular method level in order to avoid OSRs during each
1518   // invocation of the method.
1519   if (osr_level == CompLevel_full_optimization && cur_level == CompLevel_full_profile) {
1520     MethodData* mdo = method->method_data();
1521     guarantee(mdo != nullptr, "MDO should not be nullptr");
1522     if (mdo->invocation_count() >= 1) {
1523       next_level = CompLevel_full_optimization;
1524     }
1525   } else {
1526     next_level = MAX2(osr_level, next_level);
1527   }
1528 
1529   return next_level;
1530 }
1531 
1532 // Determine if we should do an OSR compilation of a given method.
1533 CompLevel CompilationPolicy::loop_event(const methodHandle& method, CompLevel cur_level, JavaThread* THREAD) {
1534   CompLevel next_level = common<LoopPredicate>(method, cur_level, THREAD, true);
1535   if (cur_level == CompLevel_none) {
1536     // If there is a live OSR method that means that we deopted to the interpreter
1537     // for the transition.
1538     CompLevel osr_level = MIN2((CompLevel)method->highest_osr_comp_level(), next_level);
1539     if (osr_level > CompLevel_none) {
1540       return osr_level;
1541     }
1542   }
1543   return next_level;
1544 }
1545 
1546 // Handle the invocation event.
1547 void CompilationPolicy::method_invocation_event(const methodHandle& mh, const methodHandle& imh,
1548                                                       CompLevel level, nmethod* nm, TRAPS) {
1549   if (should_create_mdo(mh, level)) {
1550     create_mdo(mh, THREAD);
1551   }
1552   CompLevel next_level = call_event(mh, level, THREAD);
1553   if (next_level != level) {
1554     if (is_compilation_enabled() && !CompileBroker::compilation_is_in_queue(mh)) {
1555       compile(mh, InvocationEntryBci, next_level, THREAD);
1556     }
1557   }
1558 }
1559 
1560 // Handle the back branch event. Notice that we can compile the method
1561 // with a regular entry from here.
1562 void CompilationPolicy::method_back_branch_event(const methodHandle& mh, const methodHandle& imh,
1563                                                      int bci, CompLevel level, nmethod* nm, TRAPS) {
1564   if (should_create_mdo(mh, level)) {
1565     create_mdo(mh, THREAD);
1566   }
1567   // Check if MDO should be created for the inlined method
1568   if (should_create_mdo(imh, level)) {
1569     create_mdo(imh, THREAD);
1570   }
1571 
1572   if (is_compilation_enabled()) {
1573     CompLevel next_osr_level = loop_event(imh, level, THREAD);
1574     CompLevel max_osr_level = (CompLevel)imh->highest_osr_comp_level();
1575     // At the very least compile the OSR version
1576     if (!CompileBroker::compilation_is_in_queue(imh) && (next_osr_level != level)) {
1577       compile(imh, bci, next_osr_level, CHECK);
1578     }
1579 
1580     // Use loop event as an opportunity to also check if there's been
1581     // enough calls.
1582     CompLevel cur_level, next_level;
1583     if (mh() != imh()) { // If there is an enclosing method
1584       {
1585         guarantee(nm != nullptr, "Should have nmethod here");
1586         cur_level = comp_level(mh());
1587         next_level = call_event(mh, cur_level, THREAD);
1588 
1589         if (max_osr_level == CompLevel_full_optimization) {
1590           // The inlinee OSRed to full opt, we need to modify the enclosing method to avoid deopts
1591           bool make_not_entrant = false;
1592           if (nm->is_osr_method()) {
1593             // This is an osr method, just make it not entrant and recompile later if needed
1594             make_not_entrant = true;
1595           } else {
1596             if (next_level != CompLevel_full_optimization) {
1597               // next_level is not full opt, so we need to recompile the
1598               // enclosing method without the inlinee
1599               cur_level = CompLevel_none;
1600               make_not_entrant = true;
1601             }
1602           }
1603           if (make_not_entrant) {
1604             if (PrintTieredEvents) {
1605               int osr_bci = nm->is_osr_method() ? nm->osr_entry_bci() : InvocationEntryBci;
1606               print_event(MAKE_NOT_ENTRANT, mh(), mh(), osr_bci, level);
1607             }
1608             nm->make_not_entrant(nmethod::InvalidationReason::OSR_INVALIDATION_BACK_BRANCH);
1609           }
1610         }
1611         // Fix up next_level if necessary to avoid deopts
1612         if (next_level == CompLevel_limited_profile && max_osr_level == CompLevel_full_profile) {
1613           next_level = CompLevel_full_profile;
1614         }
1615         if (cur_level != next_level) {
1616           if (!CompileBroker::compilation_is_in_queue(mh)) {
1617             compile(mh, InvocationEntryBci, next_level, THREAD);
1618           }
1619         }
1620       }
1621     } else {
1622       cur_level = comp_level(mh());
1623       next_level = call_event(mh, cur_level, THREAD);
1624       if (next_level != cur_level) {
1625         if (!CompileBroker::compilation_is_in_queue(mh)) {
1626           compile(mh, InvocationEntryBci, next_level, THREAD);
1627         }
1628       }
1629     }
1630   }
1631 }
1632