1 /*
   2  * Copyright (c) 2024, Oracle and/or its affiliates. All rights reserved.
   3  * Copyright (c) 2021, 2022, Red Hat, Inc. All rights reserved.
   4  * Copyright Amazon.com Inc. or its affiliates. All Rights Reserved.
   5  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
   6  *
   7  * This code is free software; you can redistribute it and/or modify it
   8  * under the terms of the GNU General Public License version 2 only, as
   9  * published by the Free Software Foundation.
  10  *
  11  * This code is distributed in the hope that it will be useful, but WITHOUT
  12  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  13  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  14  * version 2 for more details (a copy is included in the LICENSE file that
  15  * accompanied this code).
  16  *
  17  * You should have received a copy of the GNU General Public License version
  18  * 2 along with this work; if not, write to the Free Software Foundation,
  19  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
  20  *
  21  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
  22  * or visit www.oracle.com if you need additional information or have any
  23  * questions.
  24  *
  25  */
  26 
  27 #include "precompiled.hpp"
  28 
  29 #include "gc/shared/barrierSetNMethod.hpp"
  30 #include "gc/shared/collectorCounters.hpp"
  31 #include "gc/shared/continuationGCSupport.inline.hpp"
  32 #include "gc/shenandoah/shenandoahBreakpoint.hpp"
  33 #include "gc/shenandoah/shenandoahCollectorPolicy.hpp"
  34 #include "gc/shenandoah/shenandoahConcurrentGC.hpp"
  35 #include "gc/shenandoah/shenandoahFreeSet.hpp"
  36 #include "gc/shenandoah/shenandoahGeneration.hpp"
  37 #include "gc/shenandoah/shenandoahGenerationalHeap.hpp"
  38 #include "gc/shenandoah/shenandoahOldGeneration.hpp"
  39 #include "gc/shenandoah/shenandoahYoungGeneration.hpp"
  40 #include "gc/shenandoah/shenandoahLock.hpp"
  41 #include "gc/shenandoah/shenandoahMark.inline.hpp"
  42 #include "gc/shenandoah/shenandoahMonitoringSupport.hpp"
  43 #include "gc/shenandoah/shenandoahOopClosures.inline.hpp"
  44 #include "gc/shenandoah/shenandoahPhaseTimings.hpp"
  45 #include "gc/shenandoah/shenandoahReferenceProcessor.hpp"
  46 #include "gc/shenandoah/shenandoahRootProcessor.inline.hpp"
  47 #include "gc/shenandoah/shenandoahStackWatermark.hpp"
  48 #include "gc/shenandoah/shenandoahUtils.hpp"
  49 #include "gc/shenandoah/shenandoahVerifier.hpp"
  50 #include "gc/shenandoah/shenandoahVMOperations.hpp"
  51 #include "gc/shenandoah/shenandoahWorkGroup.hpp"
  52 #include "gc/shenandoah/shenandoahWorkerPolicy.hpp"
  53 #include "memory/allocation.hpp"
  54 #include "prims/jvmtiTagMap.hpp"
  55 #include "runtime/vmThread.hpp"
  56 #include "utilities/events.hpp"
  57 
  58 // Breakpoint support
  59 class ShenandoahBreakpointGCScope : public StackObj {
  60 private:
  61   const GCCause::Cause _cause;
  62 public:
  63   ShenandoahBreakpointGCScope(GCCause::Cause cause) : _cause(cause) {
  64     if (cause == GCCause::_wb_breakpoint) {
  65       ShenandoahBreakpoint::start_gc();
  66       ShenandoahBreakpoint::at_before_gc();
  67     }
  68   }
  69 
  70   ~ShenandoahBreakpointGCScope() {
  71     if (_cause == GCCause::_wb_breakpoint) {
  72       ShenandoahBreakpoint::at_after_gc();
  73     }
  74   }
  75 };
  76 
  77 class ShenandoahBreakpointMarkScope : public StackObj {
  78 private:
  79   const GCCause::Cause _cause;
  80 public:
  81   ShenandoahBreakpointMarkScope(GCCause::Cause cause) : _cause(cause) {
  82     if (_cause == GCCause::_wb_breakpoint) {
  83       ShenandoahBreakpoint::at_after_marking_started();
  84     }
  85   }
  86 
  87   ~ShenandoahBreakpointMarkScope() {
  88     if (_cause == GCCause::_wb_breakpoint) {
  89       ShenandoahBreakpoint::at_before_marking_completed();
  90     }
  91   }
  92 };
  93 
  94 ShenandoahConcurrentGC::ShenandoahConcurrentGC(ShenandoahGeneration* generation, bool do_old_gc_bootstrap) :
  95   _mark(generation),
  96   _degen_point(ShenandoahDegenPoint::_degenerated_unset),
  97   _abbreviated(false),
  98   _do_old_gc_bootstrap(do_old_gc_bootstrap),
  99   _generation(generation) {
 100 }
 101 
 102 ShenandoahGC::ShenandoahDegenPoint ShenandoahConcurrentGC::degen_point() const {
 103   return _degen_point;
 104 }
 105 
 106 bool ShenandoahConcurrentGC::collect(GCCause::Cause cause) {
 107   ShenandoahHeap* const heap = ShenandoahHeap::heap();
 108 
 109   ShenandoahBreakpointGCScope breakpoint_gc_scope(cause);
 110 
 111   // Reset for upcoming marking
 112   entry_reset();
 113 
 114   // Start initial mark under STW
 115   vmop_entry_init_mark();
 116 
 117   {
 118     ShenandoahBreakpointMarkScope breakpoint_mark_scope(cause);
 119 
 120     // Reset task queue stats here, rather than in mark_concurrent_roots,
 121     // because remembered set scan will `push` oops into the queues and
 122     // resetting after this happens will lose those counts.
 123     TASKQUEUE_STATS_ONLY(_mark.task_queues()->reset_taskqueue_stats());
 124 
 125     // Concurrent remembered set scanning
 126     entry_scan_remembered_set();
 127 
 128     // Concurrent mark roots
 129     entry_mark_roots();
 130     if (check_cancellation_and_abort(ShenandoahDegenPoint::_degenerated_roots)) {
 131       return false;
 132     }
 133 
 134     // Continue concurrent mark
 135     entry_mark();
 136     if (check_cancellation_and_abort(ShenandoahDegenPoint::_degenerated_mark)) {
 137       return false;
 138     }
 139   }
 140 
 141   // Complete marking under STW, and start evacuation
 142   vmop_entry_final_mark();
 143 
 144   // If the GC was cancelled just before final mark (but after the preceding cancellation check),
 145   // then the safepoint operation will do nothing and the concurrent mark will still be in progress.
 146   // In this case it is safe (and necessary) to resume the degenerated cycle from the marking phase.
 147   //
 148   // On the other hand, if the GC is cancelled after final mark (but before this check), then the
 149   // final mark safepoint operation will have finished the mark (setting concurrent mark in progress
 150   // to false). In this case (final mark has completed), we need control to fall past the next
 151   // cancellation check and resume the degenerated cycle from the evacuation phase.
 152   if (_generation->is_concurrent_mark_in_progress()) {
 153     // If the concurrent mark is still in progress after the final mark safepoint, then the GC has
 154     // been cancelled. The degenerated cycle must resume from the marking phase. Without this check,
 155     // the non-generational mode may fall all the way to the end of this collect routine without
 156     // having done anything (besides mark most of the heap). Without having collected anything, we
 157     // can expect an 'out of cycle' degenerated GC which will again mark the entire heap. This is
 158     // not optimal.
 159     // For the generational mode, we cannot allow this. The generational mode relies on marking
 160     // (including the final mark) to rebuild portions of the card table. If the generational mode does
 161     // not complete marking after it has swapped the card tables, the root set on subsequent GCs will
 162     // be incomplete, heap corruption may follow.
 163     bool cancelled = check_cancellation_and_abort(ShenandoahDegenPoint::_degenerated_mark);
 164     assert(cancelled, "GC must have been cancelled between concurrent and final mark");
 165     return false;
 166   }
 167 
 168   // Concurrent stack processing
 169   if (heap->is_evacuation_in_progress()) {
 170     entry_thread_roots();
 171   }
 172 
 173   // Process weak roots that might still point to regions that would be broken by cleanup
 174   if (heap->is_concurrent_weak_root_in_progress()) {
 175     entry_weak_refs();
 176     entry_weak_roots();
 177   }
 178 
 179   // Final mark might have reclaimed some immediate garbage, kick cleanup to reclaim
 180   // the space. This would be the last action if there is nothing to evacuate.  Note that
 181   // we will not age young-gen objects in the case that we skip evacuation.
 182   entry_cleanup_early();
 183 
 184   heap->free_set()->log_status_under_lock();
 185 
 186   // Perform concurrent class unloading
 187   if (heap->unload_classes() &&
 188       heap->is_concurrent_weak_root_in_progress()) {
 189     entry_class_unloading();
 190   }
 191 
 192   // Processing strong roots
 193   // This may be skipped if there is nothing to update/evacuate.
 194   // If so, strong_root_in_progress would be unset.
 195   if (heap->is_concurrent_strong_root_in_progress()) {
 196     entry_strong_roots();
 197   }
 198 
 199   // Continue the cycle with evacuation and optional update-refs.
 200   // This may be skipped if there is nothing to evacuate.
 201   // If so, evac_in_progress would be unset by collection set preparation code.
 202   if (heap->is_evacuation_in_progress()) {
 203     // Concurrently evacuate
 204     entry_evacuate();
 205     if (check_cancellation_and_abort(ShenandoahDegenPoint::_degenerated_evac)) {
 206       return false;
 207     }
 208 
 209     // Perform update-refs phase.
 210     vmop_entry_init_updaterefs();
 211     entry_updaterefs();
 212     if (check_cancellation_and_abort(ShenandoahDegenPoint::_degenerated_updaterefs)) {
 213       return false;
 214     }
 215 
 216     // Concurrent update thread roots
 217     entry_update_thread_roots();
 218     if (check_cancellation_and_abort(ShenandoahDegenPoint::_degenerated_updaterefs)) {
 219       return false;
 220     }
 221 
 222     vmop_entry_final_updaterefs();
 223 
 224     // Update references freed up collection set, kick the cleanup to reclaim the space.
 225     entry_cleanup_complete();
 226   } else {
 227     // We chose not to evacuate because we found sufficient immediate garbage.
 228     // However, there may still be regions to promote in place, so do that now.
 229     if (has_in_place_promotions(heap)) {
 230       entry_promote_in_place();
 231 
 232       // If the promote-in-place operation was cancelled, we can have the degenerated
 233       // cycle complete the operation. It will see that no evacuations are in progress,
 234       // and that there are regions wanting promotion. The risk with not handling the
 235       // cancellation would be failing to restore top for these regions and leaving
 236       // them unable to serve allocations for the old generation.
 237       if (check_cancellation_and_abort(ShenandoahDegenPoint::_degenerated_evac)) {
 238         return false;
 239       }
 240     }
 241 
 242     // At this point, the cycle is effectively complete. If the cycle has been cancelled here,
 243     // the control thread will detect it on its next iteration and run a degenerated young cycle.
 244     vmop_entry_final_roots();
 245     _abbreviated = true;
 246   }
 247 
 248   // We defer generation resizing actions until after cset regions have been recycled.  We do this even following an
 249   // abbreviated cycle.
 250   if (heap->mode()->is_generational()) {
 251     ShenandoahGenerationalHeap::heap()->complete_concurrent_cycle();
 252   }
 253   return true;
 254 }
 255 
 256 void ShenandoahConcurrentGC::vmop_entry_init_mark() {
 257   ShenandoahHeap* const heap = ShenandoahHeap::heap();
 258   TraceCollectorStats tcs(heap->monitoring_support()->stw_collection_counters());
 259   ShenandoahTimingsTracker timing(ShenandoahPhaseTimings::init_mark_gross);
 260 
 261   heap->try_inject_alloc_failure();
 262   VM_ShenandoahInitMark op(this);
 263   VMThread::execute(&op); // jump to entry_init_mark() under safepoint
 264 }
 265 
 266 void ShenandoahConcurrentGC::vmop_entry_final_mark() {
 267   ShenandoahHeap* const heap = ShenandoahHeap::heap();
 268   TraceCollectorStats tcs(heap->monitoring_support()->stw_collection_counters());
 269   ShenandoahTimingsTracker timing(ShenandoahPhaseTimings::final_mark_gross);
 270 
 271   heap->try_inject_alloc_failure();
 272   VM_ShenandoahFinalMarkStartEvac op(this);
 273   VMThread::execute(&op); // jump to entry_final_mark under safepoint
 274 }
 275 
 276 void ShenandoahConcurrentGC::vmop_entry_init_updaterefs() {
 277   ShenandoahHeap* const heap = ShenandoahHeap::heap();
 278   TraceCollectorStats tcs(heap->monitoring_support()->stw_collection_counters());
 279   ShenandoahTimingsTracker timing(ShenandoahPhaseTimings::init_update_refs_gross);
 280 
 281   heap->try_inject_alloc_failure();
 282   VM_ShenandoahInitUpdateRefs op(this);
 283   VMThread::execute(&op);
 284 }
 285 
 286 void ShenandoahConcurrentGC::vmop_entry_final_updaterefs() {
 287   ShenandoahHeap* const heap = ShenandoahHeap::heap();
 288   TraceCollectorStats tcs(heap->monitoring_support()->stw_collection_counters());
 289   ShenandoahTimingsTracker timing(ShenandoahPhaseTimings::final_update_refs_gross);
 290 
 291   heap->try_inject_alloc_failure();
 292   VM_ShenandoahFinalUpdateRefs op(this);
 293   VMThread::execute(&op);
 294 }
 295 
 296 void ShenandoahConcurrentGC::vmop_entry_final_roots() {
 297   ShenandoahHeap* const heap = ShenandoahHeap::heap();
 298   TraceCollectorStats tcs(heap->monitoring_support()->stw_collection_counters());
 299   ShenandoahTimingsTracker timing(ShenandoahPhaseTimings::final_roots_gross);
 300 
 301   // This phase does not use workers, no need for setup
 302   heap->try_inject_alloc_failure();
 303   VM_ShenandoahFinalRoots op(this);
 304   VMThread::execute(&op);
 305 }
 306 
 307 void ShenandoahConcurrentGC::entry_init_mark() {
 308   const char* msg = init_mark_event_message();
 309   ShenandoahPausePhase gc_phase(msg, ShenandoahPhaseTimings::init_mark);
 310   EventMark em("%s", msg);
 311 
 312   ShenandoahWorkerScope scope(ShenandoahHeap::heap()->workers(),
 313                               ShenandoahWorkerPolicy::calc_workers_for_init_marking(),
 314                               "init marking");
 315 
 316   op_init_mark();
 317 }
 318 
 319 void ShenandoahConcurrentGC::entry_final_mark() {
 320   const char* msg = final_mark_event_message();
 321   ShenandoahPausePhase gc_phase(msg, ShenandoahPhaseTimings::final_mark);
 322   EventMark em("%s", msg);
 323 
 324   ShenandoahWorkerScope scope(ShenandoahHeap::heap()->workers(),
 325                               ShenandoahWorkerPolicy::calc_workers_for_final_marking(),
 326                               "final marking");
 327 
 328   op_final_mark();
 329 }
 330 
 331 void ShenandoahConcurrentGC::entry_init_updaterefs() {
 332   static const char* msg = "Pause Init Update Refs";
 333   ShenandoahPausePhase gc_phase(msg, ShenandoahPhaseTimings::init_update_refs);
 334   EventMark em("%s", msg);
 335 
 336   // No workers used in this phase, no setup required
 337   op_init_updaterefs();
 338 }
 339 
 340 void ShenandoahConcurrentGC::entry_final_updaterefs() {
 341   static const char* msg = "Pause Final Update Refs";
 342   ShenandoahPausePhase gc_phase(msg, ShenandoahPhaseTimings::final_update_refs);
 343   EventMark em("%s", msg);
 344 
 345   ShenandoahWorkerScope scope(ShenandoahHeap::heap()->workers(),
 346                               ShenandoahWorkerPolicy::calc_workers_for_final_update_ref(),
 347                               "final reference update");
 348 
 349   op_final_updaterefs();
 350 }
 351 
 352 void ShenandoahConcurrentGC::entry_final_roots() {
 353   const char* msg = final_roots_event_message();
 354   ShenandoahPausePhase gc_phase(msg, ShenandoahPhaseTimings::final_roots);
 355   EventMark em("%s", msg);
 356 
 357   op_final_roots();
 358 }
 359 
 360 void ShenandoahConcurrentGC::entry_reset() {
 361   ShenandoahHeap* const heap = ShenandoahHeap::heap();
 362   heap->try_inject_alloc_failure();
 363 
 364   TraceCollectorStats tcs(heap->monitoring_support()->concurrent_collection_counters());
 365   {
 366     const char* msg = conc_reset_event_message();
 367     ShenandoahConcurrentPhase gc_phase(msg, ShenandoahPhaseTimings::conc_reset);
 368     EventMark em("%s", msg);
 369 
 370     ShenandoahWorkerScope scope(heap->workers(),
 371                                 ShenandoahWorkerPolicy::calc_workers_for_conc_reset(),
 372                                 msg);
 373     op_reset();
 374   }
 375 
 376   if (_do_old_gc_bootstrap) {
 377     static const char* msg = "Concurrent reset (OLD)";
 378     ShenandoahConcurrentPhase gc_phase(msg, ShenandoahPhaseTimings::conc_reset_old);
 379     ShenandoahWorkerScope scope(ShenandoahHeap::heap()->workers(),
 380                                 ShenandoahWorkerPolicy::calc_workers_for_conc_reset(),
 381                                 msg);
 382     EventMark em("%s", msg);
 383 
 384     heap->old_generation()->prepare_gc();
 385   }
 386 }
 387 
 388 void ShenandoahConcurrentGC::entry_scan_remembered_set() {
 389   if (_generation->is_young()) {
 390     ShenandoahHeap* const heap = ShenandoahHeap::heap();
 391     TraceCollectorStats tcs(heap->monitoring_support()->concurrent_collection_counters());
 392     const char* msg = "Concurrent remembered set scanning";
 393     ShenandoahConcurrentPhase gc_phase(msg, ShenandoahPhaseTimings::init_scan_rset);
 394     EventMark em("%s", msg);
 395 
 396     ShenandoahWorkerScope scope(heap->workers(),
 397                                 ShenandoahWorkerPolicy::calc_workers_for_rs_scanning(),
 398                                 msg);
 399 
 400     heap->try_inject_alloc_failure();
 401     _generation->scan_remembered_set(true /* is_concurrent */);
 402   }
 403 }
 404 
 405 void ShenandoahConcurrentGC::entry_mark_roots() {
 406   ShenandoahHeap* const heap = ShenandoahHeap::heap();
 407   TraceCollectorStats tcs(heap->monitoring_support()->concurrent_collection_counters());
 408   const char* msg = "Concurrent marking roots";
 409   ShenandoahConcurrentPhase gc_phase(msg, ShenandoahPhaseTimings::conc_mark_roots);
 410   EventMark em("%s", msg);
 411 
 412   ShenandoahWorkerScope scope(heap->workers(),
 413                               ShenandoahWorkerPolicy::calc_workers_for_conc_marking(),
 414                               "concurrent marking roots");
 415 
 416   heap->try_inject_alloc_failure();
 417   op_mark_roots();
 418 }
 419 
 420 void ShenandoahConcurrentGC::entry_mark() {
 421   ShenandoahHeap* const heap = ShenandoahHeap::heap();
 422   TraceCollectorStats tcs(heap->monitoring_support()->concurrent_collection_counters());
 423   const char* msg = conc_mark_event_message();
 424   ShenandoahConcurrentPhase gc_phase(msg, ShenandoahPhaseTimings::conc_mark);
 425   EventMark em("%s", msg);
 426 
 427   ShenandoahWorkerScope scope(heap->workers(),
 428                               ShenandoahWorkerPolicy::calc_workers_for_conc_marking(),
 429                               "concurrent marking");
 430 
 431   heap->try_inject_alloc_failure();
 432   op_mark();
 433 }
 434 
 435 void ShenandoahConcurrentGC::entry_thread_roots() {
 436   ShenandoahHeap* const heap = ShenandoahHeap::heap();
 437   static const char* msg = "Concurrent thread roots";
 438   ShenandoahConcurrentPhase gc_phase(msg, ShenandoahPhaseTimings::conc_thread_roots);
 439   EventMark em("%s", msg);
 440 
 441   ShenandoahWorkerScope scope(heap->workers(),
 442                               ShenandoahWorkerPolicy::calc_workers_for_conc_root_processing(),
 443                               msg);
 444 
 445   heap->try_inject_alloc_failure();
 446   op_thread_roots();
 447 }
 448 
 449 void ShenandoahConcurrentGC::entry_weak_refs() {
 450   ShenandoahHeap* const heap = ShenandoahHeap::heap();
 451   const char* msg = conc_weak_refs_event_message();
 452   ShenandoahConcurrentPhase gc_phase(msg, ShenandoahPhaseTimings::conc_weak_refs);
 453   EventMark em("%s", msg);
 454 
 455   ShenandoahWorkerScope scope(heap->workers(),
 456                               ShenandoahWorkerPolicy::calc_workers_for_conc_refs_processing(),
 457                               "concurrent weak references");
 458 
 459   heap->try_inject_alloc_failure();
 460   op_weak_refs();
 461 }
 462 
 463 void ShenandoahConcurrentGC::entry_weak_roots() {
 464   ShenandoahHeap* const heap = ShenandoahHeap::heap();
 465   TraceCollectorStats tcs(heap->monitoring_support()->concurrent_collection_counters());
 466   const char* msg = conc_weak_roots_event_message();
 467   ShenandoahConcurrentPhase gc_phase(msg, ShenandoahPhaseTimings::conc_weak_roots);
 468   EventMark em("%s", msg);
 469 
 470   ShenandoahWorkerScope scope(heap->workers(),
 471                               ShenandoahWorkerPolicy::calc_workers_for_conc_root_processing(),
 472                               "concurrent weak root");
 473 
 474   heap->try_inject_alloc_failure();
 475   op_weak_roots();
 476 }
 477 
 478 void ShenandoahConcurrentGC::entry_class_unloading() {
 479   ShenandoahHeap* const heap = ShenandoahHeap::heap();
 480   TraceCollectorStats tcs(heap->monitoring_support()->concurrent_collection_counters());
 481   static const char* msg = "Concurrent class unloading";
 482   ShenandoahConcurrentPhase gc_phase(msg, ShenandoahPhaseTimings::conc_class_unload);
 483   EventMark em("%s", msg);
 484 
 485   ShenandoahWorkerScope scope(heap->workers(),
 486                               ShenandoahWorkerPolicy::calc_workers_for_conc_root_processing(),
 487                               "concurrent class unloading");
 488 
 489   heap->try_inject_alloc_failure();
 490   op_class_unloading();
 491 }
 492 
 493 void ShenandoahConcurrentGC::entry_strong_roots() {
 494   ShenandoahHeap* const heap = ShenandoahHeap::heap();
 495   TraceCollectorStats tcs(heap->monitoring_support()->concurrent_collection_counters());
 496   static const char* msg = "Concurrent strong roots";
 497   ShenandoahConcurrentPhase gc_phase(msg, ShenandoahPhaseTimings::conc_strong_roots);
 498   EventMark em("%s", msg);
 499 
 500   ShenandoahGCWorkerPhase worker_phase(ShenandoahPhaseTimings::conc_strong_roots);
 501 
 502   ShenandoahWorkerScope scope(heap->workers(),
 503                               ShenandoahWorkerPolicy::calc_workers_for_conc_root_processing(),
 504                               "concurrent strong root");
 505 
 506   heap->try_inject_alloc_failure();
 507   op_strong_roots();
 508 }
 509 
 510 void ShenandoahConcurrentGC::entry_cleanup_early() {
 511   ShenandoahHeap* const heap = ShenandoahHeap::heap();
 512   TraceCollectorStats tcs(heap->monitoring_support()->concurrent_collection_counters());
 513   const char* msg = conc_cleanup_event_message();
 514   ShenandoahConcurrentPhase gc_phase(msg, ShenandoahPhaseTimings::conc_cleanup_early, true /* log_heap_usage */);
 515   EventMark em("%s", msg);
 516 
 517   // This phase does not use workers, no need for setup
 518   heap->try_inject_alloc_failure();
 519   op_cleanup_early();
 520 }
 521 
 522 void ShenandoahConcurrentGC::entry_evacuate() {
 523   ShenandoahHeap* const heap = ShenandoahHeap::heap();
 524   TraceCollectorStats tcs(heap->monitoring_support()->concurrent_collection_counters());
 525 
 526   static const char* msg = "Concurrent evacuation";
 527   ShenandoahConcurrentPhase gc_phase(msg, ShenandoahPhaseTimings::conc_evac);
 528   EventMark em("%s", msg);
 529 
 530   ShenandoahWorkerScope scope(heap->workers(),
 531                               ShenandoahWorkerPolicy::calc_workers_for_conc_evac(),
 532                               "concurrent evacuation");
 533 
 534   heap->try_inject_alloc_failure();
 535   op_evacuate();
 536 }
 537 
 538 void ShenandoahConcurrentGC::entry_promote_in_place() {
 539   shenandoah_assert_generational();
 540 
 541   ShenandoahHeap* const heap = ShenandoahHeap::heap();
 542   TraceCollectorStats tcs(heap->monitoring_support()->concurrent_collection_counters());
 543 
 544   static const char* msg = "Promote in place";
 545   ShenandoahConcurrentPhase gc_phase(msg, ShenandoahPhaseTimings::promote_in_place);
 546   EventMark em("%s", msg);
 547 
 548   ShenandoahWorkerScope scope(heap->workers(),
 549                               ShenandoahWorkerPolicy::calc_workers_for_conc_evac(),
 550                               "promote in place");
 551 
 552   ShenandoahGenerationalHeap::heap()->promote_regions_in_place(true);
 553 }
 554 
 555 void ShenandoahConcurrentGC::entry_update_thread_roots() {
 556   ShenandoahHeap* const heap = ShenandoahHeap::heap();
 557   TraceCollectorStats tcs(heap->monitoring_support()->concurrent_collection_counters());
 558 
 559   static const char* msg = "Concurrent update thread roots";
 560   ShenandoahConcurrentPhase gc_phase(msg, ShenandoahPhaseTimings::conc_update_thread_roots);
 561   EventMark em("%s", msg);
 562 
 563   // No workers used in this phase, no setup required
 564   heap->try_inject_alloc_failure();
 565   op_update_thread_roots();
 566 }
 567 
 568 void ShenandoahConcurrentGC::entry_updaterefs() {
 569   ShenandoahHeap* const heap = ShenandoahHeap::heap();
 570   TraceCollectorStats tcs(heap->monitoring_support()->concurrent_collection_counters());
 571   static const char* msg = "Concurrent update references";
 572   ShenandoahConcurrentPhase gc_phase(msg, ShenandoahPhaseTimings::conc_update_refs);
 573   EventMark em("%s", msg);
 574 
 575   ShenandoahWorkerScope scope(heap->workers(),
 576                               ShenandoahWorkerPolicy::calc_workers_for_conc_update_ref(),
 577                               "concurrent reference update");
 578 
 579   heap->try_inject_alloc_failure();
 580   op_updaterefs();
 581 }
 582 
 583 void ShenandoahConcurrentGC::entry_cleanup_complete() {
 584   ShenandoahHeap* const heap = ShenandoahHeap::heap();
 585   TraceCollectorStats tcs(heap->monitoring_support()->concurrent_collection_counters());
 586   const char* msg = conc_cleanup_event_message();
 587   ShenandoahConcurrentPhase gc_phase(msg, ShenandoahPhaseTimings::conc_cleanup_complete, true /* log_heap_usage */);
 588   EventMark em("%s", msg);
 589 
 590   // This phase does not use workers, no need for setup
 591   heap->try_inject_alloc_failure();
 592   op_cleanup_complete();
 593 }
 594 
 595 void ShenandoahConcurrentGC::op_reset() {
 596   ShenandoahHeap* const heap = ShenandoahHeap::heap();
 597   if (ShenandoahPacing) {
 598     heap->pacer()->setup_for_reset();
 599   }
 600   _generation->prepare_gc();
 601 }
 602 
 603 class ShenandoahInitMarkUpdateRegionStateClosure : public ShenandoahHeapRegionClosure {
 604 private:
 605   ShenandoahMarkingContext* const _ctx;
 606 public:
 607   ShenandoahInitMarkUpdateRegionStateClosure() : _ctx(ShenandoahHeap::heap()->marking_context()) {}
 608 
 609   void heap_region_do(ShenandoahHeapRegion* r) {
 610     assert(!r->has_live(), "Region " SIZE_FORMAT " should have no live data", r->index());
 611     if (r->is_active()) {
 612       // Check if region needs updating its TAMS. We have updated it already during concurrent
 613       // reset, so it is very likely we don't need to do another write here.  Since most regions
 614       // are not "active", this path is relatively rare.
 615       if (_ctx->top_at_mark_start(r) != r->top()) {
 616         _ctx->capture_top_at_mark_start(r);
 617       }
 618     } else {
 619       assert(_ctx->top_at_mark_start(r) == r->top(),
 620              "Region " SIZE_FORMAT " should already have correct TAMS", r->index());
 621     }
 622   }
 623 
 624   bool is_thread_safe() { return true; }
 625 };
 626 
 627 void ShenandoahConcurrentGC::start_mark() {
 628   _mark.start_mark();
 629 }
 630 
 631 void ShenandoahConcurrentGC::op_init_mark() {
 632   ShenandoahHeap* const heap = ShenandoahHeap::heap();
 633   assert(ShenandoahSafepoint::is_at_shenandoah_safepoint(), "Should be at safepoint");
 634   assert(Thread::current()->is_VM_thread(), "can only do this in VMThread");
 635 
 636   assert(_generation->is_bitmap_clear(), "need clear marking bitmap");
 637   assert(!_generation->is_mark_complete(), "should not be complete");
 638   assert(!heap->has_forwarded_objects(), "No forwarded objects on this path");
 639 
 640 
 641   if (heap->mode()->is_generational()) {
 642     if (_generation->is_young()) {
 643       // The current implementation of swap_remembered_set() copies the write-card-table to the read-card-table.
 644       ShenandoahGCPhase phase(ShenandoahPhaseTimings::init_swap_rset);
 645       _generation->swap_remembered_set();
 646     }
 647 
 648     if (_generation->is_global()) {
 649       heap->old_generation()->cancel_gc();
 650     } else if (heap->is_concurrent_old_mark_in_progress()) {
 651       // Purge the SATB buffers, transferring any valid, old pointers to the
 652       // old generation mark queue. Any pointers in a young region will be
 653       // abandoned.
 654       ShenandoahGCPhase phase(ShenandoahPhaseTimings::init_transfer_satb);
 655       heap->old_generation()->transfer_pointers_from_satb();
 656     }
 657   }
 658 
 659   if (ShenandoahVerify) {
 660     heap->verifier()->verify_before_concmark();
 661   }
 662 
 663   if (VerifyBeforeGC) {
 664     Universe::verify();
 665   }
 666 
 667   _generation->set_concurrent_mark_in_progress(true);
 668 
 669   start_mark();
 670 
 671   if (_do_old_gc_bootstrap) {
 672     shenandoah_assert_generational();
 673     // Update region state for both young and old regions
 674     ShenandoahGCPhase phase(ShenandoahPhaseTimings::init_update_region_states);
 675     ShenandoahInitMarkUpdateRegionStateClosure cl;
 676     heap->parallel_heap_region_iterate(&cl);
 677     heap->old_generation()->ref_processor()->reset_thread_locals();
 678   } else {
 679     // Update region state for only young regions
 680     ShenandoahGCPhase phase(ShenandoahPhaseTimings::init_update_region_states);
 681     ShenandoahInitMarkUpdateRegionStateClosure cl;
 682     _generation->parallel_heap_region_iterate(&cl);
 683   }
 684 
 685   // Weak reference processing
 686   ShenandoahReferenceProcessor* rp = _generation->ref_processor();
 687   rp->reset_thread_locals();
 688   rp->set_soft_reference_policy(heap->soft_ref_policy()->should_clear_all_soft_refs());
 689 
 690   // Make above changes visible to worker threads
 691   OrderAccess::fence();
 692 
 693   // Arm nmethods for concurrent mark
 694   ShenandoahCodeRoots::arm_nmethods_for_mark();
 695 
 696   ShenandoahStackWatermark::change_epoch_id();
 697   if (ShenandoahPacing) {
 698     heap->pacer()->setup_for_mark();
 699   }
 700 }
 701 
 702 void ShenandoahConcurrentGC::op_mark_roots() {
 703   _mark.mark_concurrent_roots();
 704 }
 705 
 706 void ShenandoahConcurrentGC::op_mark() {
 707   _mark.concurrent_mark();
 708 }
 709 
 710 void ShenandoahConcurrentGC::op_final_mark() {
 711   ShenandoahHeap* const heap = ShenandoahHeap::heap();
 712   assert(ShenandoahSafepoint::is_at_shenandoah_safepoint(), "Should be at safepoint");
 713   assert(!heap->has_forwarded_objects(), "No forwarded objects on this path");
 714 
 715   if (ShenandoahVerify) {
 716     heap->verifier()->verify_roots_no_forwarded();
 717   }
 718 
 719   if (!heap->cancelled_gc()) {
 720     _mark.finish_mark();
 721     assert(!heap->cancelled_gc(), "STW mark cannot OOM");
 722 
 723     // Notify JVMTI that the tagmap table will need cleaning.
 724     JvmtiTagMap::set_needs_cleaning();
 725 
 726     // The collection set is chosen by prepare_regions_and_collection_set(). Additionally, certain parameters have been
 727     // established to govern the evacuation efforts that are about to begin.  Refer to comments on reserve members in
 728     // ShenandoahGeneration and ShenandoahOldGeneration for more detail.
 729     _generation->prepare_regions_and_collection_set(true /*concurrent*/);
 730 
 731     // Has to be done after cset selection
 732     heap->prepare_concurrent_roots();
 733 
 734     if (!heap->collection_set()->is_empty()) {
 735       LogTarget(Debug, gc, cset) lt;
 736       if (lt.is_enabled()) {
 737         ResourceMark rm;
 738         LogStream ls(lt);
 739         heap->collection_set()->print_on(&ls);
 740       }
 741 
 742       if (ShenandoahVerify) {
 743         heap->verifier()->verify_before_evacuation();
 744       }
 745 
 746       heap->set_evacuation_in_progress(true);
 747       // From here on, we need to update references.
 748       heap->set_has_forwarded_objects(true);
 749 
 750       // Arm nmethods/stack for concurrent processing
 751       ShenandoahCodeRoots::arm_nmethods_for_evac();
 752       ShenandoahStackWatermark::change_epoch_id();
 753 
 754       if (ShenandoahPacing) {
 755         heap->pacer()->setup_for_evac();
 756       }
 757     } else {
 758       if (ShenandoahVerify) {
 759         if (has_in_place_promotions(heap)) {
 760           heap->verifier()->verify_after_concmark_with_promotions();
 761         } else {
 762           heap->verifier()->verify_after_concmark();
 763         }
 764       }
 765 
 766       if (VerifyAfterGC) {
 767         Universe::verify();
 768       }
 769     }
 770   }
 771 }
 772 
 773 bool ShenandoahConcurrentGC::has_in_place_promotions(ShenandoahHeap* heap) {
 774   return heap->mode()->is_generational() && heap->old_generation()->has_in_place_promotions();
 775 }
 776 
 777 template<bool GENERATIONAL>
 778 class ShenandoahConcurrentEvacThreadClosure : public ThreadClosure {
 779 private:
 780   OopClosure* const _oops;
 781 public:
 782   explicit ShenandoahConcurrentEvacThreadClosure(OopClosure* oops) : _oops(oops) {}
 783 
 784   void do_thread(Thread* thread) override {
 785     JavaThread* const jt = JavaThread::cast(thread);
 786     StackWatermarkSet::finish_processing(jt, _oops, StackWatermarkKind::gc);
 787     if (GENERATIONAL) {
 788       ShenandoahThreadLocalData::enable_plab_promotions(thread);
 789     }
 790   }
 791 };
 792 
 793 template<bool GENERATIONAL>
 794 class ShenandoahConcurrentEvacUpdateThreadTask : public WorkerTask {
 795 private:
 796   ShenandoahJavaThreadsIterator _java_threads;
 797 
 798 public:
 799   explicit ShenandoahConcurrentEvacUpdateThreadTask(uint n_workers) :
 800     WorkerTask("Shenandoah Evacuate/Update Concurrent Thread Roots"),
 801     _java_threads(ShenandoahPhaseTimings::conc_thread_roots, n_workers) {
 802   }
 803 
 804   void work(uint worker_id) override {
 805     if (GENERATIONAL) {
 806       Thread* worker_thread = Thread::current();
 807       ShenandoahThreadLocalData::enable_plab_promotions(worker_thread);
 808     }
 809 
 810     // ShenandoahEvacOOMScope has to be setup by ShenandoahContextEvacuateUpdateRootsClosure.
 811     // Otherwise, may deadlock with watermark lock
 812     ShenandoahContextEvacuateUpdateRootsClosure oops_cl;
 813     ShenandoahConcurrentEvacThreadClosure<GENERATIONAL> thr_cl(&oops_cl);
 814     _java_threads.threads_do(&thr_cl, worker_id);
 815   }
 816 };
 817 
 818 void ShenandoahConcurrentGC::op_thread_roots() {
 819   ShenandoahHeap* const heap = ShenandoahHeap::heap();
 820   assert(heap->is_evacuation_in_progress(), "Checked by caller");
 821   ShenandoahGCWorkerPhase worker_phase(ShenandoahPhaseTimings::conc_thread_roots);
 822   if (heap->mode()->is_generational()) {
 823     ShenandoahConcurrentEvacUpdateThreadTask<true> task(heap->workers()->active_workers());
 824     heap->workers()->run_task(&task);
 825   } else {
 826     ShenandoahConcurrentEvacUpdateThreadTask<false> task(heap->workers()->active_workers());
 827     heap->workers()->run_task(&task);
 828   }
 829 }
 830 
 831 void ShenandoahConcurrentGC::op_weak_refs() {
 832   ShenandoahHeap* const heap = ShenandoahHeap::heap();
 833   assert(heap->is_concurrent_weak_root_in_progress(), "Only during this phase");
 834   // Concurrent weak refs processing
 835   ShenandoahGCWorkerPhase worker_phase(ShenandoahPhaseTimings::conc_weak_refs);
 836   if (heap->gc_cause() == GCCause::_wb_breakpoint) {
 837     ShenandoahBreakpoint::at_after_reference_processing_started();
 838   }
 839   _generation->ref_processor()->process_references(ShenandoahPhaseTimings::conc_weak_refs, heap->workers(), true /* concurrent */);
 840 }
 841 
 842 class ShenandoahEvacUpdateCleanupOopStorageRootsClosure : public BasicOopIterateClosure {
 843 private:
 844   ShenandoahHeap* const _heap;
 845   ShenandoahMarkingContext* const _mark_context;
 846   bool  _evac_in_progress;
 847   Thread* const _thread;
 848 
 849 public:
 850   ShenandoahEvacUpdateCleanupOopStorageRootsClosure();
 851   void do_oop(oop* p);
 852   void do_oop(narrowOop* p);
 853 };
 854 
 855 ShenandoahEvacUpdateCleanupOopStorageRootsClosure::ShenandoahEvacUpdateCleanupOopStorageRootsClosure() :
 856   _heap(ShenandoahHeap::heap()),
 857   _mark_context(ShenandoahHeap::heap()->marking_context()),
 858   _evac_in_progress(ShenandoahHeap::heap()->is_evacuation_in_progress()),
 859   _thread(Thread::current()) {
 860 }
 861 
 862 void ShenandoahEvacUpdateCleanupOopStorageRootsClosure::do_oop(oop* p) {
 863   const oop obj = RawAccess<>::oop_load(p);
 864   if (!CompressedOops::is_null(obj)) {
 865     if (!_mark_context->is_marked(obj)) {
 866       shenandoah_assert_generations_reconciled();
 867       if (_heap->is_in_active_generation(obj)) {
 868         // Here we are asserting that an unmarked from-space object is 'correct'. There seems to be a legitimate
 869         // use-case for accessing from-space objects during concurrent class unloading. In all modes of Shenandoah,
 870         // concurrent class unloading only happens during a global collection.
 871         shenandoah_assert_correct(p, obj);
 872         ShenandoahHeap::atomic_clear_oop(p, obj);
 873       }
 874     } else if (_evac_in_progress && _heap->in_collection_set(obj)) {
 875       oop resolved = ShenandoahBarrierSet::resolve_forwarded_not_null(obj);
 876       if (resolved == obj) {
 877         resolved = _heap->evacuate_object(obj, _thread);
 878       }
 879       shenandoah_assert_not_in_cset_except(p, resolved, _heap->cancelled_gc());
 880       ShenandoahHeap::atomic_update_oop(resolved, p, obj);
 881     }
 882   }
 883 }
 884 
 885 void ShenandoahEvacUpdateCleanupOopStorageRootsClosure::do_oop(narrowOop* p) {
 886   ShouldNotReachHere();
 887 }
 888 
 889 class ShenandoahIsCLDAliveClosure : public CLDClosure {
 890 public:
 891   void do_cld(ClassLoaderData* cld) {
 892     cld->is_alive();
 893   }
 894 };
 895 
 896 class ShenandoahIsNMethodAliveClosure: public NMethodClosure {
 897 public:
 898   void do_nmethod(nmethod* n) {
 899     n->is_unloading();
 900   }
 901 };
 902 
 903 // This task not only evacuates/updates marked weak roots, but also "null"
 904 // dead weak roots.
 905 class ShenandoahConcurrentWeakRootsEvacUpdateTask : public WorkerTask {
 906 private:
 907   ShenandoahVMWeakRoots<true /*concurrent*/> _vm_roots;
 908 
 909   // Roots related to concurrent class unloading
 910   ShenandoahClassLoaderDataRoots<true /* concurrent */>
 911                                              _cld_roots;
 912   ShenandoahConcurrentNMethodIterator        _nmethod_itr;
 913   ShenandoahPhaseTimings::Phase              _phase;
 914 
 915 public:
 916   ShenandoahConcurrentWeakRootsEvacUpdateTask(ShenandoahPhaseTimings::Phase phase) :
 917     WorkerTask("Shenandoah Evacuate/Update Concurrent Weak Roots"),
 918     _vm_roots(phase),
 919     _cld_roots(phase, ShenandoahHeap::heap()->workers()->active_workers(), false /*heap iteration*/),
 920     _nmethod_itr(ShenandoahCodeRoots::table()),
 921     _phase(phase) {}
 922 
 923   ~ShenandoahConcurrentWeakRootsEvacUpdateTask() {
 924     // Notify runtime data structures of potentially dead oops
 925     _vm_roots.report_num_dead();
 926   }
 927 
 928   void work(uint worker_id) {
 929     ShenandoahConcurrentWorkerSession worker_session(worker_id);
 930     ShenandoahSuspendibleThreadSetJoiner sts_join;
 931     {
 932       ShenandoahEvacOOMScope oom;
 933       // jni_roots and weak_roots are OopStorage backed roots, concurrent iteration
 934       // may race against OopStorage::release() calls.
 935       ShenandoahEvacUpdateCleanupOopStorageRootsClosure cl;
 936       _vm_roots.oops_do(&cl, worker_id);
 937     }
 938 
 939     // If we are going to perform concurrent class unloading later on, we need to
 940     // cleanup the weak oops in CLD and determinate nmethod's unloading state, so that we
 941     // can cleanup immediate garbage sooner.
 942     if (ShenandoahHeap::heap()->unload_classes()) {
 943       // Applies ShenandoahIsCLDAlive closure to CLDs, native barrier will either null the
 944       // CLD's holder or evacuate it.
 945       {
 946         ShenandoahIsCLDAliveClosure is_cld_alive;
 947         _cld_roots.cld_do(&is_cld_alive, worker_id);
 948       }
 949 
 950       // Applies ShenandoahIsNMethodAliveClosure to registered nmethods.
 951       // The closure calls nmethod->is_unloading(). The is_unloading
 952       // state is cached, therefore, during concurrent class unloading phase,
 953       // we will not touch the metadata of unloading nmethods
 954       {
 955         ShenandoahWorkerTimingsTracker timer(_phase, ShenandoahPhaseTimings::CodeCacheRoots, worker_id);
 956         ShenandoahIsNMethodAliveClosure is_nmethod_alive;
 957         _nmethod_itr.nmethods_do(&is_nmethod_alive);
 958       }
 959     }
 960   }
 961 };
 962 
 963 void ShenandoahConcurrentGC::op_weak_roots() {
 964   ShenandoahHeap* const heap = ShenandoahHeap::heap();
 965   assert(heap->is_concurrent_weak_root_in_progress(), "Only during this phase");
 966   // Concurrent weak root processing
 967   {
 968     ShenandoahTimingsTracker t(ShenandoahPhaseTimings::conc_weak_roots_work);
 969     ShenandoahGCWorkerPhase worker_phase(ShenandoahPhaseTimings::conc_weak_roots_work);
 970     ShenandoahConcurrentWeakRootsEvacUpdateTask task(ShenandoahPhaseTimings::conc_weak_roots_work);
 971     heap->workers()->run_task(&task);
 972   }
 973 
 974   // Perform handshake to flush out dead oops
 975   {
 976     ShenandoahTimingsTracker t(ShenandoahPhaseTimings::conc_weak_roots_rendezvous);
 977     heap->rendezvous_threads();
 978   }
 979   // We can only toggle concurrent_weak_root_in_progress flag
 980   // at a safepoint, so that mutators see a consistent
 981   // value. The flag will be cleared at the next safepoint.
 982 }
 983 
 984 void ShenandoahConcurrentGC::op_class_unloading() {
 985   ShenandoahHeap* const heap = ShenandoahHeap::heap();
 986   assert (heap->is_concurrent_weak_root_in_progress() &&
 987           heap->unload_classes(),
 988           "Checked by caller");
 989   heap->do_class_unloading();
 990 }
 991 
 992 class ShenandoahEvacUpdateCodeCacheClosure : public NMethodClosure {
 993 private:
 994   BarrierSetNMethod* const                  _bs;
 995   ShenandoahEvacuateUpdateMetadataClosure   _cl;
 996 
 997 public:
 998   ShenandoahEvacUpdateCodeCacheClosure() :
 999     _bs(BarrierSet::barrier_set()->barrier_set_nmethod()),
1000     _cl() {
1001   }
1002 
1003   void do_nmethod(nmethod* n) {
1004     ShenandoahNMethod* data = ShenandoahNMethod::gc_data(n);
1005     ShenandoahReentrantLocker locker(data->lock());
1006     // Setup EvacOOM scope below reentrant lock to avoid deadlock with
1007     // nmethod_entry_barrier
1008     ShenandoahEvacOOMScope oom;
1009     data->oops_do(&_cl, true/*fix relocation*/);
1010     _bs->disarm(n);
1011   }
1012 };
1013 
1014 class ShenandoahConcurrentRootsEvacUpdateTask : public WorkerTask {
1015 private:
1016   ShenandoahPhaseTimings::Phase                 _phase;
1017   ShenandoahVMRoots<true /*concurrent*/>        _vm_roots;
1018   ShenandoahClassLoaderDataRoots<true /*concurrent*/>
1019                                                 _cld_roots;
1020   ShenandoahConcurrentNMethodIterator           _nmethod_itr;
1021 
1022 public:
1023   ShenandoahConcurrentRootsEvacUpdateTask(ShenandoahPhaseTimings::Phase phase) :
1024     WorkerTask("Shenandoah Evacuate/Update Concurrent Strong Roots"),
1025     _phase(phase),
1026     _vm_roots(phase),
1027     _cld_roots(phase, ShenandoahHeap::heap()->workers()->active_workers(), false /*heap iteration*/),
1028     _nmethod_itr(ShenandoahCodeRoots::table()) {}
1029 
1030   void work(uint worker_id) {
1031     ShenandoahConcurrentWorkerSession worker_session(worker_id);
1032     {
1033       ShenandoahEvacOOMScope oom;
1034       {
1035         // vm_roots and weak_roots are OopStorage backed roots, concurrent iteration
1036         // may race against OopStorage::release() calls.
1037         ShenandoahContextEvacuateUpdateRootsClosure cl;
1038         _vm_roots.oops_do<ShenandoahContextEvacuateUpdateRootsClosure>(&cl, worker_id);
1039       }
1040 
1041       {
1042         ShenandoahEvacuateUpdateMetadataClosure cl;
1043         CLDToOopClosure clds(&cl, ClassLoaderData::_claim_strong);
1044         _cld_roots.cld_do(&clds, worker_id);
1045       }
1046     }
1047 
1048     // Cannot setup ShenandoahEvacOOMScope here, due to potential deadlock with nmethod_entry_barrier.
1049     if (!ShenandoahHeap::heap()->unload_classes()) {
1050       ShenandoahWorkerTimingsTracker timer(_phase, ShenandoahPhaseTimings::CodeCacheRoots, worker_id);
1051       ShenandoahEvacUpdateCodeCacheClosure cl;
1052       _nmethod_itr.nmethods_do(&cl);
1053     }
1054   }
1055 };
1056 
1057 void ShenandoahConcurrentGC::op_strong_roots() {
1058   ShenandoahHeap* const heap = ShenandoahHeap::heap();
1059   assert(heap->is_concurrent_strong_root_in_progress(), "Checked by caller");
1060   ShenandoahConcurrentRootsEvacUpdateTask task(ShenandoahPhaseTimings::conc_strong_roots);
1061   heap->workers()->run_task(&task);
1062   heap->set_concurrent_strong_root_in_progress(false);
1063 }
1064 
1065 void ShenandoahConcurrentGC::op_cleanup_early() {
1066   ShenandoahHeap::heap()->free_set()->recycle_trash();
1067 }
1068 
1069 void ShenandoahConcurrentGC::op_evacuate() {
1070   ShenandoahHeap::heap()->evacuate_collection_set(true /*concurrent*/);
1071 }
1072 
1073 void ShenandoahConcurrentGC::op_init_updaterefs() {
1074   ShenandoahHeap* const heap = ShenandoahHeap::heap();
1075   heap->set_evacuation_in_progress(false);
1076   heap->set_concurrent_weak_root_in_progress(false);
1077   heap->prepare_update_heap_references(true /*concurrent*/);
1078   heap->set_update_refs_in_progress(true);
1079   if (ShenandoahVerify) {
1080     heap->verifier()->verify_before_updaterefs();
1081   }
1082   if (ShenandoahPacing) {
1083     heap->pacer()->setup_for_updaterefs();
1084   }
1085 }
1086 
1087 void ShenandoahConcurrentGC::op_updaterefs() {
1088   ShenandoahHeap::heap()->update_heap_references(true /*concurrent*/);
1089 }
1090 
1091 class ShenandoahUpdateThreadClosure : public HandshakeClosure {
1092 private:
1093   ShenandoahUpdateRefsClosure _cl;
1094 public:
1095   ShenandoahUpdateThreadClosure();
1096   void do_thread(Thread* thread);
1097 };
1098 
1099 ShenandoahUpdateThreadClosure::ShenandoahUpdateThreadClosure() :
1100   HandshakeClosure("Shenandoah Update Thread Roots") {
1101 }
1102 
1103 void ShenandoahUpdateThreadClosure::do_thread(Thread* thread) {
1104   if (thread->is_Java_thread()) {
1105     JavaThread* jt = JavaThread::cast(thread);
1106     ResourceMark rm;
1107     jt->oops_do(&_cl, nullptr);
1108   }
1109 }
1110 
1111 void ShenandoahConcurrentGC::op_update_thread_roots() {
1112   ShenandoahUpdateThreadClosure cl;
1113   Handshake::execute(&cl);
1114 }
1115 
1116 void ShenandoahConcurrentGC::op_final_updaterefs() {
1117   ShenandoahHeap* const heap = ShenandoahHeap::heap();
1118   assert(ShenandoahSafepoint::is_at_shenandoah_safepoint(), "must be at safepoint");
1119   assert(!heap->_update_refs_iterator.has_next(), "Should have finished update references");
1120 
1121   heap->finish_concurrent_roots();
1122 
1123   // Clear cancelled GC, if set. On cancellation path, the block before would handle
1124   // everything.
1125   if (heap->cancelled_gc()) {
1126     heap->clear_cancelled_gc(true /* clear oom handler */);
1127   }
1128 
1129   // Has to be done before cset is clear
1130   if (ShenandoahVerify) {
1131     heap->verifier()->verify_roots_in_to_space();
1132   }
1133 
1134   // If we are running in generational mode and this is an aging cycle, this will also age active
1135   // regions that haven't been used for allocation.
1136   heap->update_heap_region_states(true /*concurrent*/);
1137 
1138   heap->set_update_refs_in_progress(false);
1139   heap->set_has_forwarded_objects(false);
1140 
1141   if (heap->mode()->is_generational() && heap->is_concurrent_old_mark_in_progress()) {
1142     // When the SATB barrier is left on to support concurrent old gen mark, it may pick up writes to
1143     // objects in the collection set. After those objects are evacuated, the pointers in the
1144     // SATB are no longer safe. Once we have finished update references, we are guaranteed that
1145     // no more writes to the collection set are possible.
1146     //
1147     // This will transfer any old pointers in _active_ regions from the SATB to the old gen
1148     // mark queues. All other pointers will be discarded. This would also discard any pointers
1149     // in old regions that were included in a mixed evacuation. We aren't using the SATB filter
1150     // methods here because we cannot control when they execute. If the SATB filter runs _after_
1151     // a region has been recycled, we will not be able to detect the bad pointer.
1152     //
1153     // We are not concerned about skipping this step in abbreviated cycles because regions
1154     // with no live objects cannot have been written to and so cannot have entries in the SATB
1155     // buffers.
1156     heap->old_generation()->transfer_pointers_from_satb();
1157 
1158     // Aging_cycle is only relevant during evacuation cycle for individual objects and during final mark for
1159     // entire regions.  Both of these relevant operations occur before final update refs.
1160     ShenandoahGenerationalHeap::heap()->set_aging_cycle(false);
1161   }
1162 
1163   if (ShenandoahVerify) {
1164     heap->verifier()->verify_after_updaterefs();
1165   }
1166 
1167   if (VerifyAfterGC) {
1168     Universe::verify();
1169   }
1170 
1171   heap->rebuild_free_set(true /*concurrent*/);
1172 }
1173 
1174 void ShenandoahConcurrentGC::op_final_roots() {
1175 
1176   ShenandoahHeap *heap = ShenandoahHeap::heap();
1177   heap->set_concurrent_weak_root_in_progress(false);
1178   heap->set_evacuation_in_progress(false);
1179 
1180   if (heap->mode()->is_generational()) {
1181     // If the cycle was shortened for having enough immediate garbage, this could be
1182     // the last GC safepoint before concurrent marking of old resumes. We must be sure
1183     // that old mark threads don't see any pointers to garbage in the SATB buffers.
1184     if (heap->is_concurrent_old_mark_in_progress()) {
1185       heap->old_generation()->transfer_pointers_from_satb();
1186     }
1187 
1188     if (!_generation->is_old()) {
1189       ShenandoahGenerationalHeap::heap()->update_region_ages(_generation->complete_marking_context());
1190     }
1191   }
1192 }
1193 
1194 void ShenandoahConcurrentGC::op_cleanup_complete() {
1195   ShenandoahHeap::heap()->free_set()->recycle_trash();
1196 }
1197 
1198 bool ShenandoahConcurrentGC::check_cancellation_and_abort(ShenandoahDegenPoint point) {
1199   if (ShenandoahHeap::heap()->cancelled_gc()) {
1200     _degen_point = point;
1201     return true;
1202   }
1203   return false;
1204 }
1205 
1206 const char* ShenandoahConcurrentGC::init_mark_event_message() const {
1207   ShenandoahHeap* const heap = ShenandoahHeap::heap();
1208   assert(!heap->has_forwarded_objects(), "Should not have forwarded objects here");
1209   if (heap->unload_classes()) {
1210     SHENANDOAH_RETURN_EVENT_MESSAGE(_generation->type(), "Pause Init Mark", " (unload classes)");
1211   } else {
1212     SHENANDOAH_RETURN_EVENT_MESSAGE(_generation->type(), "Pause Init Mark", "");
1213   }
1214 }
1215 
1216 const char* ShenandoahConcurrentGC::final_mark_event_message() const {
1217   ShenandoahHeap* const heap = ShenandoahHeap::heap();
1218   assert(!heap->has_forwarded_objects() || heap->is_concurrent_old_mark_in_progress(),
1219          "Should not have forwarded objects during final mark, unless old gen concurrent mark is running");
1220 
1221   if (heap->unload_classes()) {
1222     SHENANDOAH_RETURN_EVENT_MESSAGE(_generation->type(), "Pause Final Mark", " (unload classes)");
1223   } else {
1224     SHENANDOAH_RETURN_EVENT_MESSAGE(_generation->type(), "Pause Final Mark", "");
1225   }
1226 }
1227 
1228 const char* ShenandoahConcurrentGC::conc_mark_event_message() const {
1229   ShenandoahHeap* const heap = ShenandoahHeap::heap();
1230   assert(!heap->has_forwarded_objects() || heap->is_concurrent_old_mark_in_progress(),
1231          "Should not have forwarded objects concurrent mark, unless old gen concurrent mark is running");
1232   if (heap->unload_classes()) {
1233     SHENANDOAH_RETURN_EVENT_MESSAGE(_generation->type(), "Concurrent marking", " (unload classes)");
1234   } else {
1235     SHENANDOAH_RETURN_EVENT_MESSAGE(_generation->type(), "Concurrent marking", "");
1236   }
1237 }
1238 
1239 const char* ShenandoahConcurrentGC::conc_reset_event_message() const {
1240   if (ShenandoahHeap::heap()->unload_classes()) {
1241     SHENANDOAH_RETURN_EVENT_MESSAGE(_generation->type(), "Concurrent reset", " (unload classes)");
1242   } else {
1243     SHENANDOAH_RETURN_EVENT_MESSAGE(_generation->type(), "Concurrent reset", "");
1244   }
1245 }
1246 
1247 const char* ShenandoahConcurrentGC::final_roots_event_message() const {
1248   if (ShenandoahHeap::heap()->unload_classes()) {
1249     SHENANDOAH_RETURN_EVENT_MESSAGE(_generation->type(), "Pause Final Roots", " (unload classes)");
1250   } else {
1251     SHENANDOAH_RETURN_EVENT_MESSAGE(_generation->type(), "Pause Final Roots", "");
1252   }
1253 }
1254 
1255 const char* ShenandoahConcurrentGC::conc_weak_refs_event_message() const {
1256   if (ShenandoahHeap::heap()->unload_classes()) {
1257     SHENANDOAH_RETURN_EVENT_MESSAGE(_generation->type(), "Concurrent weak references", " (unload classes)");
1258   } else {
1259     SHENANDOAH_RETURN_EVENT_MESSAGE(_generation->type(), "Concurrent weak references", "");
1260   }
1261 }
1262 
1263 const char* ShenandoahConcurrentGC::conc_weak_roots_event_message() const {
1264   if (ShenandoahHeap::heap()->unload_classes()) {
1265     SHENANDOAH_RETURN_EVENT_MESSAGE(_generation->type(), "Concurrent weak roots", " (unload classes)");
1266   } else {
1267     SHENANDOAH_RETURN_EVENT_MESSAGE(_generation->type(), "Concurrent weak roots", "");
1268   }
1269 }
1270 
1271 const char* ShenandoahConcurrentGC::conc_cleanup_event_message() const {
1272   if (ShenandoahHeap::heap()->unload_classes()) {
1273     SHENANDOAH_RETURN_EVENT_MESSAGE(_generation->type(), "Concurrent cleanup", " (unload classes)");
1274   } else {
1275     SHENANDOAH_RETURN_EVENT_MESSAGE(_generation->type(), "Concurrent cleanup", "");
1276   }
1277 }