1 /*
   2  * Copyright (c) 2021, 2022, Red Hat, Inc. All rights reserved.
   3  * Copyright Amazon.com Inc. or its affiliates. All Rights Reserved.
   4  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
   5  *
   6  * This code is free software; you can redistribute it and/or modify it
   7  * under the terms of the GNU General Public License version 2 only, as
   8  * published by the Free Software Foundation.
   9  *
  10  * This code is distributed in the hope that it will be useful, but WITHOUT
  11  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  12  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  13  * version 2 for more details (a copy is included in the LICENSE file that
  14  * accompanied this code).
  15  *
  16  * You should have received a copy of the GNU General Public License version
  17  * 2 along with this work; if not, write to the Free Software Foundation,
  18  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
  19  *
  20  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
  21  * or visit www.oracle.com if you need additional information or have any
  22  * questions.
  23  *
  24  */
  25 
  26 #include "precompiled.hpp"
  27 
  28 #include "gc/shared/barrierSetNMethod.hpp"
  29 #include "gc/shared/collectorCounters.hpp"
  30 #include "gc/shared/continuationGCSupport.inline.hpp"
  31 #include "gc/shenandoah/shenandoahBreakpoint.hpp"
  32 #include "gc/shenandoah/shenandoahCollectorPolicy.hpp"
  33 #include "gc/shenandoah/shenandoahConcurrentGC.hpp"
  34 #include "gc/shenandoah/shenandoahFreeSet.hpp"
  35 #include "gc/shenandoah/shenandoahGeneration.hpp"
  36 #include "gc/shenandoah/shenandoahGenerationalHeap.hpp"
  37 #include "gc/shenandoah/shenandoahOldGeneration.hpp"
  38 #include "gc/shenandoah/shenandoahYoungGeneration.hpp"
  39 #include "gc/shenandoah/shenandoahLock.hpp"
  40 #include "gc/shenandoah/shenandoahMark.inline.hpp"
  41 #include "gc/shenandoah/shenandoahMonitoringSupport.hpp"
  42 #include "gc/shenandoah/shenandoahOopClosures.inline.hpp"
  43 #include "gc/shenandoah/shenandoahPhaseTimings.hpp"
  44 #include "gc/shenandoah/shenandoahReferenceProcessor.hpp"
  45 #include "gc/shenandoah/shenandoahRootProcessor.inline.hpp"
  46 #include "gc/shenandoah/shenandoahStackWatermark.hpp"
  47 #include "gc/shenandoah/shenandoahUtils.hpp"
  48 #include "gc/shenandoah/shenandoahVerifier.hpp"
  49 #include "gc/shenandoah/shenandoahVMOperations.hpp"
  50 #include "gc/shenandoah/shenandoahWorkGroup.hpp"
  51 #include "gc/shenandoah/shenandoahWorkerPolicy.hpp"
  52 #include "memory/allocation.hpp"
  53 #include "prims/jvmtiTagMap.hpp"
  54 #include "runtime/vmThread.hpp"
  55 #include "utilities/events.hpp"
  56 
  57 // Breakpoint support
  58 class ShenandoahBreakpointGCScope : public StackObj {
  59 private:
  60   const GCCause::Cause _cause;
  61 public:
  62   ShenandoahBreakpointGCScope(GCCause::Cause cause) : _cause(cause) {
  63     if (cause == GCCause::_wb_breakpoint) {
  64       ShenandoahBreakpoint::start_gc();
  65       ShenandoahBreakpoint::at_before_gc();
  66     }
  67   }
  68 
  69   ~ShenandoahBreakpointGCScope() {
  70     if (_cause == GCCause::_wb_breakpoint) {
  71       ShenandoahBreakpoint::at_after_gc();
  72     }
  73   }
  74 };
  75 
  76 class ShenandoahBreakpointMarkScope : public StackObj {
  77 private:
  78   const GCCause::Cause _cause;
  79 public:
  80   ShenandoahBreakpointMarkScope(GCCause::Cause cause) : _cause(cause) {
  81     if (_cause == GCCause::_wb_breakpoint) {
  82       ShenandoahBreakpoint::at_after_marking_started();
  83     }
  84   }
  85 
  86   ~ShenandoahBreakpointMarkScope() {
  87     if (_cause == GCCause::_wb_breakpoint) {
  88       ShenandoahBreakpoint::at_before_marking_completed();
  89     }
  90   }
  91 };
  92 
  93 ShenandoahConcurrentGC::ShenandoahConcurrentGC(ShenandoahGeneration* generation, bool do_old_gc_bootstrap) :
  94   _mark(generation),
  95   _degen_point(ShenandoahDegenPoint::_degenerated_unset),
  96   _abbreviated(false),
  97   _do_old_gc_bootstrap(do_old_gc_bootstrap),
  98   _generation(generation) {
  99 }
 100 
 101 ShenandoahGC::ShenandoahDegenPoint ShenandoahConcurrentGC::degen_point() const {
 102   return _degen_point;
 103 }
 104 
 105 bool ShenandoahConcurrentGC::collect(GCCause::Cause cause) {
 106   ShenandoahHeap* const heap = ShenandoahHeap::heap();
 107 
 108   ShenandoahBreakpointGCScope breakpoint_gc_scope(cause);
 109 
 110   // Reset for upcoming marking
 111   entry_reset();
 112 
 113   // Start initial mark under STW
 114   vmop_entry_init_mark();
 115 
 116   {
 117     ShenandoahBreakpointMarkScope breakpoint_mark_scope(cause);
 118 
 119     // Reset task queue stats here, rather than in mark_concurrent_roots,
 120     // because remembered set scan will `push` oops into the queues and
 121     // resetting after this happens will lose those counts.
 122     TASKQUEUE_STATS_ONLY(_mark.task_queues()->reset_taskqueue_stats());
 123 
 124     // Concurrent remembered set scanning
 125     entry_scan_remembered_set();
 126     // TODO: When RS scanning yields, we will need a check_cancellation_and_abort() degeneration point here.
 127 
 128     // Concurrent mark roots
 129     entry_mark_roots();
 130     if (check_cancellation_and_abort(ShenandoahDegenPoint::_degenerated_roots)) {
 131       return false;
 132     }
 133 
 134     // Continue concurrent mark
 135     entry_mark();
 136     if (check_cancellation_and_abort(ShenandoahDegenPoint::_degenerated_mark)) {
 137       return false;
 138     }
 139   }
 140 
 141   // Complete marking under STW, and start evacuation
 142   vmop_entry_final_mark();
 143 
 144   // If the GC was cancelled just before final mark (but after the preceding cancellation check),
 145   // then the safepoint operation will do nothing and the concurrent mark will still be in progress.
 146   // In this case it is safe (and necessary) to resume the degenerated cycle from the marking phase.
 147   //
 148   // On the other hand, if the GC is cancelled after final mark (but before this check), then the
 149   // final mark safepoint operation will have finished the mark (setting concurrent mark in progress
 150   // to false). In this case (final mark has completed), we need control to fall past the next
 151   // cancellation check and resume the degenerated cycle from the evacuation phase.
 152   if (_generation->is_concurrent_mark_in_progress()) {
 153     // If the concurrent mark is still in progress after the final mark safepoint, then the GC has
 154     // been cancelled. The degenerated cycle must resume from the marking phase. Without this check,
 155     // the non-generational mode may fall all the way to the end of this collect routine without
 156     // having done anything (besides mark most of the heap). Without having collected anything, we
 157     // can expect an 'out of cycle' degenerated GC which will again mark the entire heap. This is
 158     // not optimal.
 159     // For the generational mode, we cannot allow this. The generational mode relies on marking
 160     // (including the final mark) to rebuild portions of the card table. If the generational mode does
 161     // not complete marking after it has swapped the card tables, the root set on subsequent GCs will
 162     // be incomplete, heap corruption may follow.
 163     bool cancelled = check_cancellation_and_abort(ShenandoahDegenPoint::_degenerated_mark);
 164     assert(cancelled, "GC must have been cancelled between concurrent and final mark");
 165     return false;
 166   }
 167 
 168   // Concurrent stack processing
 169   if (heap->is_evacuation_in_progress()) {
 170     entry_thread_roots();
 171   }
 172 
 173   // Process weak roots that might still point to regions that would be broken by cleanup
 174   if (heap->is_concurrent_weak_root_in_progress()) {
 175     entry_weak_refs();
 176     entry_weak_roots();
 177   }
 178 
 179   // Final mark might have reclaimed some immediate garbage, kick cleanup to reclaim
 180   // the space. This would be the last action if there is nothing to evacuate.  Note that
 181   // we will not age young-gen objects in the case that we skip evacuation.
 182   entry_cleanup_early();
 183 
 184   {
 185     // TODO: Not sure there is value in logging free-set status right here.  Note that whenever the free set is rebuilt,
 186     // it logs the newly rebuilt status.
 187     ShenandoahHeapLocker locker(heap->lock());
 188     heap->free_set()->log_status();
 189   }
 190 
 191   // Perform concurrent class unloading
 192   if (heap->unload_classes() &&
 193       heap->is_concurrent_weak_root_in_progress()) {
 194     entry_class_unloading();
 195   }
 196 
 197   // Processing strong roots
 198   // This may be skipped if there is nothing to update/evacuate.
 199   // If so, strong_root_in_progress would be unset.
 200   if (heap->is_concurrent_strong_root_in_progress()) {
 201     entry_strong_roots();
 202   }
 203 
 204   // Continue the cycle with evacuation and optional update-refs.
 205   // This may be skipped if there is nothing to evacuate.
 206   // If so, evac_in_progress would be unset by collection set preparation code.
 207   if (heap->is_evacuation_in_progress()) {
 208     // Concurrently evacuate
 209     entry_evacuate();
 210     if (check_cancellation_and_abort(ShenandoahDegenPoint::_degenerated_evac)) {
 211       return false;
 212     }
 213   }
 214 
 215   if (heap->has_forwarded_objects()) {
 216     // Perform update-refs phase.
 217     vmop_entry_init_updaterefs();
 218     entry_updaterefs();
 219     if (check_cancellation_and_abort(ShenandoahDegenPoint::_degenerated_updaterefs)) {
 220       return false;
 221     }
 222 
 223     // Concurrent update thread roots
 224     entry_update_thread_roots();
 225     if (check_cancellation_and_abort(ShenandoahDegenPoint::_degenerated_updaterefs)) {
 226       return false;
 227     }
 228 
 229     vmop_entry_final_updaterefs();
 230 
 231     // Update references freed up collection set, kick the cleanup to reclaim the space.
 232     entry_cleanup_complete();
 233   } else {
 234     // We chose not to evacuate because we found sufficient immediate garbage. Note that we
 235     // do not check for cancellation here because, at this point, the cycle is effectively
 236     // complete. If the cycle has been cancelled here, the control thread will detect it
 237     // on its next iteration and run a degenerated young cycle.
 238     vmop_entry_final_roots();
 239     _abbreviated = true;
 240   }
 241 
 242   // We defer generation resizing actions until after cset regions have been recycled.  We do this even following an
 243   // abbreviated cycle.
 244   if (heap->mode()->is_generational()) {
 245     ShenandoahGenerationalHeap::heap()->complete_concurrent_cycle();
 246   }
 247   return true;
 248 }
 249 
 250 void ShenandoahConcurrentGC::vmop_entry_init_mark() {
 251   ShenandoahHeap* const heap = ShenandoahHeap::heap();
 252   TraceCollectorStats tcs(heap->monitoring_support()->stw_collection_counters());
 253   ShenandoahTimingsTracker timing(ShenandoahPhaseTimings::init_mark_gross);
 254 
 255   heap->try_inject_alloc_failure();
 256   VM_ShenandoahInitMark op(this);
 257   VMThread::execute(&op); // jump to entry_init_mark() under safepoint
 258 }
 259 
 260 void ShenandoahConcurrentGC::vmop_entry_final_mark() {
 261   ShenandoahHeap* const heap = ShenandoahHeap::heap();
 262   TraceCollectorStats tcs(heap->monitoring_support()->stw_collection_counters());
 263   ShenandoahTimingsTracker timing(ShenandoahPhaseTimings::final_mark_gross);
 264 
 265   heap->try_inject_alloc_failure();
 266   VM_ShenandoahFinalMarkStartEvac op(this);
 267   VMThread::execute(&op); // jump to entry_final_mark under safepoint
 268 }
 269 
 270 void ShenandoahConcurrentGC::vmop_entry_init_updaterefs() {
 271   ShenandoahHeap* const heap = ShenandoahHeap::heap();
 272   TraceCollectorStats tcs(heap->monitoring_support()->stw_collection_counters());
 273   ShenandoahTimingsTracker timing(ShenandoahPhaseTimings::init_update_refs_gross);
 274 
 275   heap->try_inject_alloc_failure();
 276   VM_ShenandoahInitUpdateRefs op(this);
 277   VMThread::execute(&op);
 278 }
 279 
 280 void ShenandoahConcurrentGC::vmop_entry_final_updaterefs() {
 281   ShenandoahHeap* const heap = ShenandoahHeap::heap();
 282   TraceCollectorStats tcs(heap->monitoring_support()->stw_collection_counters());
 283   ShenandoahTimingsTracker timing(ShenandoahPhaseTimings::final_update_refs_gross);
 284 
 285   heap->try_inject_alloc_failure();
 286   VM_ShenandoahFinalUpdateRefs op(this);
 287   VMThread::execute(&op);
 288 }
 289 
 290 void ShenandoahConcurrentGC::vmop_entry_final_roots() {
 291   ShenandoahHeap* const heap = ShenandoahHeap::heap();
 292   TraceCollectorStats tcs(heap->monitoring_support()->stw_collection_counters());
 293   ShenandoahTimingsTracker timing(ShenandoahPhaseTimings::final_roots_gross);
 294 
 295   // This phase does not use workers, no need for setup
 296   heap->try_inject_alloc_failure();
 297   VM_ShenandoahFinalRoots op(this);
 298   VMThread::execute(&op);
 299 }
 300 
 301 void ShenandoahConcurrentGC::entry_init_mark() {
 302   const char* msg = init_mark_event_message();
 303   ShenandoahPausePhase gc_phase(msg, ShenandoahPhaseTimings::init_mark);
 304   EventMark em("%s", msg);
 305 
 306   ShenandoahWorkerScope scope(ShenandoahHeap::heap()->workers(),
 307                               ShenandoahWorkerPolicy::calc_workers_for_init_marking(),
 308                               "init marking");
 309 
 310   op_init_mark();
 311 }
 312 
 313 void ShenandoahConcurrentGC::entry_final_mark() {
 314   const char* msg = final_mark_event_message();
 315   ShenandoahPausePhase gc_phase(msg, ShenandoahPhaseTimings::final_mark);
 316   EventMark em("%s", msg);
 317 
 318   ShenandoahWorkerScope scope(ShenandoahHeap::heap()->workers(),
 319                               ShenandoahWorkerPolicy::calc_workers_for_final_marking(),
 320                               "final marking");
 321 
 322   op_final_mark();
 323 }
 324 
 325 void ShenandoahConcurrentGC::entry_init_updaterefs() {
 326   static const char* msg = "Pause Init Update Refs";
 327   ShenandoahPausePhase gc_phase(msg, ShenandoahPhaseTimings::init_update_refs);
 328   EventMark em("%s", msg);
 329 
 330   // No workers used in this phase, no setup required
 331   op_init_updaterefs();
 332 }
 333 
 334 void ShenandoahConcurrentGC::entry_final_updaterefs() {
 335   static const char* msg = "Pause Final Update Refs";
 336   ShenandoahPausePhase gc_phase(msg, ShenandoahPhaseTimings::final_update_refs);
 337   EventMark em("%s", msg);
 338 
 339   ShenandoahWorkerScope scope(ShenandoahHeap::heap()->workers(),
 340                               ShenandoahWorkerPolicy::calc_workers_for_final_update_ref(),
 341                               "final reference update");
 342 
 343   op_final_updaterefs();
 344 }
 345 
 346 void ShenandoahConcurrentGC::entry_final_roots() {
 347   static const char* msg = "Pause Final Roots";
 348   ShenandoahPausePhase gc_phase(msg, ShenandoahPhaseTimings::final_roots);
 349   EventMark em("%s", msg);
 350 
 351   op_final_roots();
 352 }
 353 
 354 void ShenandoahConcurrentGC::entry_reset() {
 355   ShenandoahHeap* const heap = ShenandoahHeap::heap();
 356   heap->try_inject_alloc_failure();
 357 
 358   TraceCollectorStats tcs(heap->monitoring_support()->concurrent_collection_counters());
 359   {
 360     static const char* msg = "Concurrent reset";
 361     ShenandoahConcurrentPhase gc_phase(msg, ShenandoahPhaseTimings::conc_reset);
 362     EventMark em("%s", msg);
 363 
 364     ShenandoahWorkerScope scope(heap->workers(),
 365                                 ShenandoahWorkerPolicy::calc_workers_for_conc_reset(),
 366                                 msg);
 367     op_reset();
 368   }
 369 
 370   if (_do_old_gc_bootstrap) {
 371     static const char* msg = "Concurrent reset (OLD)";
 372     ShenandoahConcurrentPhase gc_phase(msg, ShenandoahPhaseTimings::conc_reset_old);
 373     ShenandoahWorkerScope scope(ShenandoahHeap::heap()->workers(),
 374                                 ShenandoahWorkerPolicy::calc_workers_for_conc_reset(),
 375                                 msg);
 376     EventMark em("%s", msg);
 377 
 378     heap->old_generation()->prepare_gc();
 379   }
 380 }
 381 
 382 void ShenandoahConcurrentGC::entry_scan_remembered_set() {
 383   if (_generation->is_young()) {
 384     ShenandoahHeap* const heap = ShenandoahHeap::heap();
 385     TraceCollectorStats tcs(heap->monitoring_support()->concurrent_collection_counters());
 386     const char* msg = "Concurrent remembered set scanning";
 387     ShenandoahConcurrentPhase gc_phase(msg, ShenandoahPhaseTimings::init_scan_rset);
 388     EventMark em("%s", msg);
 389 
 390     ShenandoahWorkerScope scope(heap->workers(),
 391                                 ShenandoahWorkerPolicy::calc_workers_for_rs_scanning(),
 392                                 msg);
 393 
 394     heap->try_inject_alloc_failure();
 395     _generation->scan_remembered_set(true /* is_concurrent */);
 396   }
 397 }
 398 
 399 void ShenandoahConcurrentGC::entry_mark_roots() {
 400   ShenandoahHeap* const heap = ShenandoahHeap::heap();
 401   TraceCollectorStats tcs(heap->monitoring_support()->concurrent_collection_counters());
 402   const char* msg = "Concurrent marking roots";
 403   ShenandoahConcurrentPhase gc_phase(msg, ShenandoahPhaseTimings::conc_mark_roots);
 404   EventMark em("%s", msg);
 405 
 406   ShenandoahWorkerScope scope(heap->workers(),
 407                               ShenandoahWorkerPolicy::calc_workers_for_conc_marking(),
 408                               "concurrent marking roots");
 409 
 410   heap->try_inject_alloc_failure();
 411   op_mark_roots();
 412 }
 413 
 414 void ShenandoahConcurrentGC::entry_mark() {
 415   ShenandoahHeap* const heap = ShenandoahHeap::heap();
 416   TraceCollectorStats tcs(heap->monitoring_support()->concurrent_collection_counters());
 417   const char* msg = conc_mark_event_message();
 418   ShenandoahConcurrentPhase gc_phase(msg, ShenandoahPhaseTimings::conc_mark);
 419   EventMark em("%s", msg);
 420 
 421   ShenandoahWorkerScope scope(heap->workers(),
 422                               ShenandoahWorkerPolicy::calc_workers_for_conc_marking(),
 423                               "concurrent marking");
 424 
 425   heap->try_inject_alloc_failure();
 426   op_mark();
 427 }
 428 
 429 void ShenandoahConcurrentGC::entry_thread_roots() {
 430   ShenandoahHeap* const heap = ShenandoahHeap::heap();
 431   static const char* msg = "Concurrent thread roots";
 432   ShenandoahConcurrentPhase gc_phase(msg, ShenandoahPhaseTimings::conc_thread_roots);
 433   EventMark em("%s", msg);
 434 
 435   ShenandoahWorkerScope scope(heap->workers(),
 436                               ShenandoahWorkerPolicy::calc_workers_for_conc_root_processing(),
 437                               msg);
 438 
 439   heap->try_inject_alloc_failure();
 440   op_thread_roots();
 441 }
 442 
 443 void ShenandoahConcurrentGC::entry_weak_refs() {
 444   ShenandoahHeap* const heap = ShenandoahHeap::heap();
 445   static const char* msg = "Concurrent weak references";
 446   ShenandoahConcurrentPhase gc_phase(msg, ShenandoahPhaseTimings::conc_weak_refs);
 447   EventMark em("%s", msg);
 448 
 449   ShenandoahWorkerScope scope(heap->workers(),
 450                               ShenandoahWorkerPolicy::calc_workers_for_conc_refs_processing(),
 451                               "concurrent weak references");
 452 
 453   heap->try_inject_alloc_failure();
 454   op_weak_refs();
 455 }
 456 
 457 void ShenandoahConcurrentGC::entry_weak_roots() {
 458   ShenandoahHeap* const heap = ShenandoahHeap::heap();
 459   TraceCollectorStats tcs(heap->monitoring_support()->concurrent_collection_counters());
 460   static const char* msg = "Concurrent weak roots";
 461   ShenandoahConcurrentPhase gc_phase(msg, ShenandoahPhaseTimings::conc_weak_roots);
 462   EventMark em("%s", msg);
 463 
 464   ShenandoahWorkerScope scope(heap->workers(),
 465                               ShenandoahWorkerPolicy::calc_workers_for_conc_root_processing(),
 466                               "concurrent weak root");
 467 
 468   heap->try_inject_alloc_failure();
 469   op_weak_roots();
 470 }
 471 
 472 void ShenandoahConcurrentGC::entry_class_unloading() {
 473   ShenandoahHeap* const heap = ShenandoahHeap::heap();
 474   TraceCollectorStats tcs(heap->monitoring_support()->concurrent_collection_counters());
 475   static const char* msg = "Concurrent class unloading";
 476   ShenandoahConcurrentPhase gc_phase(msg, ShenandoahPhaseTimings::conc_class_unload);
 477   EventMark em("%s", msg);
 478 
 479   ShenandoahWorkerScope scope(heap->workers(),
 480                               ShenandoahWorkerPolicy::calc_workers_for_conc_root_processing(),
 481                               "concurrent class unloading");
 482 
 483   heap->try_inject_alloc_failure();
 484   op_class_unloading();
 485 }
 486 
 487 void ShenandoahConcurrentGC::entry_strong_roots() {
 488   ShenandoahHeap* const heap = ShenandoahHeap::heap();
 489   TraceCollectorStats tcs(heap->monitoring_support()->concurrent_collection_counters());
 490   static const char* msg = "Concurrent strong roots";
 491   ShenandoahConcurrentPhase gc_phase(msg, ShenandoahPhaseTimings::conc_strong_roots);
 492   EventMark em("%s", msg);
 493 
 494   ShenandoahGCWorkerPhase worker_phase(ShenandoahPhaseTimings::conc_strong_roots);
 495 
 496   ShenandoahWorkerScope scope(heap->workers(),
 497                               ShenandoahWorkerPolicy::calc_workers_for_conc_root_processing(),
 498                               "concurrent strong root");
 499 
 500   heap->try_inject_alloc_failure();
 501   op_strong_roots();
 502 }
 503 
 504 void ShenandoahConcurrentGC::entry_cleanup_early() {
 505   ShenandoahHeap* const heap = ShenandoahHeap::heap();
 506   TraceCollectorStats tcs(heap->monitoring_support()->concurrent_collection_counters());
 507   static const char* msg = "Concurrent cleanup";
 508   ShenandoahConcurrentPhase gc_phase(msg, ShenandoahPhaseTimings::conc_cleanup_early, true /* log_heap_usage */);
 509   EventMark em("%s", msg);
 510 
 511   // This phase does not use workers, no need for setup
 512   heap->try_inject_alloc_failure();
 513   op_cleanup_early();
 514 }
 515 
 516 void ShenandoahConcurrentGC::entry_evacuate() {
 517   ShenandoahHeap* const heap = ShenandoahHeap::heap();
 518   TraceCollectorStats tcs(heap->monitoring_support()->concurrent_collection_counters());
 519 
 520   static const char* msg = "Concurrent evacuation";
 521   ShenandoahConcurrentPhase gc_phase(msg, ShenandoahPhaseTimings::conc_evac);
 522   EventMark em("%s", msg);
 523 
 524   ShenandoahWorkerScope scope(heap->workers(),
 525                               ShenandoahWorkerPolicy::calc_workers_for_conc_evac(),
 526                               "concurrent evacuation");
 527 
 528   heap->try_inject_alloc_failure();
 529   op_evacuate();
 530 }
 531 
 532 void ShenandoahConcurrentGC::entry_update_thread_roots() {
 533   ShenandoahHeap* const heap = ShenandoahHeap::heap();
 534   TraceCollectorStats tcs(heap->monitoring_support()->concurrent_collection_counters());
 535 
 536   static const char* msg = "Concurrent update thread roots";
 537   ShenandoahConcurrentPhase gc_phase(msg, ShenandoahPhaseTimings::conc_update_thread_roots);
 538   EventMark em("%s", msg);
 539 
 540   // No workers used in this phase, no setup required
 541   heap->try_inject_alloc_failure();
 542   op_update_thread_roots();
 543 }
 544 
 545 void ShenandoahConcurrentGC::entry_updaterefs() {
 546   ShenandoahHeap* const heap = ShenandoahHeap::heap();
 547   TraceCollectorStats tcs(heap->monitoring_support()->concurrent_collection_counters());
 548   static const char* msg = "Concurrent update references";
 549   ShenandoahConcurrentPhase gc_phase(msg, ShenandoahPhaseTimings::conc_update_refs);
 550   EventMark em("%s", msg);
 551 
 552   ShenandoahWorkerScope scope(heap->workers(),
 553                               ShenandoahWorkerPolicy::calc_workers_for_conc_update_ref(),
 554                               "concurrent reference update");
 555 
 556   heap->try_inject_alloc_failure();
 557   op_updaterefs();
 558 }
 559 
 560 void ShenandoahConcurrentGC::entry_cleanup_complete() {
 561   ShenandoahHeap* const heap = ShenandoahHeap::heap();
 562   TraceCollectorStats tcs(heap->monitoring_support()->concurrent_collection_counters());
 563   static const char* msg = "Concurrent cleanup";
 564   ShenandoahConcurrentPhase gc_phase(msg, ShenandoahPhaseTimings::conc_cleanup_complete, true /* log_heap_usage */);
 565   EventMark em("%s", msg);
 566 
 567   // This phase does not use workers, no need for setup
 568   heap->try_inject_alloc_failure();
 569   op_cleanup_complete();
 570 }
 571 
 572 void ShenandoahConcurrentGC::op_reset() {
 573   ShenandoahHeap* const heap = ShenandoahHeap::heap();
 574   if (ShenandoahPacing) {
 575     heap->pacer()->setup_for_reset();
 576   }
 577   _generation->prepare_gc();
 578 }
 579 
 580 class ShenandoahInitMarkUpdateRegionStateClosure : public ShenandoahHeapRegionClosure {
 581 private:
 582   ShenandoahMarkingContext* const _ctx;
 583 public:
 584   ShenandoahInitMarkUpdateRegionStateClosure() : _ctx(ShenandoahHeap::heap()->marking_context()) {}
 585 
 586   void heap_region_do(ShenandoahHeapRegion* r) {
 587     assert(!r->has_live(), "Region " SIZE_FORMAT " should have no live data", r->index());
 588     if (r->is_active()) {
 589       // Check if region needs updating its TAMS. We have updated it already during concurrent
 590       // reset, so it is very likely we don't need to do another write here.  Since most regions
 591       // are not "active", this path is relatively rare.
 592       if (_ctx->top_at_mark_start(r) != r->top()) {
 593         _ctx->capture_top_at_mark_start(r);
 594       }
 595     } else {
 596       assert(_ctx->top_at_mark_start(r) == r->top(),
 597              "Region " SIZE_FORMAT " should already have correct TAMS", r->index());
 598     }
 599   }
 600 
 601   bool is_thread_safe() { return true; }
 602 };
 603 
 604 void ShenandoahConcurrentGC::start_mark() {
 605   _mark.start_mark();
 606 }
 607 
 608 void ShenandoahConcurrentGC::op_init_mark() {
 609   ShenandoahHeap* const heap = ShenandoahHeap::heap();
 610   assert(ShenandoahSafepoint::is_at_shenandoah_safepoint(), "Should be at safepoint");
 611   assert(Thread::current()->is_VM_thread(), "can only do this in VMThread");
 612 
 613   assert(_generation->is_bitmap_clear(), "need clear marking bitmap");
 614   assert(!_generation->is_mark_complete(), "should not be complete");
 615   assert(!heap->has_forwarded_objects(), "No forwarded objects on this path");
 616 
 617 
 618   if (heap->mode()->is_generational()) {
 619     if (_generation->is_young() || (_generation->is_global() && ShenandoahVerify)) {
 620       // The current implementation of swap_remembered_set() copies the write-card-table
 621       // to the read-card-table. The remembered sets are also swapped for GLOBAL collections
 622       // so that the verifier works with the correct copy of the card table when verifying.
 623       // TODO: This path should not really depend on ShenandoahVerify.
 624       ShenandoahGCPhase phase(ShenandoahPhaseTimings::init_swap_rset);
 625       _generation->swap_remembered_set();
 626     }
 627 
 628     if (_generation->is_global()) {
 629       heap->old_generation()->cancel_gc();
 630     } else if (heap->is_concurrent_old_mark_in_progress()) {
 631       // Purge the SATB buffers, transferring any valid, old pointers to the
 632       // old generation mark queue. Any pointers in a young region will be
 633       // abandoned.
 634       ShenandoahGCPhase phase(ShenandoahPhaseTimings::init_transfer_satb);
 635       heap->old_generation()->transfer_pointers_from_satb();
 636     }
 637   }
 638 
 639   if (ShenandoahVerify) {
 640     heap->verifier()->verify_before_concmark();
 641   }
 642 
 643   if (VerifyBeforeGC) {
 644     Universe::verify();
 645   }
 646 
 647   _generation->set_concurrent_mark_in_progress(true);
 648 
 649   start_mark();
 650 
 651   if (_do_old_gc_bootstrap) {
 652     shenandoah_assert_generational();
 653     // Update region state for both young and old regions
 654     // TODO: We should be able to pull this out of the safepoint for the bootstrap
 655     // cycle. The top of an old region will only move when a GC cycle evacuates
 656     // objects into it. When we start an old cycle, we know that nothing can touch
 657     // the top of old regions.
 658     ShenandoahGCPhase phase(ShenandoahPhaseTimings::init_update_region_states);
 659     ShenandoahInitMarkUpdateRegionStateClosure cl;
 660     heap->parallel_heap_region_iterate(&cl);
 661     heap->old_generation()->ref_processor()->reset_thread_locals();
 662   } else {
 663     // Update region state for only young regions
 664     ShenandoahGCPhase phase(ShenandoahPhaseTimings::init_update_region_states);
 665     ShenandoahInitMarkUpdateRegionStateClosure cl;
 666     _generation->parallel_heap_region_iterate(&cl);
 667   }
 668 
 669   // Weak reference processing
 670   ShenandoahReferenceProcessor* rp = _generation->ref_processor();
 671   rp->reset_thread_locals();
 672   rp->set_soft_reference_policy(heap->soft_ref_policy()->should_clear_all_soft_refs());
 673 
 674   // Make above changes visible to worker threads
 675   OrderAccess::fence();
 676 
 677   // Arm nmethods for concurrent mark
 678   ShenandoahCodeRoots::arm_nmethods_for_mark();
 679 
 680   ShenandoahStackWatermark::change_epoch_id();
 681   if (ShenandoahPacing) {
 682     heap->pacer()->setup_for_mark();
 683   }
 684 }
 685 
 686 void ShenandoahConcurrentGC::op_mark_roots() {
 687   _mark.mark_concurrent_roots();
 688 }
 689 
 690 void ShenandoahConcurrentGC::op_mark() {
 691   _mark.concurrent_mark();
 692 }
 693 
 694 void ShenandoahConcurrentGC::op_final_mark() {
 695   ShenandoahHeap* const heap = ShenandoahHeap::heap();
 696   assert(ShenandoahSafepoint::is_at_shenandoah_safepoint(), "Should be at safepoint");
 697   assert(!heap->has_forwarded_objects(), "No forwarded objects on this path");
 698 
 699   if (ShenandoahVerify) {
 700     heap->verifier()->verify_roots_no_forwarded();
 701   }
 702 
 703   if (!heap->cancelled_gc()) {
 704     _mark.finish_mark();
 705     assert(!heap->cancelled_gc(), "STW mark cannot OOM");
 706 
 707     // Notify JVMTI that the tagmap table will need cleaning.
 708     JvmtiTagMap::set_needs_cleaning();
 709 
 710     // The collection set is chosen by prepare_regions_and_collection_set().
 711     //
 712     // TODO: Under severe memory overload conditions that can be checked here, we may want to limit
 713     // the inclusion of old-gen candidates within the collection set.  This would allow us to prioritize efforts on
 714     // evacuating young-gen,  This remediation is most appropriate when old-gen availability is very high (so there
 715     // are negligible negative impacts from delaying completion of old-gen evacuation) and when young-gen collections
 716     // are "under duress" (as signalled by very low availability of memory within young-gen, indicating that/ young-gen
 717     // collections are not triggering frequently enough).
 718     _generation->prepare_regions_and_collection_set(true /*concurrent*/);
 719 
 720     // Upon return from prepare_regions_and_collection_set(), certain parameters have been established to govern the
 721     // evacuation efforts that are about to begin.  In particular:
 722     //
 723     // heap->get_promoted_reserve() represents the amount of memory within old-gen's available memory that has
 724     //   been set aside to hold objects promoted from young-gen memory.  This represents an estimated percentage
 725     //   of the live young-gen memory within the collection set.  If there is more data ready to be promoted than
 726     //   can fit within this reserve, the promotion of some objects will be deferred until a subsequent evacuation
 727     //   pass.
 728     //
 729     // heap->get_old_evac_reserve() represents the amount of memory within old-gen's available memory that has been
 730     //  set aside to hold objects evacuated from the old-gen collection set.
 731     //
 732     // heap->get_young_evac_reserve() represents the amount of memory within young-gen's available memory that has
 733     //  been set aside to hold objects evacuated from the young-gen collection set.  Conservatively, this value
 734     //  equals the entire amount of live young-gen memory within the collection set, even though some of this memory
 735     //  will likely be promoted.
 736 
 737     // Has to be done after cset selection
 738     heap->prepare_concurrent_roots();
 739 
 740     if (!heap->collection_set()->is_empty() || has_in_place_promotions(heap)) {
 741       // Even if the collection set is empty, we need to do evacuation if there are regions to be promoted in place.
 742       // Concurrent evacuation takes responsibility for registering objects and setting the remembered set cards to dirty.
 743 
 744       LogTarget(Debug, gc, cset) lt;
 745       if (lt.is_enabled()) {
 746         ResourceMark rm;
 747         LogStream ls(lt);
 748         heap->collection_set()->print_on(&ls);
 749       }
 750 
 751       if (ShenandoahVerify) {
 752         heap->verifier()->verify_before_evacuation();
 753       }
 754 
 755       // TODO: Do we need to set this if we are only promoting regions in place? We don't need the barriers on for that.
 756       heap->set_evacuation_in_progress(true);
 757 
 758       // Verify before arming for concurrent processing.
 759       // Otherwise, verification can trigger stack processing.
 760       if (ShenandoahVerify) {
 761         heap->verifier()->verify_during_evacuation();
 762       }
 763 
 764       // Generational mode may promote objects in place during the evacuation phase.
 765       // If that is the only reason we are evacuating, we don't need to update references
 766       // and there will be no forwarded objects on the heap.
 767       heap->set_has_forwarded_objects(!heap->collection_set()->is_empty());
 768 
 769       // Arm nmethods/stack for concurrent processing
 770       if (!heap->collection_set()->is_empty()) {
 771         // Iff objects will be evaluated, arm the nmethod barriers. These will be disarmed
 772         // under the same condition (established in prepare_concurrent_roots) after strong
 773         // root evacuation has completed (see op_strong_roots).
 774         ShenandoahCodeRoots::arm_nmethods_for_evac();
 775         ShenandoahStackWatermark::change_epoch_id();
 776       }
 777 
 778       if (ShenandoahPacing) {
 779         heap->pacer()->setup_for_evac();
 780       }
 781     } else {
 782       if (ShenandoahVerify) {
 783         heap->verifier()->verify_after_concmark();
 784       }
 785 
 786       if (VerifyAfterGC) {
 787         Universe::verify();
 788       }
 789     }
 790   }
 791 }
 792 
 793 bool ShenandoahConcurrentGC::has_in_place_promotions(ShenandoahHeap* heap) {
 794   return heap->mode()->is_generational() && heap->old_generation()->has_in_place_promotions();
 795 }
 796 
 797 template<bool GENERATIONAL>
 798 class ShenandoahConcurrentEvacThreadClosure : public ThreadClosure {
 799 private:
 800   OopClosure* const _oops;
 801 public:
 802   explicit ShenandoahConcurrentEvacThreadClosure(OopClosure* oops) : _oops(oops) {}
 803 
 804   void do_thread(Thread* thread) override {
 805     JavaThread* const jt = JavaThread::cast(thread);
 806     StackWatermarkSet::finish_processing(jt, _oops, StackWatermarkKind::gc);
 807     if (GENERATIONAL) {
 808       ShenandoahThreadLocalData::enable_plab_promotions(thread);
 809     }
 810   }
 811 };
 812 
 813 template<bool GENERATIONAL>
 814 class ShenandoahConcurrentEvacUpdateThreadTask : public WorkerTask {
 815 private:
 816   ShenandoahJavaThreadsIterator _java_threads;
 817 
 818 public:
 819   explicit ShenandoahConcurrentEvacUpdateThreadTask(uint n_workers) :
 820     WorkerTask("Shenandoah Evacuate/Update Concurrent Thread Roots"),
 821     _java_threads(ShenandoahPhaseTimings::conc_thread_roots, n_workers) {
 822   }
 823 
 824   void work(uint worker_id) override {
 825     if (GENERATIONAL) {
 826       Thread* worker_thread = Thread::current();
 827       ShenandoahThreadLocalData::enable_plab_promotions(worker_thread);
 828     }
 829 
 830     // ShenandoahEvacOOMScope has to be setup by ShenandoahContextEvacuateUpdateRootsClosure.
 831     // Otherwise, may deadlock with watermark lock
 832     ShenandoahContextEvacuateUpdateRootsClosure oops_cl;
 833     ShenandoahConcurrentEvacThreadClosure<GENERATIONAL> thr_cl(&oops_cl);
 834     _java_threads.threads_do(&thr_cl, worker_id);
 835   }
 836 };
 837 
 838 void ShenandoahConcurrentGC::op_thread_roots() {
 839   ShenandoahHeap* const heap = ShenandoahHeap::heap();
 840   assert(heap->is_evacuation_in_progress(), "Checked by caller");
 841   ShenandoahGCWorkerPhase worker_phase(ShenandoahPhaseTimings::conc_thread_roots);
 842   if (heap->mode()->is_generational()) {
 843     ShenandoahConcurrentEvacUpdateThreadTask<true> task(heap->workers()->active_workers());
 844     heap->workers()->run_task(&task);
 845   } else {
 846     ShenandoahConcurrentEvacUpdateThreadTask<false> task(heap->workers()->active_workers());
 847     heap->workers()->run_task(&task);
 848   }
 849 }
 850 
 851 void ShenandoahConcurrentGC::op_weak_refs() {
 852   ShenandoahHeap* const heap = ShenandoahHeap::heap();
 853   assert(heap->is_concurrent_weak_root_in_progress(), "Only during this phase");
 854   // Concurrent weak refs processing
 855   ShenandoahGCWorkerPhase worker_phase(ShenandoahPhaseTimings::conc_weak_refs);
 856   if (heap->gc_cause() == GCCause::_wb_breakpoint) {
 857     ShenandoahBreakpoint::at_after_reference_processing_started();
 858   }
 859   _generation->ref_processor()->process_references(ShenandoahPhaseTimings::conc_weak_refs, heap->workers(), true /* concurrent */);
 860 }
 861 
 862 class ShenandoahEvacUpdateCleanupOopStorageRootsClosure : public BasicOopIterateClosure {
 863 private:
 864   ShenandoahHeap* const _heap;
 865   ShenandoahMarkingContext* const _mark_context;
 866   bool  _evac_in_progress;
 867   Thread* const _thread;
 868 
 869 public:
 870   ShenandoahEvacUpdateCleanupOopStorageRootsClosure();
 871   void do_oop(oop* p);
 872   void do_oop(narrowOop* p);
 873 };
 874 
 875 ShenandoahEvacUpdateCleanupOopStorageRootsClosure::ShenandoahEvacUpdateCleanupOopStorageRootsClosure() :
 876   _heap(ShenandoahHeap::heap()),
 877   _mark_context(ShenandoahHeap::heap()->marking_context()),
 878   _evac_in_progress(ShenandoahHeap::heap()->is_evacuation_in_progress()),
 879   _thread(Thread::current()) {
 880 }
 881 
 882 void ShenandoahEvacUpdateCleanupOopStorageRootsClosure::do_oop(oop* p) {
 883   const oop obj = RawAccess<>::oop_load(p);
 884   if (!CompressedOops::is_null(obj)) {
 885     if (!_mark_context->is_marked(obj)) {
 886       if (_heap->is_in_active_generation(obj)) {
 887         // TODO: This worries me. Here we are asserting that an unmarked from-space object is 'correct'.
 888         // Normally, I would call this a bogus assert, but there seems to be a legitimate use-case for
 889         // accessing from-space objects during class unloading. However, the from-space object may have
 890         // been "filled". We've made no effort to prevent old generation classes being unloaded by young
 891         // gen (and vice-versa).
 892         shenandoah_assert_correct(p, obj);
 893         ShenandoahHeap::atomic_clear_oop(p, obj);
 894       }
 895     } else if (_evac_in_progress && _heap->in_collection_set(obj)) {
 896       oop resolved = ShenandoahBarrierSet::resolve_forwarded_not_null(obj);
 897       if (resolved == obj) {
 898         resolved = _heap->evacuate_object(obj, _thread);
 899       }
 900       shenandoah_assert_not_in_cset_except(p, resolved, _heap->cancelled_gc());
 901       ShenandoahHeap::atomic_update_oop(resolved, p, obj);
 902     }
 903   }
 904 }
 905 
 906 void ShenandoahEvacUpdateCleanupOopStorageRootsClosure::do_oop(narrowOop* p) {
 907   ShouldNotReachHere();
 908 }
 909 
 910 class ShenandoahIsCLDAliveClosure : public CLDClosure {
 911 public:
 912   void do_cld(ClassLoaderData* cld) {
 913     cld->is_alive();
 914   }
 915 };
 916 
 917 class ShenandoahIsNMethodAliveClosure: public NMethodClosure {
 918 public:
 919   void do_nmethod(nmethod* n) {
 920     n->is_unloading();
 921   }
 922 };
 923 
 924 // This task not only evacuates/updates marked weak roots, but also "null"
 925 // dead weak roots.
 926 class ShenandoahConcurrentWeakRootsEvacUpdateTask : public WorkerTask {
 927 private:
 928   ShenandoahVMWeakRoots<true /*concurrent*/> _vm_roots;
 929 
 930   // Roots related to concurrent class unloading
 931   ShenandoahClassLoaderDataRoots<true /* concurrent */>
 932                                              _cld_roots;
 933   ShenandoahConcurrentNMethodIterator        _nmethod_itr;
 934   ShenandoahPhaseTimings::Phase              _phase;
 935 
 936 public:
 937   ShenandoahConcurrentWeakRootsEvacUpdateTask(ShenandoahPhaseTimings::Phase phase) :
 938     WorkerTask("Shenandoah Evacuate/Update Concurrent Weak Roots"),
 939     _vm_roots(phase),
 940     _cld_roots(phase, ShenandoahHeap::heap()->workers()->active_workers(), false /*heap iteration*/),
 941     _nmethod_itr(ShenandoahCodeRoots::table()),
 942     _phase(phase) {
 943     if (ShenandoahHeap::heap()->unload_classes()) {
 944       MutexLocker mu(CodeCache_lock, Mutex::_no_safepoint_check_flag);
 945       _nmethod_itr.nmethods_do_begin();
 946     }
 947   }
 948 
 949   ~ShenandoahConcurrentWeakRootsEvacUpdateTask() {
 950     if (ShenandoahHeap::heap()->unload_classes()) {
 951       MutexLocker mu(CodeCache_lock, Mutex::_no_safepoint_check_flag);
 952       _nmethod_itr.nmethods_do_end();
 953     }
 954     // Notify runtime data structures of potentially dead oops
 955     _vm_roots.report_num_dead();
 956   }
 957 
 958   void work(uint worker_id) {
 959     ShenandoahConcurrentWorkerSession worker_session(worker_id);
 960     ShenandoahSuspendibleThreadSetJoiner sts_join;
 961     {
 962       ShenandoahEvacOOMScope oom;
 963       // jni_roots and weak_roots are OopStorage backed roots, concurrent iteration
 964       // may race against OopStorage::release() calls.
 965       ShenandoahEvacUpdateCleanupOopStorageRootsClosure cl;
 966       _vm_roots.oops_do(&cl, worker_id);
 967     }
 968 
 969     // If we are going to perform concurrent class unloading later on, we need to
 970     // cleanup the weak oops in CLD and determinate nmethod's unloading state, so that we
 971     // can cleanup immediate garbage sooner.
 972     if (ShenandoahHeap::heap()->unload_classes()) {
 973       // Applies ShenandoahIsCLDAlive closure to CLDs, native barrier will either null the
 974       // CLD's holder or evacuate it.
 975       {
 976         ShenandoahIsCLDAliveClosure is_cld_alive;
 977         _cld_roots.cld_do(&is_cld_alive, worker_id);
 978       }
 979 
 980       // Applies ShenandoahIsNMethodAliveClosure to registered nmethods.
 981       // The closure calls nmethod->is_unloading(). The is_unloading
 982       // state is cached, therefore, during concurrent class unloading phase,
 983       // we will not touch the metadata of unloading nmethods
 984       {
 985         ShenandoahWorkerTimingsTracker timer(_phase, ShenandoahPhaseTimings::CodeCacheRoots, worker_id);
 986         ShenandoahIsNMethodAliveClosure is_nmethod_alive;
 987         _nmethod_itr.nmethods_do(&is_nmethod_alive);
 988       }
 989     }
 990   }
 991 };
 992 
 993 void ShenandoahConcurrentGC::op_weak_roots() {
 994   ShenandoahHeap* const heap = ShenandoahHeap::heap();
 995   assert(heap->is_concurrent_weak_root_in_progress(), "Only during this phase");
 996   // Concurrent weak root processing
 997   {
 998     ShenandoahTimingsTracker t(ShenandoahPhaseTimings::conc_weak_roots_work);
 999     ShenandoahGCWorkerPhase worker_phase(ShenandoahPhaseTimings::conc_weak_roots_work);
1000     ShenandoahConcurrentWeakRootsEvacUpdateTask task(ShenandoahPhaseTimings::conc_weak_roots_work);
1001     heap->workers()->run_task(&task);
1002   }
1003 
1004   // Perform handshake to flush out dead oops
1005   {
1006     ShenandoahTimingsTracker t(ShenandoahPhaseTimings::conc_weak_roots_rendezvous);
1007     heap->rendezvous_threads();
1008   }
1009 }
1010 
1011 void ShenandoahConcurrentGC::op_class_unloading() {
1012   ShenandoahHeap* const heap = ShenandoahHeap::heap();
1013   assert (heap->is_concurrent_weak_root_in_progress() &&
1014           heap->unload_classes(),
1015           "Checked by caller");
1016   heap->do_class_unloading();
1017 }
1018 
1019 class ShenandoahEvacUpdateCodeCacheClosure : public NMethodClosure {
1020 private:
1021   BarrierSetNMethod* const                  _bs;
1022   ShenandoahEvacuateUpdateMetadataClosure   _cl;
1023 
1024 public:
1025   ShenandoahEvacUpdateCodeCacheClosure() :
1026     _bs(BarrierSet::barrier_set()->barrier_set_nmethod()),
1027     _cl() {
1028   }
1029 
1030   void do_nmethod(nmethod* n) {
1031     ShenandoahNMethod* data = ShenandoahNMethod::gc_data(n);
1032     ShenandoahReentrantLocker locker(data->lock());
1033     // Setup EvacOOM scope below reentrant lock to avoid deadlock with
1034     // nmethod_entry_barrier
1035     ShenandoahEvacOOMScope oom;
1036     data->oops_do(&_cl, true/*fix relocation*/);
1037     _bs->disarm(n);
1038   }
1039 };
1040 
1041 class ShenandoahConcurrentRootsEvacUpdateTask : public WorkerTask {
1042 private:
1043   ShenandoahPhaseTimings::Phase                 _phase;
1044   ShenandoahVMRoots<true /*concurrent*/>        _vm_roots;
1045   ShenandoahClassLoaderDataRoots<true /*concurrent*/>
1046                                                 _cld_roots;
1047   ShenandoahConcurrentNMethodIterator           _nmethod_itr;
1048 
1049 public:
1050   ShenandoahConcurrentRootsEvacUpdateTask(ShenandoahPhaseTimings::Phase phase) :
1051     WorkerTask("Shenandoah Evacuate/Update Concurrent Strong Roots"),
1052     _phase(phase),
1053     _vm_roots(phase),
1054     _cld_roots(phase, ShenandoahHeap::heap()->workers()->active_workers(), false /*heap iteration*/),
1055     _nmethod_itr(ShenandoahCodeRoots::table()) {
1056     if (!ShenandoahHeap::heap()->unload_classes()) {
1057       MutexLocker mu(CodeCache_lock, Mutex::_no_safepoint_check_flag);
1058       _nmethod_itr.nmethods_do_begin();
1059     }
1060   }
1061 
1062   ~ShenandoahConcurrentRootsEvacUpdateTask() {
1063     if (!ShenandoahHeap::heap()->unload_classes()) {
1064       MutexLocker mu(CodeCache_lock, Mutex::_no_safepoint_check_flag);
1065       _nmethod_itr.nmethods_do_end();
1066     }
1067   }
1068 
1069   void work(uint worker_id) {
1070     ShenandoahConcurrentWorkerSession worker_session(worker_id);
1071     {
1072       ShenandoahEvacOOMScope oom;
1073       {
1074         // vm_roots and weak_roots are OopStorage backed roots, concurrent iteration
1075         // may race against OopStorage::release() calls.
1076         ShenandoahContextEvacuateUpdateRootsClosure cl;
1077         _vm_roots.oops_do<ShenandoahContextEvacuateUpdateRootsClosure>(&cl, worker_id);
1078       }
1079 
1080       {
1081         ShenandoahEvacuateUpdateMetadataClosure cl;
1082         CLDToOopClosure clds(&cl, ClassLoaderData::_claim_strong);
1083         _cld_roots.cld_do(&clds, worker_id);
1084       }
1085     }
1086 
1087     // Cannot setup ShenandoahEvacOOMScope here, due to potential deadlock with nmethod_entry_barrier.
1088     if (!ShenandoahHeap::heap()->unload_classes()) {
1089       ShenandoahWorkerTimingsTracker timer(_phase, ShenandoahPhaseTimings::CodeCacheRoots, worker_id);
1090       ShenandoahEvacUpdateCodeCacheClosure cl;
1091       _nmethod_itr.nmethods_do(&cl);
1092     }
1093   }
1094 };
1095 
1096 void ShenandoahConcurrentGC::op_strong_roots() {
1097   ShenandoahHeap* const heap = ShenandoahHeap::heap();
1098   assert(heap->is_concurrent_strong_root_in_progress(), "Checked by caller");
1099   ShenandoahConcurrentRootsEvacUpdateTask task(ShenandoahPhaseTimings::conc_strong_roots);
1100   heap->workers()->run_task(&task);
1101   heap->set_concurrent_strong_root_in_progress(false);
1102 }
1103 
1104 void ShenandoahConcurrentGC::op_cleanup_early() {
1105   ShenandoahHeap::heap()->free_set()->recycle_trash();
1106 }
1107 
1108 void ShenandoahConcurrentGC::op_evacuate() {
1109   ShenandoahHeap::heap()->evacuate_collection_set(true /*concurrent*/);
1110 }
1111 
1112 void ShenandoahConcurrentGC::op_init_updaterefs() {
1113   ShenandoahHeap* const heap = ShenandoahHeap::heap();
1114   heap->set_evacuation_in_progress(false);
1115   heap->set_concurrent_weak_root_in_progress(false);
1116   heap->prepare_update_heap_references(true /*concurrent*/);
1117   heap->set_update_refs_in_progress(true);
1118   if (ShenandoahVerify) {
1119     heap->verifier()->verify_before_updaterefs();
1120   }
1121   if (ShenandoahPacing) {
1122     heap->pacer()->setup_for_updaterefs();
1123   }
1124 }
1125 
1126 void ShenandoahConcurrentGC::op_updaterefs() {
1127   ShenandoahHeap::heap()->update_heap_references(true /*concurrent*/);
1128 }
1129 
1130 class ShenandoahUpdateThreadClosure : public HandshakeClosure {
1131 private:
1132   ShenandoahUpdateRefsClosure _cl;
1133 public:
1134   ShenandoahUpdateThreadClosure();
1135   void do_thread(Thread* thread);
1136 };
1137 
1138 ShenandoahUpdateThreadClosure::ShenandoahUpdateThreadClosure() :
1139   HandshakeClosure("Shenandoah Update Thread Roots") {
1140 }
1141 
1142 void ShenandoahUpdateThreadClosure::do_thread(Thread* thread) {
1143   if (thread->is_Java_thread()) {
1144     JavaThread* jt = JavaThread::cast(thread);
1145     ResourceMark rm;
1146     jt->oops_do(&_cl, nullptr);
1147   }
1148 }
1149 
1150 void ShenandoahConcurrentGC::op_update_thread_roots() {
1151   ShenandoahUpdateThreadClosure cl;
1152   Handshake::execute(&cl);
1153 }
1154 
1155 void ShenandoahConcurrentGC::op_final_updaterefs() {
1156   ShenandoahHeap* const heap = ShenandoahHeap::heap();
1157   assert(ShenandoahSafepoint::is_at_shenandoah_safepoint(), "must be at safepoint");
1158   assert(!heap->_update_refs_iterator.has_next(), "Should have finished update references");
1159 
1160   heap->finish_concurrent_roots();
1161 
1162   // Clear cancelled GC, if set. On cancellation path, the block before would handle
1163   // everything.
1164   if (heap->cancelled_gc()) {
1165     heap->clear_cancelled_gc(true /* clear oom handler */);
1166   }
1167 
1168   // Has to be done before cset is clear
1169   if (ShenandoahVerify) {
1170     heap->verifier()->verify_roots_in_to_space();
1171   }
1172 
1173   // If we are running in generational mode and this is an aging cycle, this will also age active
1174   // regions that haven't been used for allocation.
1175   heap->update_heap_region_states(true /*concurrent*/);
1176 
1177   heap->set_update_refs_in_progress(false);
1178   heap->set_has_forwarded_objects(false);
1179 
1180   if (heap->mode()->is_generational() && heap->is_concurrent_old_mark_in_progress()) {
1181     // When the SATB barrier is left on to support concurrent old gen mark, it may pick up writes to
1182     // objects in the collection set. After those objects are evacuated, the pointers in the
1183     // SATB are no longer safe. Once we have finished update references, we are guaranteed that
1184     // no more writes to the collection set are possible.
1185     //
1186     // This will transfer any old pointers in _active_ regions from the SATB to the old gen
1187     // mark queues. All other pointers will be discarded. This would also discard any pointers
1188     // in old regions that were included in a mixed evacuation. We aren't using the SATB filter
1189     // methods here because we cannot control when they execute. If the SATB filter runs _after_
1190     // a region has been recycled, we will not be able to detect the bad pointer.
1191     //
1192     // We are not concerned about skipping this step in abbreviated cycles because regions
1193     // with no live objects cannot have been written to and so cannot have entries in the SATB
1194     // buffers.
1195     heap->old_generation()->transfer_pointers_from_satb();
1196 
1197     // Aging_cycle is only relevant during evacuation cycle for individual objects and during final mark for
1198     // entire regions.  Both of these relevant operations occur before final update refs.
1199     ShenandoahGenerationalHeap::heap()->set_aging_cycle(false);
1200   }
1201 
1202   if (ShenandoahVerify) {
1203     heap->verifier()->verify_after_updaterefs();
1204   }
1205 
1206   if (VerifyAfterGC) {
1207     Universe::verify();
1208   }
1209 
1210   heap->rebuild_free_set(true /*concurrent*/);
1211 }
1212 
1213 void ShenandoahConcurrentGC::op_final_roots() {
1214 
1215   ShenandoahHeap *heap = ShenandoahHeap::heap();
1216   heap->set_concurrent_weak_root_in_progress(false);
1217   heap->set_evacuation_in_progress(false);
1218 
1219   if (heap->mode()->is_generational()) {
1220     // If the cycle was shortened for having enough immediate garbage, this could be
1221     // the last GC safepoint before concurrent marking of old resumes. We must be sure
1222     // that old mark threads don't see any pointers to garbage in the SATB buffers.
1223     if (heap->is_concurrent_old_mark_in_progress()) {
1224       heap->old_generation()->transfer_pointers_from_satb();
1225     }
1226 
1227     if (!_generation->is_old()) {
1228       ShenandoahGenerationalHeap::heap()->update_region_ages(_generation->complete_marking_context());
1229     }
1230   }
1231 }
1232 
1233 void ShenandoahConcurrentGC::op_cleanup_complete() {
1234   ShenandoahHeap::heap()->free_set()->recycle_trash();
1235 }
1236 
1237 bool ShenandoahConcurrentGC::check_cancellation_and_abort(ShenandoahDegenPoint point) {
1238   if (ShenandoahHeap::heap()->cancelled_gc()) {
1239     _degen_point = point;
1240     return true;
1241   }
1242   return false;
1243 }
1244 
1245 const char* ShenandoahConcurrentGC::init_mark_event_message() const {
1246   ShenandoahHeap* const heap = ShenandoahHeap::heap();
1247   assert(!heap->has_forwarded_objects(), "Should not have forwarded objects here");
1248   if (heap->unload_classes()) {
1249     SHENANDOAH_RETURN_EVENT_MESSAGE(_generation->type(), "Pause Init Mark", " (unload classes)");
1250   } else {
1251     SHENANDOAH_RETURN_EVENT_MESSAGE(_generation->type(), "Pause Init Mark", "");
1252   }
1253 }
1254 
1255 const char* ShenandoahConcurrentGC::final_mark_event_message() const {
1256   ShenandoahHeap* const heap = ShenandoahHeap::heap();
1257   assert(!heap->has_forwarded_objects() || heap->is_concurrent_old_mark_in_progress(),
1258          "Should not have forwarded objects during final mark, unless old gen concurrent mark is running");
1259 
1260   if (heap->unload_classes()) {
1261     SHENANDOAH_RETURN_EVENT_MESSAGE(_generation->type(), "Pause Final Mark", " (unload classes)");
1262   } else {
1263     SHENANDOAH_RETURN_EVENT_MESSAGE(_generation->type(), "Pause Final Mark", "");
1264   }
1265 }
1266 
1267 const char* ShenandoahConcurrentGC::conc_mark_event_message() const {
1268   ShenandoahHeap* const heap = ShenandoahHeap::heap();
1269   assert(!heap->has_forwarded_objects() || heap->is_concurrent_old_mark_in_progress(),
1270          "Should not have forwarded objects concurrent mark, unless old gen concurrent mark is running");
1271   if (heap->unload_classes()) {
1272     SHENANDOAH_RETURN_EVENT_MESSAGE(_generation->type(), "Concurrent marking", " (unload classes)");
1273   } else {
1274     SHENANDOAH_RETURN_EVENT_MESSAGE(_generation->type(), "Concurrent marking", "");
1275   }
1276 }