1 /*
   2  * Copyright (c) 2021, 2022, Red Hat, Inc. All rights reserved.
   3  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
   4  *
   5  * This code is free software; you can redistribute it and/or modify it
   6  * under the terms of the GNU General Public License version 2 only, as
   7  * published by the Free Software Foundation.
   8  *
   9  * This code is distributed in the hope that it will be useful, but WITHOUT
  10  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  11  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  12  * version 2 for more details (a copy is included in the LICENSE file that
  13  * accompanied this code).
  14  *
  15  * You should have received a copy of the GNU General Public License version
  16  * 2 along with this work; if not, write to the Free Software Foundation,
  17  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
  18  *
  19  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
  20  * or visit www.oracle.com if you need additional information or have any
  21  * questions.
  22  *
  23  */
  24 
  25 #include "precompiled.hpp"
  26 
  27 #include "gc/shared/barrierSetNMethod.hpp"
  28 #include "gc/shared/collectorCounters.hpp"
  29 #include "gc/shared/continuationGCSupport.inline.hpp"
  30 #include "gc/shenandoah/shenandoahBreakpoint.hpp"
  31 #include "gc/shenandoah/shenandoahCollectorPolicy.hpp"
  32 #include "gc/shenandoah/shenandoahConcurrentGC.hpp"
  33 #include "gc/shenandoah/shenandoahFreeSet.hpp"
  34 #include "gc/shenandoah/shenandoahLock.hpp"
  35 #include "gc/shenandoah/shenandoahMark.inline.hpp"
  36 #include "gc/shenandoah/shenandoahMonitoringSupport.hpp"
  37 #include "gc/shenandoah/shenandoahOopClosures.inline.hpp"
  38 #include "gc/shenandoah/shenandoahPhaseTimings.hpp"
  39 #include "gc/shenandoah/shenandoahReferenceProcessor.hpp"
  40 #include "gc/shenandoah/shenandoahRootProcessor.inline.hpp"
  41 #include "gc/shenandoah/shenandoahStackWatermark.hpp"
  42 #include "gc/shenandoah/shenandoahUtils.hpp"
  43 #include "gc/shenandoah/shenandoahVerifier.hpp"
  44 #include "gc/shenandoah/shenandoahVMOperations.hpp"
  45 #include "gc/shenandoah/shenandoahWorkGroup.hpp"
  46 #include "gc/shenandoah/shenandoahWorkerPolicy.hpp"
  47 #include "memory/allocation.hpp"
  48 #include "prims/jvmtiTagMap.hpp"
  49 #include "runtime/vmThread.hpp"
  50 #include "utilities/events.hpp"
  51 
  52 // Breakpoint support
  53 class ShenandoahBreakpointGCScope : public StackObj {
  54 private:
  55   const GCCause::Cause _cause;
  56 public:
  57   ShenandoahBreakpointGCScope(GCCause::Cause cause) : _cause(cause) {
  58     if (cause == GCCause::_wb_breakpoint) {
  59       ShenandoahBreakpoint::start_gc();
  60       ShenandoahBreakpoint::at_before_gc();
  61     }
  62   }
  63 
  64   ~ShenandoahBreakpointGCScope() {
  65     if (_cause == GCCause::_wb_breakpoint) {
  66       ShenandoahBreakpoint::at_after_gc();
  67     }
  68   }
  69 };
  70 
  71 class ShenandoahBreakpointMarkScope : public StackObj {
  72 private:
  73   const GCCause::Cause _cause;
  74 public:
  75   ShenandoahBreakpointMarkScope(GCCause::Cause cause) : _cause(cause) {
  76     if (_cause == GCCause::_wb_breakpoint) {
  77       ShenandoahBreakpoint::at_after_marking_started();
  78     }
  79   }
  80 
  81   ~ShenandoahBreakpointMarkScope() {
  82     if (_cause == GCCause::_wb_breakpoint) {
  83       ShenandoahBreakpoint::at_before_marking_completed();
  84     }
  85   }
  86 };
  87 
  88 ShenandoahConcurrentGC::ShenandoahConcurrentGC() :
  89   _mark(),
  90   _degen_point(ShenandoahDegenPoint::_degenerated_unset) {
  91 }
  92 
  93 ShenandoahGC::ShenandoahDegenPoint ShenandoahConcurrentGC::degen_point() const {
  94   return _degen_point;
  95 }
  96 
  97 void ShenandoahConcurrentGC::cancel() {
  98   ShenandoahConcurrentMark::cancel();
  99 }
 100 
 101 bool ShenandoahConcurrentGC::collect(GCCause::Cause cause) {
 102   ShenandoahHeap* const heap = ShenandoahHeap::heap();
 103   ShenandoahBreakpointGCScope breakpoint_gc_scope(cause);
 104 
 105   // Reset for upcoming marking
 106   entry_reset();
 107 
 108   // Start initial mark under STW
 109   vmop_entry_init_mark();
 110 
 111   {
 112     ShenandoahBreakpointMarkScope breakpoint_mark_scope(cause);
 113     // Concurrent mark roots
 114     entry_mark_roots();
 115     if (check_cancellation_and_abort(ShenandoahDegenPoint::_degenerated_outside_cycle)) return false;
 116 
 117     // Continue concurrent mark
 118     entry_mark();
 119     if (check_cancellation_and_abort(ShenandoahDegenPoint::_degenerated_mark)) return false;
 120   }
 121 
 122   // Complete marking under STW, and start evacuation
 123   vmop_entry_final_mark();
 124 
 125   // Concurrent stack processing
 126   if (heap->is_evacuation_in_progress()) {
 127     entry_thread_roots();
 128   }
 129 
 130   // Process weak roots that might still point to regions that would be broken by cleanup
 131   if (heap->is_concurrent_weak_root_in_progress()) {
 132     entry_weak_refs();
 133     entry_weak_roots();
 134   }
 135 
 136   // Final mark might have reclaimed some immediate garbage, kick cleanup to reclaim
 137   // the space. This would be the last action if there is nothing to evacuate.
 138   entry_cleanup_early();
 139 
 140   {
 141     ShenandoahHeapLocker locker(heap->lock());
 142     heap->free_set()->log_status();
 143   }
 144 
 145   // Perform concurrent class unloading
 146   if (heap->unload_classes() &&
 147       heap->is_concurrent_weak_root_in_progress()) {
 148     entry_class_unloading();
 149   }
 150 
 151   // Processing strong roots
 152   // This may be skipped if there is nothing to update/evacuate.
 153   // If so, strong_root_in_progress would be unset.
 154   if (heap->is_concurrent_strong_root_in_progress()) {
 155     entry_strong_roots();
 156   }
 157 
 158   // Continue the cycle with evacuation and optional update-refs.
 159   // This may be skipped if there is nothing to evacuate.
 160   // If so, evac_in_progress would be unset by collection set preparation code.
 161   if (heap->is_evacuation_in_progress()) {
 162     // Concurrently evacuate
 163     entry_evacuate();
 164     if (check_cancellation_and_abort(ShenandoahDegenPoint::_degenerated_evac)) return false;
 165 
 166     // Perform update-refs phase.
 167     vmop_entry_init_updaterefs();
 168     entry_updaterefs();
 169     if (check_cancellation_and_abort(ShenandoahDegenPoint::_degenerated_updaterefs)) return false;
 170 
 171     // Concurrent update thread roots
 172     entry_update_thread_roots();
 173     if (check_cancellation_and_abort(ShenandoahDegenPoint::_degenerated_updaterefs)) return false;
 174 
 175     vmop_entry_final_updaterefs();
 176 
 177     // Update references freed up collection set, kick the cleanup to reclaim the space.
 178     entry_cleanup_complete();
 179   } else {
 180     vmop_entry_final_roots();
 181   }
 182 
 183   return true;
 184 }
 185 
 186 void ShenandoahConcurrentGC::vmop_entry_init_mark() {
 187   ShenandoahHeap* const heap = ShenandoahHeap::heap();
 188   TraceCollectorStats tcs(heap->monitoring_support()->stw_collection_counters());
 189   ShenandoahTimingsTracker timing(ShenandoahPhaseTimings::init_mark_gross);
 190 
 191   heap->try_inject_alloc_failure();
 192   VM_ShenandoahInitMark op(this);
 193   VMThread::execute(&op); // jump to entry_init_mark() under safepoint
 194 }
 195 
 196 void ShenandoahConcurrentGC::vmop_entry_final_mark() {
 197   ShenandoahHeap* const heap = ShenandoahHeap::heap();
 198   TraceCollectorStats tcs(heap->monitoring_support()->stw_collection_counters());
 199   ShenandoahTimingsTracker timing(ShenandoahPhaseTimings::final_mark_gross);
 200 
 201   heap->try_inject_alloc_failure();
 202   VM_ShenandoahFinalMarkStartEvac op(this);
 203   VMThread::execute(&op); // jump to entry_final_mark under safepoint
 204 }
 205 
 206 void ShenandoahConcurrentGC::vmop_entry_init_updaterefs() {
 207   ShenandoahHeap* const heap = ShenandoahHeap::heap();
 208   TraceCollectorStats tcs(heap->monitoring_support()->stw_collection_counters());
 209   ShenandoahTimingsTracker timing(ShenandoahPhaseTimings::init_update_refs_gross);
 210 
 211   heap->try_inject_alloc_failure();
 212   VM_ShenandoahInitUpdateRefs op(this);
 213   VMThread::execute(&op);
 214 }
 215 
 216 void ShenandoahConcurrentGC::vmop_entry_final_updaterefs() {
 217   ShenandoahHeap* const heap = ShenandoahHeap::heap();
 218   TraceCollectorStats tcs(heap->monitoring_support()->stw_collection_counters());
 219   ShenandoahTimingsTracker timing(ShenandoahPhaseTimings::final_update_refs_gross);
 220 
 221   heap->try_inject_alloc_failure();
 222   VM_ShenandoahFinalUpdateRefs op(this);
 223   VMThread::execute(&op);
 224 }
 225 
 226 void ShenandoahConcurrentGC::vmop_entry_final_roots() {
 227   ShenandoahHeap* const heap = ShenandoahHeap::heap();
 228   TraceCollectorStats tcs(heap->monitoring_support()->stw_collection_counters());
 229   ShenandoahTimingsTracker timing(ShenandoahPhaseTimings::final_roots_gross);
 230 
 231   // This phase does not use workers, no need for setup
 232   heap->try_inject_alloc_failure();
 233   VM_ShenandoahFinalRoots op(this);
 234   VMThread::execute(&op);
 235 }
 236 
 237 void ShenandoahConcurrentGC::entry_init_mark() {
 238   const char* msg = init_mark_event_message();
 239   ShenandoahPausePhase gc_phase(msg, ShenandoahPhaseTimings::init_mark);
 240   EventMark em("%s", msg);
 241 
 242   ShenandoahWorkerScope scope(ShenandoahHeap::heap()->workers(),
 243                               ShenandoahWorkerPolicy::calc_workers_for_init_marking(),
 244                               "init marking");
 245 
 246   op_init_mark();
 247 }
 248 
 249 void ShenandoahConcurrentGC::entry_final_mark() {
 250   const char* msg = final_mark_event_message();
 251   ShenandoahPausePhase gc_phase(msg, ShenandoahPhaseTimings::final_mark);
 252   EventMark em("%s", msg);
 253 
 254   ShenandoahWorkerScope scope(ShenandoahHeap::heap()->workers(),
 255                               ShenandoahWorkerPolicy::calc_workers_for_final_marking(),
 256                               "final marking");
 257 
 258   op_final_mark();
 259 }
 260 
 261 void ShenandoahConcurrentGC::entry_init_updaterefs() {
 262   static const char* msg = "Pause Init Update Refs";
 263   ShenandoahPausePhase gc_phase(msg, ShenandoahPhaseTimings::init_update_refs);
 264   EventMark em("%s", msg);
 265 
 266   // No workers used in this phase, no setup required
 267   op_init_updaterefs();
 268 }
 269 
 270 void ShenandoahConcurrentGC::entry_final_updaterefs() {
 271   static const char* msg = "Pause Final Update Refs";
 272   ShenandoahPausePhase gc_phase(msg, ShenandoahPhaseTimings::final_update_refs);
 273   EventMark em("%s", msg);
 274 
 275   ShenandoahWorkerScope scope(ShenandoahHeap::heap()->workers(),
 276                               ShenandoahWorkerPolicy::calc_workers_for_final_update_ref(),
 277                               "final reference update");
 278 
 279   op_final_updaterefs();
 280 }
 281 
 282 void ShenandoahConcurrentGC::entry_final_roots() {
 283   static const char* msg = "Pause Final Roots";
 284   ShenandoahPausePhase gc_phase(msg, ShenandoahPhaseTimings::final_roots);
 285   EventMark em("%s", msg);
 286 
 287   op_final_roots();
 288 }
 289 
 290 void ShenandoahConcurrentGC::entry_reset() {
 291   ShenandoahHeap* const heap = ShenandoahHeap::heap();
 292   TraceCollectorStats tcs(heap->monitoring_support()->concurrent_collection_counters());
 293   static const char* msg = "Concurrent reset";
 294   ShenandoahConcurrentPhase gc_phase(msg, ShenandoahPhaseTimings::conc_reset);
 295   EventMark em("%s", msg);
 296 
 297   ShenandoahWorkerScope scope(heap->workers(),
 298                               ShenandoahWorkerPolicy::calc_workers_for_conc_reset(),
 299                               "concurrent reset");
 300 
 301   heap->try_inject_alloc_failure();
 302   op_reset();
 303 }
 304 
 305 void ShenandoahConcurrentGC::entry_mark_roots() {
 306   ShenandoahHeap* const heap = ShenandoahHeap::heap();
 307   TraceCollectorStats tcs(heap->monitoring_support()->concurrent_collection_counters());
 308   const char* msg = "Concurrent marking roots";
 309   ShenandoahConcurrentPhase gc_phase(msg, ShenandoahPhaseTimings::conc_mark_roots);
 310   EventMark em("%s", msg);
 311 
 312   ShenandoahWorkerScope scope(heap->workers(),
 313                               ShenandoahWorkerPolicy::calc_workers_for_conc_marking(),
 314                               "concurrent marking roots");
 315 
 316   heap->try_inject_alloc_failure();
 317   op_mark_roots();
 318 }
 319 
 320 void ShenandoahConcurrentGC::entry_mark() {
 321   ShenandoahHeap* const heap = ShenandoahHeap::heap();
 322   TraceCollectorStats tcs(heap->monitoring_support()->concurrent_collection_counters());
 323   const char* msg = conc_mark_event_message();
 324   ShenandoahConcurrentPhase gc_phase(msg, ShenandoahPhaseTimings::conc_mark);
 325   EventMark em("%s", msg);
 326 
 327   ShenandoahWorkerScope scope(heap->workers(),
 328                               ShenandoahWorkerPolicy::calc_workers_for_conc_marking(),
 329                               "concurrent marking");
 330 
 331   heap->try_inject_alloc_failure();
 332   op_mark();
 333 }
 334 
 335 void ShenandoahConcurrentGC::entry_thread_roots() {
 336   ShenandoahHeap* const heap = ShenandoahHeap::heap();
 337   static const char* msg = "Concurrent thread roots";
 338   ShenandoahConcurrentPhase gc_phase(msg, ShenandoahPhaseTimings::conc_thread_roots);
 339   EventMark em("%s", msg);
 340 
 341   ShenandoahWorkerScope scope(heap->workers(),
 342                               ShenandoahWorkerPolicy::calc_workers_for_conc_root_processing(),
 343                               msg);
 344 
 345   heap->try_inject_alloc_failure();
 346   op_thread_roots();
 347 }
 348 
 349 void ShenandoahConcurrentGC::entry_weak_refs() {
 350   ShenandoahHeap* const heap = ShenandoahHeap::heap();
 351   static const char* msg = "Concurrent weak references";
 352   ShenandoahConcurrentPhase gc_phase(msg, ShenandoahPhaseTimings::conc_weak_refs);
 353   EventMark em("%s", msg);
 354 
 355   ShenandoahWorkerScope scope(heap->workers(),
 356                               ShenandoahWorkerPolicy::calc_workers_for_conc_refs_processing(),
 357                               "concurrent weak references");
 358 
 359   heap->try_inject_alloc_failure();
 360   op_weak_refs();
 361 }
 362 
 363 void ShenandoahConcurrentGC::entry_weak_roots() {
 364   ShenandoahHeap* const heap = ShenandoahHeap::heap();
 365   TraceCollectorStats tcs(heap->monitoring_support()->concurrent_collection_counters());
 366   static const char* msg = "Concurrent weak roots";
 367   ShenandoahConcurrentPhase gc_phase(msg, ShenandoahPhaseTimings::conc_weak_roots);
 368   EventMark em("%s", msg);
 369 
 370   ShenandoahWorkerScope scope(heap->workers(),
 371                               ShenandoahWorkerPolicy::calc_workers_for_conc_root_processing(),
 372                               "concurrent weak root");
 373 
 374   heap->try_inject_alloc_failure();
 375   op_weak_roots();
 376 }
 377 
 378 void ShenandoahConcurrentGC::entry_class_unloading() {
 379   ShenandoahHeap* const heap = ShenandoahHeap::heap();
 380   TraceCollectorStats tcs(heap->monitoring_support()->concurrent_collection_counters());
 381   static const char* msg = "Concurrent class unloading";
 382   ShenandoahConcurrentPhase gc_phase(msg, ShenandoahPhaseTimings::conc_class_unload);
 383   EventMark em("%s", msg);
 384 
 385   ShenandoahWorkerScope scope(heap->workers(),
 386                               ShenandoahWorkerPolicy::calc_workers_for_conc_root_processing(),
 387                               "concurrent class unloading");
 388 
 389   heap->try_inject_alloc_failure();
 390   op_class_unloading();
 391 }
 392 
 393 void ShenandoahConcurrentGC::entry_strong_roots() {
 394   ShenandoahHeap* const heap = ShenandoahHeap::heap();
 395   TraceCollectorStats tcs(heap->monitoring_support()->concurrent_collection_counters());
 396   static const char* msg = "Concurrent strong roots";
 397   ShenandoahConcurrentPhase gc_phase(msg, ShenandoahPhaseTimings::conc_strong_roots);
 398   EventMark em("%s", msg);
 399 
 400   ShenandoahGCWorkerPhase worker_phase(ShenandoahPhaseTimings::conc_strong_roots);
 401 
 402   ShenandoahWorkerScope scope(heap->workers(),
 403                               ShenandoahWorkerPolicy::calc_workers_for_conc_root_processing(),
 404                               "concurrent strong root");
 405 
 406   heap->try_inject_alloc_failure();
 407   op_strong_roots();
 408 }
 409 
 410 void ShenandoahConcurrentGC::entry_cleanup_early() {
 411   ShenandoahHeap* const heap = ShenandoahHeap::heap();
 412   TraceCollectorStats tcs(heap->monitoring_support()->concurrent_collection_counters());
 413   static const char* msg = "Concurrent cleanup";
 414   ShenandoahConcurrentPhase gc_phase(msg, ShenandoahPhaseTimings::conc_cleanup_early, true /* log_heap_usage */);
 415   EventMark em("%s", msg);
 416 
 417   // This phase does not use workers, no need for setup
 418   heap->try_inject_alloc_failure();
 419   op_cleanup_early();
 420 }
 421 
 422 void ShenandoahConcurrentGC::entry_evacuate() {
 423   ShenandoahHeap* const heap = ShenandoahHeap::heap();
 424   TraceCollectorStats tcs(heap->monitoring_support()->concurrent_collection_counters());
 425 
 426   static const char* msg = "Concurrent evacuation";
 427   ShenandoahConcurrentPhase gc_phase(msg, ShenandoahPhaseTimings::conc_evac);
 428   EventMark em("%s", msg);
 429 
 430   ShenandoahWorkerScope scope(heap->workers(),
 431                               ShenandoahWorkerPolicy::calc_workers_for_conc_evac(),
 432                               "concurrent evacuation");
 433 
 434   heap->try_inject_alloc_failure();
 435   op_evacuate();
 436 }
 437 
 438 void ShenandoahConcurrentGC::entry_update_thread_roots() {
 439   ShenandoahHeap* const heap = ShenandoahHeap::heap();
 440   TraceCollectorStats tcs(heap->monitoring_support()->concurrent_collection_counters());
 441 
 442   static const char* msg = "Concurrent update thread roots";
 443   ShenandoahConcurrentPhase gc_phase(msg, ShenandoahPhaseTimings::conc_update_thread_roots);
 444   EventMark em("%s", msg);
 445 
 446   // No workers used in this phase, no setup required
 447   heap->try_inject_alloc_failure();
 448   op_update_thread_roots();
 449 }
 450 
 451 void ShenandoahConcurrentGC::entry_updaterefs() {
 452   ShenandoahHeap* const heap = ShenandoahHeap::heap();
 453   TraceCollectorStats tcs(heap->monitoring_support()->concurrent_collection_counters());
 454   static const char* msg = "Concurrent update references";
 455   ShenandoahConcurrentPhase gc_phase(msg, ShenandoahPhaseTimings::conc_update_refs);
 456   EventMark em("%s", msg);
 457 
 458   ShenandoahWorkerScope scope(heap->workers(),
 459                               ShenandoahWorkerPolicy::calc_workers_for_conc_update_ref(),
 460                               "concurrent reference update");
 461 
 462   heap->try_inject_alloc_failure();
 463   op_updaterefs();
 464 }
 465 
 466 void ShenandoahConcurrentGC::entry_cleanup_complete() {
 467   ShenandoahHeap* const heap = ShenandoahHeap::heap();
 468   TraceCollectorStats tcs(heap->monitoring_support()->concurrent_collection_counters());
 469   static const char* msg = "Concurrent cleanup";
 470   ShenandoahConcurrentPhase gc_phase(msg, ShenandoahPhaseTimings::conc_cleanup_complete, true /* log_heap_usage */);
 471   EventMark em("%s", msg);
 472 
 473   // This phase does not use workers, no need for setup
 474   heap->try_inject_alloc_failure();
 475   op_cleanup_complete();
 476 }
 477 
 478 void ShenandoahConcurrentGC::op_reset() {
 479   ShenandoahHeap* const heap = ShenandoahHeap::heap();
 480   if (ShenandoahPacing) {
 481     heap->pacer()->setup_for_reset();
 482   }
 483 
 484   heap->prepare_gc();
 485 }
 486 
 487 class ShenandoahInitMarkUpdateRegionStateClosure : public ShenandoahHeapRegionClosure {
 488 private:
 489   ShenandoahMarkingContext* const _ctx;
 490 public:
 491   ShenandoahInitMarkUpdateRegionStateClosure() : _ctx(ShenandoahHeap::heap()->marking_context()) {}
 492 
 493   void heap_region_do(ShenandoahHeapRegion* r) {
 494     assert(!r->has_live(), "Region " SIZE_FORMAT " should have no live data", r->index());
 495     if (r->is_active()) {
 496       // Check if region needs updating its TAMS. We have updated it already during concurrent
 497       // reset, so it is very likely we don't need to do another write here.
 498       if (_ctx->top_at_mark_start(r) != r->top()) {
 499         _ctx->capture_top_at_mark_start(r);
 500       }
 501     } else {
 502       assert(_ctx->top_at_mark_start(r) == r->top(),
 503              "Region " SIZE_FORMAT " should already have correct TAMS", r->index());
 504     }
 505   }
 506 
 507   bool is_thread_safe() { return true; }
 508 };
 509 
 510 void ShenandoahConcurrentGC::start_mark() {
 511   _mark.start_mark();
 512 }
 513 
 514 void ShenandoahConcurrentGC::op_init_mark() {
 515   ShenandoahHeap* const heap = ShenandoahHeap::heap();
 516   assert(ShenandoahSafepoint::is_at_shenandoah_safepoint(), "Should be at safepoint");
 517   assert(Thread::current()->is_VM_thread(), "can only do this in VMThread");
 518 
 519   assert(heap->marking_context()->is_bitmap_clear(), "need clear marking bitmap");
 520   assert(!heap->marking_context()->is_complete(), "should not be complete");
 521   assert(!heap->has_forwarded_objects(), "No forwarded objects on this path");
 522 
 523   if (ShenandoahVerify) {
 524     heap->verifier()->verify_before_concmark();
 525   }
 526 
 527   if (VerifyBeforeGC) {
 528     Universe::verify();
 529   }
 530 
 531   heap->set_concurrent_mark_in_progress(true);
 532 
 533   start_mark();
 534 
 535   {
 536     ShenandoahGCPhase phase(ShenandoahPhaseTimings::init_update_region_states);
 537     ShenandoahInitMarkUpdateRegionStateClosure cl;
 538     heap->parallel_heap_region_iterate(&cl);
 539   }
 540 
 541   // Weak reference processing
 542   ShenandoahReferenceProcessor* rp = heap->ref_processor();
 543   rp->reset_thread_locals();
 544   rp->set_soft_reference_policy(heap->soft_ref_policy()->should_clear_all_soft_refs());
 545 
 546   // Make above changes visible to worker threads
 547   OrderAccess::fence();
 548   // Arm nmethods for concurrent marking. When a nmethod is about to be executed,
 549   // we need to make sure that all its metadata are marked. alternative is to remark
 550   // thread roots at final mark pause, but it can be potential latency killer.
 551   if (heap->unload_classes()) {
 552     ShenandoahCodeRoots::arm_nmethods();
 553   }
 554 
 555   ShenandoahStackWatermark::change_epoch_id();
 556   if (ShenandoahPacing) {
 557     heap->pacer()->setup_for_mark();
 558   }
 559 }
 560 
 561 void ShenandoahConcurrentGC::op_mark_roots() {
 562   _mark.mark_concurrent_roots();
 563 }
 564 
 565 void ShenandoahConcurrentGC::op_mark() {
 566   _mark.concurrent_mark();
 567 }
 568 
 569 void ShenandoahConcurrentGC::op_final_mark() {
 570   ShenandoahHeap* const heap = ShenandoahHeap::heap();
 571   assert(ShenandoahSafepoint::is_at_shenandoah_safepoint(), "Should be at safepoint");
 572   assert(!heap->has_forwarded_objects(), "No forwarded objects on this path");
 573 
 574   if (ShenandoahVerify) {
 575     heap->verifier()->verify_roots_no_forwarded();
 576   }
 577 
 578   if (!heap->cancelled_gc()) {
 579     _mark.finish_mark();
 580     assert(!heap->cancelled_gc(), "STW mark cannot OOM");
 581 
 582     // Notify JVMTI that the tagmap table will need cleaning.
 583     JvmtiTagMap::set_needs_cleaning();
 584 
 585     heap->prepare_regions_and_collection_set(true /*concurrent*/);
 586 
 587     // Has to be done after cset selection
 588     heap->prepare_concurrent_roots();
 589 
 590     if (!heap->collection_set()->is_empty()) {
 591       if (ShenandoahVerify) {
 592         heap->verifier()->verify_before_evacuation();
 593       }
 594 
 595       heap->set_evacuation_in_progress(true);
 596       // From here on, we need to update references.
 597       heap->set_has_forwarded_objects(true);
 598 
 599       // Verify before arming for concurrent processing.
 600       // Otherwise, verification can trigger stack processing.
 601       if (ShenandoahVerify) {
 602         heap->verifier()->verify_during_evacuation();
 603       }
 604 
 605       // Arm nmethods/stack for concurrent processing
 606       ShenandoahCodeRoots::arm_nmethods();
 607       ShenandoahStackWatermark::change_epoch_id();
 608 
 609       // Notify JVMTI that oops are changed.
 610       JvmtiTagMap::set_needs_rehashing();
 611 
 612       if (ShenandoahPacing) {
 613         heap->pacer()->setup_for_evac();
 614       }
 615     } else {
 616       if (ShenandoahVerify) {
 617         heap->verifier()->verify_after_concmark();
 618       }
 619 
 620       if (VerifyAfterGC) {
 621         Universe::verify();
 622       }
 623     }
 624   }
 625 }
 626 
 627 class ShenandoahConcurrentEvacThreadClosure : public ThreadClosure {
 628 private:
 629   OopClosure* const _oops;
 630 
 631 public:
 632   ShenandoahConcurrentEvacThreadClosure(OopClosure* oops);
 633   void do_thread(Thread* thread);
 634 };
 635 
 636 ShenandoahConcurrentEvacThreadClosure::ShenandoahConcurrentEvacThreadClosure(OopClosure* oops) :
 637   _oops(oops) {
 638 }
 639 
 640 void ShenandoahConcurrentEvacThreadClosure::do_thread(Thread* thread) {
 641   JavaThread* const jt = JavaThread::cast(thread);
 642   StackWatermarkSet::finish_processing(jt, _oops, StackWatermarkKind::gc);
 643 }
 644 
 645 class ShenandoahConcurrentEvacUpdateThreadTask : public WorkerTask {
 646 private:
 647   ShenandoahJavaThreadsIterator _java_threads;
 648 
 649 public:
 650   ShenandoahConcurrentEvacUpdateThreadTask(uint n_workers) :
 651     WorkerTask("Shenandoah Evacuate/Update Concurrent Thread Roots"),
 652     _java_threads(ShenandoahPhaseTimings::conc_thread_roots, n_workers) {
 653   }
 654 
 655   void work(uint worker_id) {
 656     // ShenandoahEvacOOMScope has to be setup by ShenandoahContextEvacuateUpdateRootsClosure.
 657     // Otherwise, may deadlock with watermark lock
 658     ShenandoahContextEvacuateUpdateRootsClosure oops_cl;
 659     ShenandoahConcurrentEvacThreadClosure thr_cl(&oops_cl);
 660     _java_threads.threads_do(&thr_cl, worker_id);
 661   }
 662 };
 663 
 664 void ShenandoahConcurrentGC::op_thread_roots() {
 665   ShenandoahHeap* const heap = ShenandoahHeap::heap();
 666   assert(heap->is_evacuation_in_progress(), "Checked by caller");
 667   ShenandoahGCWorkerPhase worker_phase(ShenandoahPhaseTimings::conc_thread_roots);
 668   ShenandoahConcurrentEvacUpdateThreadTask task(heap->workers()->active_workers());
 669   heap->workers()->run_task(&task);
 670 }
 671 
 672 void ShenandoahConcurrentGC::op_weak_refs() {
 673   ShenandoahHeap* const heap = ShenandoahHeap::heap();
 674   assert(heap->is_concurrent_weak_root_in_progress(), "Only during this phase");
 675   // Concurrent weak refs processing
 676   ShenandoahGCWorkerPhase worker_phase(ShenandoahPhaseTimings::conc_weak_refs);
 677   if (heap->gc_cause() == GCCause::_wb_breakpoint) {
 678     ShenandoahBreakpoint::at_after_reference_processing_started();
 679   }
 680   heap->ref_processor()->process_references(ShenandoahPhaseTimings::conc_weak_refs, heap->workers(), true /* concurrent */);
 681 }
 682 
 683 class ShenandoahEvacUpdateCleanupOopStorageRootsClosure : public BasicOopIterateClosure {
 684 private:
 685   ShenandoahHeap* const _heap;
 686   ShenandoahMarkingContext* const _mark_context;
 687   bool  _evac_in_progress;
 688   Thread* const _thread;
 689 
 690 public:
 691   ShenandoahEvacUpdateCleanupOopStorageRootsClosure();
 692   void do_oop(oop* p);
 693   void do_oop(narrowOop* p);
 694 };
 695 
 696 ShenandoahEvacUpdateCleanupOopStorageRootsClosure::ShenandoahEvacUpdateCleanupOopStorageRootsClosure() :
 697   _heap(ShenandoahHeap::heap()),
 698   _mark_context(ShenandoahHeap::heap()->marking_context()),
 699   _evac_in_progress(ShenandoahHeap::heap()->is_evacuation_in_progress()),
 700   _thread(Thread::current()) {
 701 }
 702 
 703 void ShenandoahEvacUpdateCleanupOopStorageRootsClosure::do_oop(oop* p) {
 704   const oop obj = RawAccess<>::oop_load(p);
 705   if (!CompressedOops::is_null(obj)) {
 706     if (!_mark_context->is_marked(obj)) {
 707       // shenandoah_assert_correct(p, obj);
 708       ShenandoahHeap::atomic_clear_oop(p, obj);
 709     } else if (_evac_in_progress && _heap->in_collection_set(obj)) {
 710       oop resolved = ShenandoahBarrierSet::resolve_forwarded_not_null(obj);
 711       if (resolved == obj) {
 712         resolved = _heap->evacuate_object(obj, _thread);
 713       }
 714       ShenandoahHeap::atomic_update_oop(resolved, p, obj);
 715       assert(_heap->cancelled_gc() ||
 716              _mark_context->is_marked(resolved) && !_heap->in_collection_set(resolved),
 717              "Sanity");
 718     }
 719   }
 720 }
 721 
 722 void ShenandoahEvacUpdateCleanupOopStorageRootsClosure::do_oop(narrowOop* p) {
 723   ShouldNotReachHere();
 724 }
 725 
 726 class ShenandoahIsCLDAliveClosure : public CLDClosure {
 727 public:
 728   void do_cld(ClassLoaderData* cld) {
 729     cld->is_alive();
 730   }
 731 };
 732 
 733 class ShenandoahIsNMethodAliveClosure: public NMethodClosure {
 734 public:
 735   void do_nmethod(nmethod* n) {
 736     n->is_unloading();
 737   }
 738 };
 739 
 740 // This task not only evacuates/updates marked weak roots, but also "NULL"
 741 // dead weak roots.
 742 class ShenandoahConcurrentWeakRootsEvacUpdateTask : public WorkerTask {
 743 private:
 744   ShenandoahVMWeakRoots<true /*concurrent*/> _vm_roots;
 745 
 746   // Roots related to concurrent class unloading
 747   ShenandoahClassLoaderDataRoots<true /* concurrent */>
 748                                              _cld_roots;
 749   ShenandoahConcurrentNMethodIterator        _nmethod_itr;
 750   ShenandoahPhaseTimings::Phase              _phase;
 751 
 752 public:
 753   ShenandoahConcurrentWeakRootsEvacUpdateTask(ShenandoahPhaseTimings::Phase phase) :
 754     WorkerTask("Shenandoah Evacuate/Update Concurrent Weak Roots"),
 755     _vm_roots(phase),
 756     _cld_roots(phase, ShenandoahHeap::heap()->workers()->active_workers(), false /*heap iteration*/),
 757     _nmethod_itr(ShenandoahCodeRoots::table()),
 758     _phase(phase) {
 759     if (ShenandoahHeap::heap()->unload_classes()) {
 760       MutexLocker mu(CodeCache_lock, Mutex::_no_safepoint_check_flag);
 761       _nmethod_itr.nmethods_do_begin();
 762     }
 763   }
 764 
 765   ~ShenandoahConcurrentWeakRootsEvacUpdateTask() {
 766     if (ShenandoahHeap::heap()->unload_classes()) {
 767       MutexLocker mu(CodeCache_lock, Mutex::_no_safepoint_check_flag);
 768       _nmethod_itr.nmethods_do_end();
 769     }
 770     // Notify runtime data structures of potentially dead oops
 771     _vm_roots.report_num_dead();
 772   }
 773 
 774   void work(uint worker_id) {
 775     ShenandoahConcurrentWorkerSession worker_session(worker_id);
 776     {
 777       ShenandoahEvacOOMScope oom;
 778       // jni_roots and weak_roots are OopStorage backed roots, concurrent iteration
 779       // may race against OopStorage::release() calls.
 780       ShenandoahEvacUpdateCleanupOopStorageRootsClosure cl;
 781       _vm_roots.oops_do(&cl, worker_id);
 782     }
 783 
 784     // If we are going to perform concurrent class unloading later on, we need to
 785     // cleanup the weak oops in CLD and determinate nmethod's unloading state, so that we
 786     // can cleanup immediate garbage sooner.
 787     if (ShenandoahHeap::heap()->unload_classes()) {
 788       // Applies ShenandoahIsCLDAlive closure to CLDs, native barrier will either NULL the
 789       // CLD's holder or evacuate it.
 790       {
 791         ShenandoahIsCLDAliveClosure is_cld_alive;
 792         _cld_roots.cld_do(&is_cld_alive, worker_id);
 793       }
 794 
 795       // Applies ShenandoahIsNMethodAliveClosure to registered nmethods.
 796       // The closure calls nmethod->is_unloading(). The is_unloading
 797       // state is cached, therefore, during concurrent class unloading phase,
 798       // we will not touch the metadata of unloading nmethods
 799       {
 800         ShenandoahWorkerTimingsTracker timer(_phase, ShenandoahPhaseTimings::CodeCacheRoots, worker_id);
 801         ShenandoahIsNMethodAliveClosure is_nmethod_alive;
 802         _nmethod_itr.nmethods_do(&is_nmethod_alive);
 803       }
 804     }
 805   }
 806 };
 807 
 808 void ShenandoahConcurrentGC::op_weak_roots() {
 809   ShenandoahHeap* const heap = ShenandoahHeap::heap();
 810   assert(heap->is_concurrent_weak_root_in_progress(), "Only during this phase");
 811   // Concurrent weak root processing
 812   {
 813     ShenandoahTimingsTracker t(ShenandoahPhaseTimings::conc_weak_roots_work);
 814     ShenandoahGCWorkerPhase worker_phase(ShenandoahPhaseTimings::conc_weak_roots_work);
 815     ShenandoahConcurrentWeakRootsEvacUpdateTask task(ShenandoahPhaseTimings::conc_weak_roots_work);
 816     heap->workers()->run_task(&task);
 817   }
 818 
 819   // Perform handshake to flush out dead oops
 820   {
 821     ShenandoahTimingsTracker t(ShenandoahPhaseTimings::conc_weak_roots_rendezvous);
 822     heap->rendezvous_threads();
 823   }
 824 }
 825 
 826 void ShenandoahConcurrentGC::op_class_unloading() {
 827   ShenandoahHeap* const heap = ShenandoahHeap::heap();
 828   assert (heap->is_concurrent_weak_root_in_progress() &&
 829           heap->unload_classes(),
 830           "Checked by caller");
 831   heap->do_class_unloading();
 832 }
 833 
 834 class ShenandoahEvacUpdateCodeCacheClosure : public NMethodClosure {
 835 private:
 836   BarrierSetNMethod* const                  _bs;
 837   ShenandoahEvacuateUpdateMetadataClosure   _cl;
 838 
 839 public:
 840   ShenandoahEvacUpdateCodeCacheClosure() :
 841     _bs(BarrierSet::barrier_set()->barrier_set_nmethod()),
 842     _cl() {
 843   }
 844 
 845   void do_nmethod(nmethod* n) {
 846     ShenandoahNMethod* data = ShenandoahNMethod::gc_data(n);
 847     ShenandoahReentrantLocker locker(data->lock());
 848     // Setup EvacOOM scope below reentrant lock to avoid deadlock with
 849     // nmethod_entry_barrier
 850     ShenandoahEvacOOMScope oom;
 851     data->oops_do(&_cl, true/*fix relocation*/);
 852     _bs->disarm(n);
 853   }
 854 };
 855 
 856 class ShenandoahConcurrentRootsEvacUpdateTask : public WorkerTask {
 857 private:
 858   ShenandoahPhaseTimings::Phase                 _phase;
 859   ShenandoahVMRoots<true /*concurrent*/>        _vm_roots;
 860   ShenandoahClassLoaderDataRoots<true /*concurrent*/>
 861                                                 _cld_roots;
 862   ShenandoahConcurrentNMethodIterator           _nmethod_itr;
 863 
 864 public:
 865   ShenandoahConcurrentRootsEvacUpdateTask(ShenandoahPhaseTimings::Phase phase) :
 866     WorkerTask("Shenandoah Evacuate/Update Concurrent Strong Roots"),
 867     _phase(phase),
 868     _vm_roots(phase),
 869     _cld_roots(phase, ShenandoahHeap::heap()->workers()->active_workers(), false /*heap iteration*/),
 870     _nmethod_itr(ShenandoahCodeRoots::table()) {
 871     if (!ShenandoahHeap::heap()->unload_classes()) {
 872       MutexLocker mu(CodeCache_lock, Mutex::_no_safepoint_check_flag);
 873       _nmethod_itr.nmethods_do_begin();
 874     }
 875   }
 876 
 877   ~ShenandoahConcurrentRootsEvacUpdateTask() {
 878     if (!ShenandoahHeap::heap()->unload_classes()) {
 879       MutexLocker mu(CodeCache_lock, Mutex::_no_safepoint_check_flag);
 880       _nmethod_itr.nmethods_do_end();
 881     }
 882   }
 883 
 884   void work(uint worker_id) {
 885     ShenandoahConcurrentWorkerSession worker_session(worker_id);
 886     {
 887       ShenandoahEvacOOMScope oom;
 888       {
 889         // vm_roots and weak_roots are OopStorage backed roots, concurrent iteration
 890         // may race against OopStorage::release() calls.
 891         ShenandoahContextEvacuateUpdateRootsClosure cl;
 892         _vm_roots.oops_do<ShenandoahContextEvacuateUpdateRootsClosure>(&cl, worker_id);
 893       }
 894 
 895       {
 896         ShenandoahEvacuateUpdateMetadataClosure cl;
 897         CLDToOopClosure clds(&cl, ClassLoaderData::_claim_strong);
 898         _cld_roots.cld_do(&clds, worker_id);
 899       }
 900     }
 901 
 902     // Cannot setup ShenandoahEvacOOMScope here, due to potential deadlock with nmethod_entry_barrier.
 903     if (!ShenandoahHeap::heap()->unload_classes()) {
 904       ShenandoahWorkerTimingsTracker timer(_phase, ShenandoahPhaseTimings::CodeCacheRoots, worker_id);
 905       ShenandoahEvacUpdateCodeCacheClosure cl;
 906       _nmethod_itr.nmethods_do(&cl);
 907     }
 908   }
 909 };
 910 
 911 void ShenandoahConcurrentGC::op_strong_roots() {
 912   ShenandoahHeap* const heap = ShenandoahHeap::heap();
 913   assert(heap->is_concurrent_strong_root_in_progress(), "Checked by caller");
 914   ShenandoahConcurrentRootsEvacUpdateTask task(ShenandoahPhaseTimings::conc_strong_roots);
 915   heap->workers()->run_task(&task);
 916   heap->set_concurrent_strong_root_in_progress(false);
 917 }
 918 
 919 void ShenandoahConcurrentGC::op_cleanup_early() {
 920   ShenandoahHeap::heap()->free_set()->recycle_trash();
 921 }
 922 
 923 void ShenandoahConcurrentGC::op_evacuate() {
 924   ShenandoahHeap::heap()->evacuate_collection_set(true /*concurrent*/);
 925 }
 926 
 927 void ShenandoahConcurrentGC::op_init_updaterefs() {
 928   ShenandoahHeap* const heap = ShenandoahHeap::heap();
 929   heap->set_evacuation_in_progress(false);
 930   heap->set_concurrent_weak_root_in_progress(false);
 931   heap->prepare_update_heap_references(true /*concurrent*/);
 932   heap->set_update_refs_in_progress(true);
 933 
 934   if (ShenandoahPacing) {
 935     heap->pacer()->setup_for_updaterefs();
 936   }
 937 }
 938 
 939 void ShenandoahConcurrentGC::op_updaterefs() {
 940   ShenandoahHeap::heap()->update_heap_references(true /*concurrent*/);
 941 }
 942 
 943 class ShenandoahUpdateThreadClosure : public HandshakeClosure {
 944 private:
 945   ShenandoahUpdateRefsClosure _cl;
 946 public:
 947   ShenandoahUpdateThreadClosure();
 948   void do_thread(Thread* thread);
 949 };
 950 
 951 ShenandoahUpdateThreadClosure::ShenandoahUpdateThreadClosure() :
 952   HandshakeClosure("Shenandoah Update Thread Roots") {
 953 }
 954 
 955 void ShenandoahUpdateThreadClosure::do_thread(Thread* thread) {
 956   if (thread->is_Java_thread()) {
 957     JavaThread* jt = JavaThread::cast(thread);
 958     ResourceMark rm;
 959     jt->oops_do(&_cl, NULL);
 960   }
 961 }
 962 
 963 void ShenandoahConcurrentGC::op_update_thread_roots() {
 964   ShenandoahUpdateThreadClosure cl;
 965   Handshake::execute(&cl);
 966 }
 967 
 968 void ShenandoahConcurrentGC::op_final_updaterefs() {
 969   ShenandoahHeap* const heap = ShenandoahHeap::heap();
 970   assert(ShenandoahSafepoint::is_at_shenandoah_safepoint(), "must be at safepoint");
 971   assert(!heap->_update_refs_iterator.has_next(), "Should have finished update references");
 972 
 973   heap->finish_concurrent_roots();
 974 
 975   // Clear cancelled GC, if set. On cancellation path, the block before would handle
 976   // everything.
 977   if (heap->cancelled_gc()) {
 978     heap->clear_cancelled_gc();
 979   }
 980 
 981   // Has to be done before cset is clear
 982   if (ShenandoahVerify) {
 983     heap->verifier()->verify_roots_in_to_space();
 984   }
 985 
 986   heap->update_heap_region_states(true /*concurrent*/);
 987 
 988   heap->set_update_refs_in_progress(false);
 989   heap->set_has_forwarded_objects(false);
 990 
 991   if (ShenandoahVerify) {
 992     heap->verifier()->verify_after_updaterefs();
 993   }
 994 
 995   if (VerifyAfterGC) {
 996     Universe::verify();
 997   }
 998 
 999   heap->rebuild_free_set(true /*concurrent*/);
1000 }
1001 
1002 void ShenandoahConcurrentGC::op_final_roots() {
1003   ShenandoahHeap::heap()->set_concurrent_weak_root_in_progress(false);
1004 }
1005 
1006 void ShenandoahConcurrentGC::op_cleanup_complete() {
1007   ShenandoahHeap::heap()->free_set()->recycle_trash();
1008 }
1009 
1010 bool ShenandoahConcurrentGC::check_cancellation_and_abort(ShenandoahDegenPoint point) {
1011   if (ShenandoahHeap::heap()->cancelled_gc()) {
1012     _degen_point = point;
1013     return true;
1014   }
1015   return false;
1016 }
1017 
1018 const char* ShenandoahConcurrentGC::init_mark_event_message() const {
1019   ShenandoahHeap* const heap = ShenandoahHeap::heap();
1020   assert(!heap->has_forwarded_objects(), "Should not have forwarded objects here");
1021   if (heap->unload_classes()) {
1022     return "Pause Init Mark (unload classes)";
1023   } else {
1024     return "Pause Init Mark";
1025   }
1026 }
1027 
1028 const char* ShenandoahConcurrentGC::final_mark_event_message() const {
1029   ShenandoahHeap* const heap = ShenandoahHeap::heap();
1030   assert(!heap->has_forwarded_objects(), "Should not have forwarded objects here");
1031   if (heap->unload_classes()) {
1032     return "Pause Final Mark (unload classes)";
1033   } else {
1034     return "Pause Final Mark";
1035   }
1036 }
1037 
1038 const char* ShenandoahConcurrentGC::conc_mark_event_message() const {
1039   ShenandoahHeap* const heap = ShenandoahHeap::heap();
1040   assert(!heap->has_forwarded_objects(), "Should not have forwarded objects here");
1041   if (heap->unload_classes()) {
1042     return "Concurrent marking (unload classes)";
1043   } else {
1044     return "Concurrent marking";
1045   }
1046 }