1 /* 2 * Copyright (c) 2013, 2021, Red Hat, Inc. All rights reserved. 3 * Copyright (C) 2022 THL A29 Limited, a Tencent company. All rights reserved. 4 * Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. 5 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. 6 * 7 * This code is free software; you can redistribute it and/or modify it 8 * under the terms of the GNU General Public License version 2 only, as 9 * published by the Free Software Foundation. 10 * 11 * This code is distributed in the hope that it will be useful, but WITHOUT 12 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 13 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License 14 * version 2 for more details (a copy is included in the LICENSE file that 15 * accompanied this code). 16 * 17 * You should have received a copy of the GNU General Public License version 18 * 2 along with this work; if not, write to the Free Software Foundation, 19 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. 20 * 21 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA 22 * or visit www.oracle.com if you need additional information or have any 23 * questions. 24 * 25 */ 26 27 #include "precompiled.hpp" 28 #include "gc/shenandoah/shenandoahCollectorPolicy.hpp" 29 #include "gc/shenandoah/shenandoahConcurrentGC.hpp" 30 #include "gc/shenandoah/shenandoahControlThread.hpp" 31 #include "gc/shenandoah/shenandoahDegeneratedGC.hpp" 32 #include "gc/shenandoah/shenandoahEvacTracker.hpp" 33 #include "gc/shenandoah/shenandoahFreeSet.hpp" 34 #include "gc/shenandoah/shenandoahFullGC.hpp" 35 #include "gc/shenandoah/shenandoahGeneration.hpp" 36 #include "gc/shenandoah/shenandoahGlobalGeneration.hpp" 37 #include "gc/shenandoah/shenandoahYoungGeneration.hpp" 38 #include "gc/shenandoah/shenandoahOldGeneration.hpp" 39 #include "gc/shenandoah/shenandoahPhaseTimings.hpp" 40 #include "gc/shenandoah/shenandoahHeap.inline.hpp" 41 #include "gc/shenandoah/shenandoahMark.inline.hpp" 42 #include "gc/shenandoah/shenandoahMonitoringSupport.hpp" 43 #include "gc/shenandoah/shenandoahOopClosures.inline.hpp" 44 #include "gc/shenandoah/shenandoahOldGC.hpp" 45 #include "gc/shenandoah/shenandoahRootProcessor.inline.hpp" 46 #include "gc/shenandoah/shenandoahUtils.hpp" 47 #include "gc/shenandoah/shenandoahVMOperations.hpp" 48 #include "gc/shenandoah/shenandoahWorkerPolicy.hpp" 49 #include "gc/shenandoah/heuristics/shenandoahHeuristics.hpp" 50 #include "gc/shenandoah/mode/shenandoahMode.hpp" 51 #include "memory/iterator.hpp" 52 #include "memory/metaspaceUtils.hpp" 53 #include "memory/metaspaceStats.hpp" 54 #include "memory/universe.hpp" 55 #include "runtime/atomic.hpp" 56 57 ShenandoahControlThread::ShenandoahControlThread() : 58 ConcurrentGCThread(), 59 _alloc_failure_waiters_lock(Mutex::safepoint - 2, "ShenandoahAllocFailureGC_lock", true), 60 _gc_waiters_lock(Mutex::safepoint - 2, "ShenandoahRequestedGC_lock", true), 61 _control_lock(Mutex::nosafepoint - 2, "ShenandoahControlGC_lock", true), 62 _regulator_lock(Mutex::nosafepoint - 2, "ShenandoahRegulatorGC_lock", true), 63 _periodic_task(this), 64 _requested_gc_cause(GCCause::_no_cause_specified), 65 _requested_generation(select_global_generation()), 66 _degen_point(ShenandoahGC::_degenerated_outside_cycle), 67 _degen_generation(nullptr), 68 _allocs_seen(0), 69 _mode(none) { 70 set_name("Shenandoah Control Thread"); 71 reset_gc_id(); 72 create_and_start(); 73 _periodic_task.enroll(); 74 if (ShenandoahPacing) { 75 _periodic_pacer_notify_task.enroll(); 76 } 77 } 78 79 ShenandoahControlThread::~ShenandoahControlThread() { 80 // This is here so that super is called. 81 } 82 83 void ShenandoahPeriodicTask::task() { 84 _thread->handle_force_counters_update(); 85 _thread->handle_counters_update(); 86 } 87 88 void ShenandoahPeriodicPacerNotify::task() { 89 assert(ShenandoahPacing, "Should not be here otherwise"); 90 ShenandoahHeap::heap()->pacer()->notify_waiters(); 91 } 92 93 void ShenandoahControlThread::run_service() { 94 ShenandoahHeap* heap = ShenandoahHeap::heap(); 95 96 GCMode default_mode = concurrent_normal; 97 ShenandoahGenerationType generation = select_global_generation(); 98 GCCause::Cause default_cause = GCCause::_shenandoah_concurrent_gc; 99 100 double last_shrink_time = os::elapsedTime(); 101 uint age_period = 0; 102 103 // Shrink period avoids constantly polling regions for shrinking. 104 // Having a period 10x lower than the delay would mean we hit the 105 // shrinking with lag of less than 1/10-th of true delay. 106 // ShenandoahUncommitDelay is in msecs, but shrink_period is in seconds. 107 double shrink_period = (double)ShenandoahUncommitDelay / 1000 / 10; 108 109 ShenandoahCollectorPolicy* policy = heap->shenandoah_policy(); 110 111 // Heuristics are notified of allocation failures here and other outcomes 112 // of the cycle. They're also used here to control whether the Nth consecutive 113 // degenerated cycle should be 'promoted' to a full cycle. The decision to 114 // trigger a cycle or not is evaluated on the regulator thread. 115 ShenandoahHeuristics* global_heuristics = heap->global_generation()->heuristics(); 116 bool old_bootstrap_requested = false; 117 while (!in_graceful_shutdown() && !should_terminate()) { 118 // Figure out if we have pending requests. 119 bool alloc_failure_pending = _alloc_failure_gc.is_set(); 120 bool is_gc_requested = _gc_requested.is_set(); 121 GCCause::Cause requested_gc_cause = _requested_gc_cause; 122 bool explicit_gc_requested = is_gc_requested && is_explicit_gc(requested_gc_cause); 123 bool implicit_gc_requested = is_gc_requested && is_implicit_gc(requested_gc_cause); 124 125 // This control loop iteration have seen this much allocations. 126 size_t allocs_seen = Atomic::xchg(&_allocs_seen, (size_t)0, memory_order_relaxed); 127 128 // Check if we have seen a new target for soft max heap size. 129 bool soft_max_changed = check_soft_max_changed(); 130 131 // Choose which GC mode to run in. The block below should select a single mode. 132 set_gc_mode(none); 133 GCCause::Cause cause = GCCause::_last_gc_cause; 134 ShenandoahGC::ShenandoahDegenPoint degen_point = ShenandoahGC::_degenerated_unset; 135 136 if (alloc_failure_pending) { 137 // Allocation failure takes precedence: we have to deal with it first thing 138 log_info(gc)("Trigger: Handle Allocation Failure"); 139 140 cause = GCCause::_allocation_failure; 141 142 // Consume the degen point, and seed it with default value 143 degen_point = _degen_point; 144 _degen_point = ShenandoahGC::_degenerated_outside_cycle; 145 146 if (degen_point == ShenandoahGC::_degenerated_outside_cycle) { 147 _degen_generation = heap->mode()->is_generational() ? 148 heap->young_generation() : heap->global_generation(); 149 } else { 150 assert(_degen_generation != nullptr, "Need to know which generation to resume"); 151 } 152 153 ShenandoahHeuristics* heuristics = _degen_generation->heuristics(); 154 generation = _degen_generation->type(); 155 bool old_gen_evacuation_failed = heap->clear_old_evacuation_failure(); 156 157 // Do not bother with degenerated cycle if old generation evacuation failed 158 if (ShenandoahDegeneratedGC && heuristics->should_degenerate_cycle() && !old_gen_evacuation_failed) { 159 heuristics->record_allocation_failure_gc(); 160 policy->record_alloc_failure_to_degenerated(degen_point); 161 set_gc_mode(stw_degenerated); 162 } else { 163 heuristics->record_allocation_failure_gc(); 164 policy->record_alloc_failure_to_full(); 165 generation = select_global_generation(); 166 set_gc_mode(stw_full); 167 } 168 } else if (explicit_gc_requested) { 169 cause = requested_gc_cause; 170 generation = select_global_generation(); 171 log_info(gc)("Trigger: Explicit GC request (%s)", GCCause::to_string(cause)); 172 173 global_heuristics->record_requested_gc(); 174 175 if (ExplicitGCInvokesConcurrent) { 176 policy->record_explicit_to_concurrent(); 177 set_gc_mode(default_mode); 178 // Unload and clean up everything 179 heap->set_unload_classes(global_heuristics->can_unload_classes()); 180 } else { 181 policy->record_explicit_to_full(); 182 set_gc_mode(stw_full); 183 } 184 } else if (implicit_gc_requested) { 185 cause = requested_gc_cause; 186 generation = select_global_generation(); 187 log_info(gc)("Trigger: Implicit GC request (%s)", GCCause::to_string(cause)); 188 189 global_heuristics->record_requested_gc(); 190 191 if (ShenandoahImplicitGCInvokesConcurrent) { 192 policy->record_implicit_to_concurrent(); 193 set_gc_mode(default_mode); 194 195 // Unload and clean up everything 196 heap->set_unload_classes(global_heuristics->can_unload_classes()); 197 } else { 198 policy->record_implicit_to_full(); 199 set_gc_mode(stw_full); 200 } 201 } else { 202 // We should only be here if the regulator requested a cycle or if 203 // there is an old generation mark in progress. 204 if (_requested_gc_cause == GCCause::_shenandoah_concurrent_gc) { 205 if (_requested_generation == OLD && heap->doing_mixed_evacuations()) { 206 // If a request to start an old cycle arrived while an old cycle was running, but _before_ 207 // it chose any regions for evacuation we don't want to start a new old cycle. Rather, we want 208 // the heuristic to run a young collection so that we can evacuate some old regions. 209 assert(!heap->is_concurrent_old_mark_in_progress(), "Should not be running mixed collections and concurrent marking"); 210 generation = YOUNG; 211 } else if (_requested_generation == OLD && !old_bootstrap_requested) { 212 // Arrange to perform a young GC immediately followed by a bootstrap OLD GC. OLD GC typically requires more 213 // than twice the time required for YOUNG GC, so we run a YOUNG GC to replenish the YOUNG allocation pool before 214 // we start the longer OLD GC effort. 215 old_bootstrap_requested = true; 216 generation = YOUNG; 217 } else { 218 // if (old_bootstrap_requested && (_requested_generation == OLD)), this starts the bootstrap GC that 219 // immediately follows the preparatory young GC. 220 // But we will abandon the planned bootstrap GC if a GLOBAL GC has been now been requested. 221 generation = _requested_generation; 222 old_bootstrap_requested = false; 223 } 224 // preemption was requested or this is a regular cycle 225 cause = GCCause::_shenandoah_concurrent_gc; 226 set_gc_mode(default_mode); 227 228 // Don't start a new old marking if there is one already in progress 229 if (generation == OLD && heap->is_concurrent_old_mark_in_progress()) { 230 set_gc_mode(servicing_old); 231 } 232 233 if (generation == select_global_generation()) { 234 heap->set_unload_classes(global_heuristics->should_unload_classes()); 235 } else { 236 heap->set_unload_classes(false); 237 } 238 239 // Don't want to spin in this loop and start a cycle every time, so 240 // clear requested gc cause. This creates a race with callers of the 241 // blocking 'request_gc' method, but there it loops and resets the 242 // '_requested_gc_cause' until a full cycle is completed. 243 _requested_gc_cause = GCCause::_no_gc; 244 } else if (heap->is_concurrent_old_mark_in_progress() || heap->is_prepare_for_old_mark_in_progress()) { 245 // Nobody asked us to do anything, but we have an old-generation mark or old-generation preparation for 246 // mixed evacuation in progress, so resume working on that. 247 log_info(gc)("Resume old GC: marking is%s in progress, preparing is%s in progress", 248 heap->is_concurrent_old_mark_in_progress() ? "" : " NOT", 249 heap->is_prepare_for_old_mark_in_progress() ? "" : " NOT"); 250 251 cause = GCCause::_shenandoah_concurrent_gc; 252 generation = OLD; 253 set_gc_mode(servicing_old); 254 } 255 } 256 257 // Blow all soft references on this cycle, if handling allocation failure, 258 // either implicit or explicit GC request, or we are requested to do so unconditionally. 259 if (generation == select_global_generation() && (alloc_failure_pending || implicit_gc_requested || explicit_gc_requested || ShenandoahAlwaysClearSoftRefs)) { 260 heap->soft_ref_policy()->set_should_clear_all_soft_refs(true); 261 } 262 263 bool gc_requested = (_mode != none); 264 assert (!gc_requested || cause != GCCause::_last_gc_cause, "GC cause should be set"); 265 266 if (gc_requested) { 267 // GC is starting, bump the internal ID 268 update_gc_id(); 269 270 heap->reset_bytes_allocated_since_gc_start(); 271 272 MetaspaceCombinedStats meta_sizes = MetaspaceUtils::get_combined_statistics(); 273 274 // If GC was requested, we are sampling the counters even without actual triggers 275 // from allocation machinery. This captures GC phases more accurately. 276 set_forced_counters_update(true); 277 278 // If GC was requested, we better dump freeset data for performance debugging 279 { 280 ShenandoahHeapLocker locker(heap->lock()); 281 heap->free_set()->log_status(); 282 } 283 // In case this is a degenerated cycle, remember whether original cycle was aging. 284 bool was_aging_cycle = heap->is_aging_cycle(); 285 heap->set_aging_cycle(false); 286 287 switch (_mode) { 288 case concurrent_normal: { 289 // At this point: 290 // if (generation == YOUNG), this is a normal YOUNG cycle 291 // if (generation == OLD), this is a bootstrap OLD cycle 292 // if (generation == GLOBAL), this is a GLOBAL cycle triggered by System.gc() 293 // In all three cases, we want to age old objects if this is an aging cycle 294 if (age_period-- == 0) { 295 heap->set_aging_cycle(true); 296 age_period = ShenandoahAgingCyclePeriod - 1; 297 } 298 service_concurrent_normal_cycle(heap, generation, cause); 299 break; 300 } 301 case stw_degenerated: { 302 heap->set_aging_cycle(was_aging_cycle); 303 if (!service_stw_degenerated_cycle(cause, degen_point)) { 304 // The degenerated GC was upgraded to a Full GC 305 generation = select_global_generation(); 306 } 307 break; 308 } 309 case stw_full: { 310 if (age_period-- == 0) { 311 heap->set_aging_cycle(true); 312 age_period = ShenandoahAgingCyclePeriod - 1; 313 } 314 service_stw_full_cycle(cause); 315 break; 316 } 317 case servicing_old: { 318 assert(generation == OLD, "Expected old generation here"); 319 GCIdMark gc_id_mark; 320 service_concurrent_old_cycle(heap, cause); 321 break; 322 } 323 default: 324 ShouldNotReachHere(); 325 } 326 327 // If this was the requested GC cycle, notify waiters about it 328 if (explicit_gc_requested || implicit_gc_requested) { 329 notify_gc_waiters(); 330 } 331 332 // If this was the allocation failure GC cycle, notify waiters about it 333 if (alloc_failure_pending) { 334 notify_alloc_failure_waiters(); 335 } 336 337 // Report current free set state at the end of cycle, whether 338 // it is a normal completion, or the abort. 339 { 340 ShenandoahHeapLocker locker(heap->lock()); 341 heap->free_set()->log_status(); 342 343 // Notify Universe about new heap usage. This has implications for 344 // global soft refs policy, and we better report it every time heap 345 // usage goes down. 346 Universe::heap()->update_capacity_and_used_at_gc(); 347 348 // Signal that we have completed a visit to all live objects. 349 Universe::heap()->record_whole_heap_examined_timestamp(); 350 } 351 352 // Disable forced counters update, and update counters one more time 353 // to capture the state at the end of GC session. 354 handle_force_counters_update(); 355 set_forced_counters_update(false); 356 357 // Retract forceful part of soft refs policy 358 heap->soft_ref_policy()->set_should_clear_all_soft_refs(false); 359 360 // Clear metaspace oom flag, if current cycle unloaded classes 361 if (heap->unload_classes()) { 362 assert(generation == select_global_generation(), "Only unload classes during GLOBAL cycle"); 363 global_heuristics->clear_metaspace_oom(); 364 } 365 366 process_phase_timings(heap); 367 368 // Print Metaspace change following GC (if logging is enabled). 369 MetaspaceUtils::print_metaspace_change(meta_sizes); 370 371 // GC is over, we are at idle now 372 if (ShenandoahPacing) { 373 heap->pacer()->setup_for_idle(); 374 } 375 } else { 376 // Allow allocators to know we have seen this much regions 377 if (ShenandoahPacing && (allocs_seen > 0)) { 378 heap->pacer()->report_alloc(allocs_seen); 379 } 380 } 381 382 double current = os::elapsedTime(); 383 384 if (ShenandoahUncommit && (explicit_gc_requested || soft_max_changed || (current - last_shrink_time > shrink_period))) { 385 // Explicit GC tries to uncommit everything down to min capacity. 386 // Soft max change tries to uncommit everything down to target capacity. 387 // Periodic uncommit tries to uncommit suitable regions down to min capacity. 388 389 double shrink_before = (explicit_gc_requested || soft_max_changed) ? 390 current : 391 current - (ShenandoahUncommitDelay / 1000.0); 392 393 size_t shrink_until = soft_max_changed ? 394 heap->soft_max_capacity() : 395 heap->min_capacity(); 396 397 service_uncommit(shrink_before, shrink_until); 398 heap->phase_timings()->flush_cycle_to_global(); 399 last_shrink_time = current; 400 } 401 402 // Don't wait around if there was an allocation failure - start the next cycle immediately. 403 if (!is_alloc_failure_gc()) { 404 if (old_bootstrap_requested) { 405 _requested_generation = OLD; 406 _requested_gc_cause = GCCause::_shenandoah_concurrent_gc; 407 } else { 408 // The timed wait is necessary because this thread has a responsibility to send 409 // 'alloc_words' to the pacer when it does not perform a GC. 410 MonitorLocker lock(&_control_lock, Mutex::_no_safepoint_check_flag); 411 lock.wait(ShenandoahControlIntervalMax); 412 } 413 } else { 414 // in case of alloc_failure, abandon any plans to do immediate OLD Bootstrap 415 old_bootstrap_requested = false; 416 } 417 } 418 419 // Wait for the actual stop(), can't leave run_service() earlier. 420 while (!should_terminate()) { 421 os::naked_short_sleep(ShenandoahControlIntervalMin); 422 } 423 } 424 425 void ShenandoahControlThread::process_phase_timings(const ShenandoahHeap* heap) { 426 // Commit worker statistics to cycle data 427 heap->phase_timings()->flush_par_workers_to_cycle(); 428 if (ShenandoahPacing) { 429 heap->pacer()->flush_stats_to_cycle(); 430 } 431 432 ShenandoahEvacuationTracker* evac_tracker = heap->evac_tracker(); 433 ShenandoahCycleStats evac_stats = evac_tracker->flush_cycle_to_global(); 434 435 // Print GC stats for current cycle 436 { 437 LogTarget(Info, gc, stats) lt; 438 if (lt.is_enabled()) { 439 ResourceMark rm; 440 LogStream ls(lt); 441 heap->phase_timings()->print_cycle_on(&ls); 442 evac_tracker->print_evacuations_on(&ls, &evac_stats.workers, 443 &evac_stats.mutators); 444 if (ShenandoahPacing) { 445 heap->pacer()->print_cycle_on(&ls); 446 } 447 } 448 } 449 450 // Commit statistics to globals 451 heap->phase_timings()->flush_cycle_to_global(); 452 } 453 454 // Young and old concurrent cycles are initiated by the regulator. Implicit 455 // and explicit GC requests are handled by the controller thread and always 456 // run a global cycle (which is concurrent by default, but may be overridden 457 // by command line options). Old cycles always degenerate to a global cycle. 458 // Young cycles are degenerated to complete the young cycle. Young 459 // and old degen may upgrade to Full GC. Full GC may also be 460 // triggered directly by a System.gc() invocation. 461 // 462 // 463 // +-----+ Idle +-----+-----------+---------------------+ 464 // | + | | | 465 // | | | | | 466 // | | v | | 467 // | | Bootstrap Old +-- | ------------+ | 468 // | | + | | | 469 // | | | | | | 470 // | v v v v | 471 // | Resume Old <----------+ Young +--> Young Degen | 472 // | + + ^ + + | 473 // v | | | | | | 474 // Global <-+ | +----------------------------+ | | 475 // + | | | 476 // | v v | 477 // +---> Global Degen +--------------------> Full <----+ 478 // 479 void ShenandoahControlThread::service_concurrent_normal_cycle(ShenandoahHeap* heap, 480 const ShenandoahGenerationType generation, 481 GCCause::Cause cause) { 482 GCIdMark gc_id_mark; 483 ShenandoahGeneration* the_generation = nullptr; 484 switch (generation) { 485 case YOUNG: { 486 // Run a young cycle. This might or might not, have interrupted an ongoing 487 // concurrent mark in the old generation. We need to think about promotions 488 // in this case. Promoted objects should be above the TAMS in the old regions 489 // they end up in, but we have to be sure we don't promote into any regions 490 // that are in the cset. 491 log_info(gc, ergo)("Start GC cycle (YOUNG)"); 492 the_generation = heap->young_generation(); 493 service_concurrent_cycle(the_generation, cause, false); 494 break; 495 } 496 case OLD: { 497 log_info(gc, ergo)("Start GC cycle (OLD)"); 498 the_generation = heap->old_generation(); 499 service_concurrent_old_cycle(heap, cause); 500 break; 501 } 502 case GLOBAL_GEN: { 503 log_info(gc, ergo)("Start GC cycle (GLOBAL)"); 504 the_generation = heap->global_generation(); 505 service_concurrent_cycle(the_generation, cause, false); 506 break; 507 } 508 case GLOBAL_NON_GEN: { 509 log_info(gc, ergo)("Start GC cycle"); 510 the_generation = heap->global_generation(); 511 service_concurrent_cycle(the_generation, cause, false); 512 break; 513 } 514 default: 515 ShouldNotReachHere(); 516 } 517 } 518 519 void ShenandoahControlThread::service_concurrent_old_cycle(ShenandoahHeap* heap, GCCause::Cause &cause) { 520 ShenandoahOldGeneration* old_generation = heap->old_generation(); 521 ShenandoahYoungGeneration* young_generation = heap->young_generation(); 522 ShenandoahOldGeneration::State original_state = old_generation->state(); 523 524 TraceCollectorStats tcs(heap->monitoring_support()->concurrent_collection_counters()); 525 526 switch (original_state) { 527 case ShenandoahOldGeneration::WAITING_FOR_FILL: 528 case ShenandoahOldGeneration::IDLE: { 529 assert(!heap->is_concurrent_old_mark_in_progress(), "Old already in progress"); 530 assert(old_generation->task_queues()->is_empty(), "Old mark queues should be empty"); 531 } 532 case ShenandoahOldGeneration::FILLING: { 533 _allow_old_preemption.set(); 534 ShenandoahGCSession session(cause, old_generation); 535 old_generation->prepare_gc(); 536 _allow_old_preemption.unset(); 537 538 if (heap->is_prepare_for_old_mark_in_progress()) { 539 // Coalescing threads detected the cancellation request and aborted. Stay 540 // in this state so control thread may resume the coalescing work. 541 assert(old_generation->state() == ShenandoahOldGeneration::FILLING, "Prepare for mark should be in progress"); 542 assert(heap->cancelled_gc(), "Preparation for GC is not complete, expected cancellation"); 543 } 544 545 // Before bootstrapping begins, we must acknowledge any cancellation request. 546 // If the gc has not been cancelled, this does nothing. If it has been cancelled, 547 // this will clear the cancellation request and exit before starting the bootstrap 548 // phase. This will allow the young GC cycle to proceed normally. If we do not 549 // acknowledge the cancellation request, the subsequent young cycle will observe 550 // the request and essentially cancel itself. 551 if (check_cancellation_or_degen(ShenandoahGC::_degenerated_outside_cycle)) { 552 log_info(gc)("Preparation for old generation cycle was cancelled"); 553 return; 554 } 555 556 // Coalescing threads completed and nothing was cancelled. it is safe to transition 557 // to the bootstrapping state now. 558 old_generation->transition_to(ShenandoahOldGeneration::BOOTSTRAPPING); 559 } 560 case ShenandoahOldGeneration::BOOTSTRAPPING: { 561 // Configure the young generation's concurrent mark to put objects in 562 // old regions into the concurrent mark queues associated with the old 563 // generation. The young cycle will run as normal except that rather than 564 // ignore old references it will mark and enqueue them in the old concurrent 565 // task queues but it will not traverse them. 566 set_gc_mode(bootstrapping_old); 567 young_generation->set_old_gen_task_queues(old_generation->task_queues()); 568 ShenandoahGCSession session(cause, young_generation); 569 service_concurrent_cycle(heap, young_generation, cause, true); 570 process_phase_timings(heap); 571 if (heap->cancelled_gc()) { 572 // Young generation bootstrap cycle has failed. Concurrent mark for old generation 573 // is going to resume after degenerated bootstrap cycle completes. 574 log_info(gc)("Bootstrap cycle for old generation was cancelled"); 575 return; 576 } 577 578 // Reset the degenerated point. Normally this would happen at the top 579 // of the control loop, but here we have just completed a young cycle 580 // which has bootstrapped the old concurrent marking. 581 _degen_point = ShenandoahGC::_degenerated_outside_cycle; 582 583 // From here we will 'resume' the old concurrent mark. This will skip reset 584 // and init mark for the concurrent mark. All of that work will have been 585 // done by the bootstrapping young cycle. 586 set_gc_mode(servicing_old); 587 old_generation->transition_to(ShenandoahOldGeneration::MARKING); 588 } 589 case ShenandoahOldGeneration::MARKING: { 590 ShenandoahGCSession session(cause, old_generation); 591 bool marking_complete = resume_concurrent_old_cycle(old_generation, cause); 592 if (marking_complete) { 593 assert(old_generation->state() != ShenandoahOldGeneration::MARKING, "Should not still be marking"); 594 if (original_state == ShenandoahOldGeneration::MARKING) { 595 heap->mmu_tracker()->record_old_marking_increment(old_generation, GCId::current(), true, 596 heap->collection_set()->has_old_regions()); 597 heap->log_heap_status("At end of Concurrent Old Marking finishing increment"); 598 } 599 } else if (original_state == ShenandoahOldGeneration::MARKING) { 600 heap->mmu_tracker()->record_old_marking_increment(old_generation, GCId::current(), false, 601 heap->collection_set()->has_old_regions()); 602 heap->log_heap_status("At end of Concurrent Old Marking increment"); 603 } 604 break; 605 } 606 default: 607 fatal("Unexpected state for old GC: %s", ShenandoahOldGeneration::state_name(old_generation->state())); 608 } 609 } 610 611 bool ShenandoahControlThread::resume_concurrent_old_cycle(ShenandoahGeneration* generation, GCCause::Cause cause) { 612 assert(ShenandoahHeap::heap()->is_concurrent_old_mark_in_progress(), "Old mark should be in progress"); 613 log_debug(gc)("Resuming old generation with " UINT32_FORMAT " marking tasks queued", generation->task_queues()->tasks()); 614 615 ShenandoahHeap* heap = ShenandoahHeap::heap(); 616 617 // We can only tolerate being cancelled during concurrent marking or during preparation for mixed 618 // evacuation. This flag here (passed by reference) is used to control precisely where the regulator 619 // is allowed to cancel a GC. 620 ShenandoahOldGC gc(generation, _allow_old_preemption); 621 if (gc.collect(cause)) { 622 generation->record_success_concurrent(false); 623 } 624 625 if (heap->cancelled_gc()) { 626 // It's possible the gc cycle was cancelled after the last time 627 // the collection checked for cancellation. In which case, the 628 // old gc cycle is still completed, and we have to deal with this 629 // cancellation. We set the degeneration point to be outside 630 // the cycle because if this is an allocation failure, that is 631 // what must be done (there is no degenerated old cycle). If the 632 // cancellation was due to a heuristic wanting to start a young 633 // cycle, then we are not actually going to a degenerated cycle, 634 // so the degenerated point doesn't matter here. 635 check_cancellation_or_degen(ShenandoahGC::_degenerated_outside_cycle); 636 if (_requested_gc_cause == GCCause::_shenandoah_concurrent_gc) { 637 heap->shenandoah_policy()->record_interrupted_old(); 638 } 639 return false; 640 } 641 return true; 642 } 643 644 bool ShenandoahControlThread::check_soft_max_changed() const { 645 ShenandoahHeap* heap = ShenandoahHeap::heap(); 646 size_t new_soft_max = Atomic::load(&SoftMaxHeapSize); 647 size_t old_soft_max = heap->soft_max_capacity(); 648 if (new_soft_max != old_soft_max) { 649 new_soft_max = MAX2(heap->min_capacity(), new_soft_max); 650 new_soft_max = MIN2(heap->max_capacity(), new_soft_max); 651 if (new_soft_max != old_soft_max) { 652 log_info(gc)("Soft Max Heap Size: " SIZE_FORMAT "%s -> " SIZE_FORMAT "%s", 653 byte_size_in_proper_unit(old_soft_max), proper_unit_for_byte_size(old_soft_max), 654 byte_size_in_proper_unit(new_soft_max), proper_unit_for_byte_size(new_soft_max) 655 ); 656 heap->set_soft_max_capacity(new_soft_max); 657 return true; 658 } 659 } 660 return false; 661 } 662 663 void ShenandoahControlThread::service_concurrent_cycle(ShenandoahGeneration* generation, GCCause::Cause cause, bool do_old_gc_bootstrap) { 664 // Normal cycle goes via all concurrent phases. If allocation failure (af) happens during 665 // any of the concurrent phases, it first degrades to Degenerated GC and completes GC there. 666 // If second allocation failure happens during Degenerated GC cycle (for example, when GC 667 // tries to evac something and no memory is available), cycle degrades to Full GC. 668 // 669 // There are also a shortcut through the normal cycle: immediate garbage shortcut, when 670 // heuristics says there are no regions to compact, and all the collection comes from immediately 671 // reclaimable regions. 672 // 673 // ................................................................................................ 674 // 675 // (immediate garbage shortcut) Concurrent GC 676 // /-------------------------------------------\ 677 // | | 678 // | | 679 // | | 680 // | v 681 // [START] ----> Conc Mark ----o----> Conc Evac --o--> Conc Update-Refs ---o----> [END] 682 // | | | ^ 683 // | (af) | (af) | (af) | 684 // ..................|....................|.................|..............|....................... 685 // | | | | 686 // | | | | Degenerated GC 687 // v v v | 688 // STW Mark ----------> STW Evac ----> STW Update-Refs ----->o 689 // | | | ^ 690 // | (af) | (af) | (af) | 691 // ..................|....................|.................|..............|....................... 692 // | | | | 693 // | v | | Full GC 694 // \------------------->o<----------------/ | 695 // | | 696 // v | 697 // Full GC --------------------------/ 698 // 699 if (check_cancellation_or_degen(ShenandoahGC::_degenerated_outside_cycle)) return; 700 701 ShenandoahHeap* heap = ShenandoahHeap::heap(); 702 ShenandoahGCSession session(cause, generation); 703 TraceCollectorStats tcs(heap->monitoring_support()->concurrent_collection_counters()); 704 705 service_concurrent_cycle(heap, generation, cause, do_old_gc_bootstrap); 706 } 707 708 void ShenandoahControlThread::service_concurrent_cycle(ShenandoahHeap* heap, 709 ShenandoahGeneration* generation, 710 GCCause::Cause& cause, 711 bool do_old_gc_bootstrap) { 712 ShenandoahConcurrentGC gc(generation, do_old_gc_bootstrap); 713 if (gc.collect(cause)) { 714 // Cycle is complete 715 generation->record_success_concurrent(gc.abbreviated()); 716 } else { 717 assert(heap->cancelled_gc(), "Must have been cancelled"); 718 check_cancellation_or_degen(gc.degen_point()); 719 assert(!generation->is_old(), "Old GC takes a different control path"); 720 // Concurrent young-gen collection degenerates to young 721 // collection. Same for global collections. 722 _degen_generation = generation; 723 } 724 const char* msg; 725 if (heap->mode()->is_generational()) { 726 ShenandoahMmuTracker* mmu_tracker = heap->mmu_tracker(); 727 if (generation->is_young()) { 728 if (heap->cancelled_gc()) { 729 msg = (do_old_gc_bootstrap) ? "At end of Interrupted Concurrent Bootstrap GC": 730 "At end of Interrupted Concurrent Young GC"; 731 } else { 732 // We only record GC results if GC was successful 733 msg = (do_old_gc_bootstrap) ? "At end of Concurrent Bootstrap GC": 734 "At end of Concurrent Young GC"; 735 if (heap->collection_set()->has_old_regions()) { 736 bool mixed_is_done = (heap->old_heuristics()->unprocessed_old_collection_candidates() == 0); 737 mmu_tracker->record_mixed(generation, get_gc_id(), mixed_is_done); 738 } else if (do_old_gc_bootstrap) { 739 mmu_tracker->record_bootstrap(generation, get_gc_id(), heap->collection_set()->has_old_regions()); 740 } else { 741 mmu_tracker->record_young(generation, get_gc_id()); 742 } 743 } 744 } else { 745 assert(generation->is_global(), "If not young, must be GLOBAL"); 746 assert(!do_old_gc_bootstrap, "Do not bootstrap with GLOBAL GC"); 747 if (heap->cancelled_gc()) { 748 msg = "At end of Interrupted Concurrent GLOBAL GC"; 749 } else { 750 // We only record GC results if GC was successful 751 msg = "At end of Concurrent Global GC"; 752 mmu_tracker->record_global(generation, get_gc_id()); 753 } 754 } 755 } else { 756 msg = heap->cancelled_gc() ? "At end of cancelled GC" : 757 "At end of GC"; 758 } 759 heap->log_heap_status(msg); 760 } 761 762 bool ShenandoahControlThread::check_cancellation_or_degen(ShenandoahGC::ShenandoahDegenPoint point) { 763 ShenandoahHeap* heap = ShenandoahHeap::heap(); 764 if (!heap->cancelled_gc()) { 765 return false; 766 } 767 768 if (in_graceful_shutdown()) { 769 return true; 770 } 771 772 assert(_degen_point == ShenandoahGC::_degenerated_outside_cycle, 773 "Should not be set yet: %s", ShenandoahGC::degen_point_to_string(_degen_point)); 774 775 if (is_alloc_failure_gc()) { 776 _degen_point = point; 777 return true; 778 } 779 780 if (_preemption_requested.is_set()) { 781 assert(_requested_generation == YOUNG, "Only young GCs may preempt old."); 782 _preemption_requested.unset(); 783 784 // Old generation marking is only cancellable during concurrent marking. 785 // Once final mark is complete, the code does not check again for cancellation. 786 // If old generation was cancelled for an allocation failure, we wouldn't 787 // make it to this case. The calling code is responsible for forcing a 788 // cancellation due to allocation failure into a degenerated cycle. 789 _degen_point = point; 790 heap->clear_cancelled_gc(false /* clear oom handler */); 791 return true; 792 } 793 794 fatal("Cancel GC either for alloc failure GC, or gracefully exiting, or to pause old generation marking"); 795 return false; 796 } 797 798 void ShenandoahControlThread::stop_service() { 799 // Nothing to do here. 800 } 801 802 void ShenandoahControlThread::service_stw_full_cycle(GCCause::Cause cause) { 803 ShenandoahHeap* const heap = ShenandoahHeap::heap(); 804 805 GCIdMark gc_id_mark; 806 ShenandoahGCSession session(cause, heap->global_generation()); 807 808 ShenandoahFullGC gc; 809 gc.collect(cause); 810 811 heap->global_generation()->heuristics()->record_success_full(); 812 heap->shenandoah_policy()->record_success_full(); 813 } 814 815 bool ShenandoahControlThread::service_stw_degenerated_cycle(GCCause::Cause cause, 816 ShenandoahGC::ShenandoahDegenPoint point) { 817 assert(point != ShenandoahGC::_degenerated_unset, "Degenerated point should be set"); 818 ShenandoahHeap* const heap = ShenandoahHeap::heap(); 819 820 GCIdMark gc_id_mark; 821 ShenandoahGCSession session(cause, _degen_generation); 822 823 ShenandoahDegenGC gc(point, _degen_generation); 824 gc.collect(cause); 825 826 assert(heap->young_generation()->task_queues()->is_empty(), "Unexpected young generation marking tasks"); 827 if (_degen_generation->is_global()) { 828 assert(heap->old_generation()->task_queues()->is_empty(), "Unexpected old generation marking tasks"); 829 assert(heap->global_generation()->task_queues()->is_empty(), "Unexpected global generation marking tasks"); 830 } else { 831 assert(_degen_generation->is_young(), "Expected degenerated young cycle, if not global."); 832 ShenandoahOldGeneration* old = heap->old_generation(); 833 if (old->state() == ShenandoahOldGeneration::BOOTSTRAPPING && !gc.upgraded_to_full()) { 834 old->transition_to(ShenandoahOldGeneration::MARKING); 835 } 836 } 837 838 _degen_generation->heuristics()->record_success_degenerated(); 839 heap->shenandoah_policy()->record_success_degenerated(); 840 return !gc.upgraded_to_full(); 841 } 842 843 void ShenandoahControlThread::service_uncommit(double shrink_before, size_t shrink_until) { 844 ShenandoahHeap* heap = ShenandoahHeap::heap(); 845 846 // Determine if there is work to do. This avoids taking heap lock if there is 847 // no work available, avoids spamming logs with superfluous logging messages, 848 // and minimises the amount of work while locks are taken. 849 850 if (heap->committed() <= shrink_until) return; 851 852 bool has_work = false; 853 for (size_t i = 0; i < heap->num_regions(); i++) { 854 ShenandoahHeapRegion *r = heap->get_region(i); 855 if (r->is_empty_committed() && (r->empty_time() < shrink_before)) { 856 has_work = true; 857 break; 858 } 859 } 860 861 if (has_work) { 862 heap->entry_uncommit(shrink_before, shrink_until); 863 } 864 } 865 866 bool ShenandoahControlThread::is_explicit_gc(GCCause::Cause cause) const { 867 return GCCause::is_user_requested_gc(cause) || 868 GCCause::is_serviceability_requested_gc(cause); 869 } 870 871 bool ShenandoahControlThread::is_implicit_gc(GCCause::Cause cause) const { 872 return !is_explicit_gc(cause) && 873 (cause != GCCause::_shenandoah_concurrent_gc); 874 } 875 876 void ShenandoahControlThread::request_gc(GCCause::Cause cause) { 877 assert(GCCause::is_user_requested_gc(cause) || 878 GCCause::is_serviceability_requested_gc(cause) || 879 cause == GCCause::_metadata_GC_clear_soft_refs || 880 cause == GCCause::_codecache_GC_aggressive || 881 cause == GCCause::_codecache_GC_threshold || 882 cause == GCCause::_full_gc_alot || 883 cause == GCCause::_wb_young_gc || 884 cause == GCCause::_wb_full_gc || 885 cause == GCCause::_wb_breakpoint || 886 cause == GCCause::_scavenge_alot, 887 "only requested GCs here: %s", GCCause::to_string(cause)); 888 889 if (is_explicit_gc(cause)) { 890 if (!DisableExplicitGC) { 891 handle_requested_gc(cause); 892 } 893 } else { 894 handle_requested_gc(cause); 895 } 896 } 897 898 bool ShenandoahControlThread::request_concurrent_gc(ShenandoahGenerationType generation) { 899 if (_preemption_requested.is_set() || _gc_requested.is_set() || ShenandoahHeap::heap()->cancelled_gc()) { 900 // Ignore subsequent requests from the heuristics 901 return false; 902 } 903 904 if (_mode == none) { 905 _requested_gc_cause = GCCause::_shenandoah_concurrent_gc; 906 _requested_generation = generation; 907 notify_control_thread(); 908 MonitorLocker ml(&_regulator_lock, Mutex::_no_safepoint_check_flag); 909 ml.wait(); 910 return true; 911 } 912 913 if (preempt_old_marking(generation)) { 914 log_info(gc)("Preempting old generation mark to allow %s GC", shenandoah_generation_name(generation)); 915 _requested_gc_cause = GCCause::_shenandoah_concurrent_gc; 916 _requested_generation = generation; 917 _preemption_requested.set(); 918 ShenandoahHeap::heap()->cancel_gc(GCCause::_shenandoah_concurrent_gc); 919 notify_control_thread(); 920 921 MonitorLocker ml(&_regulator_lock, Mutex::_no_safepoint_check_flag); 922 ml.wait(); 923 return true; 924 } 925 926 return false; 927 } 928 929 void ShenandoahControlThread::notify_control_thread() { 930 MonitorLocker locker(&_control_lock, Mutex::_no_safepoint_check_flag); 931 _control_lock.notify(); 932 } 933 934 bool ShenandoahControlThread::preempt_old_marking(ShenandoahGenerationType generation) { 935 return (generation == YOUNG) && _allow_old_preemption.try_unset(); 936 } 937 938 void ShenandoahControlThread::handle_requested_gc(GCCause::Cause cause) { 939 // Make sure we have at least one complete GC cycle before unblocking 940 // from the explicit GC request. 941 // 942 // This is especially important for weak references cleanup and/or native 943 // resources (e.g. DirectByteBuffers) machinery: when explicit GC request 944 // comes very late in the already running cycle, it would miss lots of new 945 // opportunities for cleanup that were made available before the caller 946 // requested the GC. 947 948 MonitorLocker ml(&_gc_waiters_lock); 949 size_t current_gc_id = get_gc_id(); 950 size_t required_gc_id = current_gc_id + 1; 951 while (current_gc_id < required_gc_id) { 952 // Although setting gc request is under _gc_waiters_lock, but read side (run_service()) 953 // does not take the lock. We need to enforce following order, so that read side sees 954 // latest requested gc cause when the flag is set. 955 _requested_gc_cause = cause; 956 _gc_requested.set(); 957 notify_control_thread(); 958 if (cause != GCCause::_wb_breakpoint) { 959 ml.wait(); 960 } 961 current_gc_id = get_gc_id(); 962 } 963 } 964 965 void ShenandoahControlThread::handle_alloc_failure(ShenandoahAllocRequest& req) { 966 ShenandoahHeap* heap = ShenandoahHeap::heap(); 967 968 assert(current()->is_Java_thread(), "expect Java thread here"); 969 970 if (try_set_alloc_failure_gc()) { 971 // Only report the first allocation failure 972 log_info(gc)("Failed to allocate %s, " SIZE_FORMAT "%s", 973 req.type_string(), 974 byte_size_in_proper_unit(req.size() * HeapWordSize), proper_unit_for_byte_size(req.size() * HeapWordSize)); 975 // Now that alloc failure GC is scheduled, we can abort everything else 976 heap->cancel_gc(GCCause::_allocation_failure); 977 } 978 979 MonitorLocker ml(&_alloc_failure_waiters_lock); 980 while (is_alloc_failure_gc()) { 981 ml.wait(); 982 } 983 } 984 985 void ShenandoahControlThread::handle_alloc_failure_evac(size_t words) { 986 ShenandoahHeap* heap = ShenandoahHeap::heap(); 987 988 if (try_set_alloc_failure_gc()) { 989 // Only report the first allocation failure 990 log_info(gc)("Failed to allocate " SIZE_FORMAT "%s for evacuation", 991 byte_size_in_proper_unit(words * HeapWordSize), proper_unit_for_byte_size(words * HeapWordSize)); 992 } 993 994 // Forcefully report allocation failure 995 heap->cancel_gc(GCCause::_shenandoah_allocation_failure_evac); 996 } 997 998 void ShenandoahControlThread::notify_alloc_failure_waiters() { 999 _alloc_failure_gc.unset(); 1000 MonitorLocker ml(&_alloc_failure_waiters_lock); 1001 ml.notify_all(); 1002 } 1003 1004 bool ShenandoahControlThread::try_set_alloc_failure_gc() { 1005 return _alloc_failure_gc.try_set(); 1006 } 1007 1008 bool ShenandoahControlThread::is_alloc_failure_gc() { 1009 return _alloc_failure_gc.is_set(); 1010 } 1011 1012 void ShenandoahControlThread::notify_gc_waiters() { 1013 _gc_requested.unset(); 1014 MonitorLocker ml(&_gc_waiters_lock); 1015 ml.notify_all(); 1016 } 1017 1018 void ShenandoahControlThread::handle_counters_update() { 1019 if (_do_counters_update.is_set()) { 1020 _do_counters_update.unset(); 1021 ShenandoahHeap::heap()->monitoring_support()->update_counters(); 1022 } 1023 } 1024 1025 void ShenandoahControlThread::handle_force_counters_update() { 1026 if (_force_counters_update.is_set()) { 1027 _do_counters_update.unset(); // reset these too, we do update now! 1028 ShenandoahHeap::heap()->monitoring_support()->update_counters(); 1029 } 1030 } 1031 1032 void ShenandoahControlThread::notify_heap_changed() { 1033 // This is called from allocation path, and thus should be fast. 1034 1035 // Update monitoring counters when we took a new region. This amortizes the 1036 // update costs on slow path. 1037 if (_do_counters_update.is_unset()) { 1038 _do_counters_update.set(); 1039 } 1040 } 1041 1042 void ShenandoahControlThread::pacing_notify_alloc(size_t words) { 1043 assert(ShenandoahPacing, "should only call when pacing is enabled"); 1044 Atomic::add(&_allocs_seen, words, memory_order_relaxed); 1045 } 1046 1047 void ShenandoahControlThread::set_forced_counters_update(bool value) { 1048 _force_counters_update.set_cond(value); 1049 } 1050 1051 void ShenandoahControlThread::reset_gc_id() { 1052 Atomic::store(&_gc_id, (size_t)0); 1053 } 1054 1055 void ShenandoahControlThread::update_gc_id() { 1056 Atomic::inc(&_gc_id); 1057 } 1058 1059 size_t ShenandoahControlThread::get_gc_id() { 1060 return Atomic::load(&_gc_id); 1061 } 1062 1063 void ShenandoahControlThread::start() { 1064 create_and_start(); 1065 } 1066 1067 void ShenandoahControlThread::prepare_for_graceful_shutdown() { 1068 _graceful_shutdown.set(); 1069 } 1070 1071 bool ShenandoahControlThread::in_graceful_shutdown() { 1072 return _graceful_shutdown.is_set(); 1073 } 1074 1075 const char* ShenandoahControlThread::gc_mode_name(ShenandoahControlThread::GCMode mode) { 1076 switch (mode) { 1077 case none: return "idle"; 1078 case concurrent_normal: return "normal"; 1079 case stw_degenerated: return "degenerated"; 1080 case stw_full: return "full"; 1081 case servicing_old: return "old"; 1082 case bootstrapping_old: return "bootstrap"; 1083 default: return "unknown"; 1084 } 1085 } 1086 1087 void ShenandoahControlThread::set_gc_mode(ShenandoahControlThread::GCMode new_mode) { 1088 if (_mode != new_mode) { 1089 log_info(gc)("Transition from: %s to: %s", gc_mode_name(_mode), gc_mode_name(new_mode)); 1090 _mode = new_mode; 1091 MonitorLocker ml(&_regulator_lock, Mutex::_no_safepoint_check_flag); 1092 ml.notify_all(); 1093 } 1094 } 1095 1096 ShenandoahGenerationType ShenandoahControlThread::select_global_generation() { 1097 if (ShenandoahHeap::heap()->mode()->is_generational()) { 1098 return GLOBAL_GEN; 1099 } else { 1100 return GLOBAL_NON_GEN; 1101 } 1102 }