1 /*
2 * Copyright (c) 2013, 2021, Red Hat, Inc. All rights reserved.
3 * Copyright (C) 2022 THL A29 Limited, a Tencent company. All rights reserved.
4 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
5 *
6 * This code is free software; you can redistribute it and/or modify it
7 * under the terms of the GNU General Public License version 2 only, as
8 * published by the Free Software Foundation.
9 *
10 * This code is distributed in the hope that it will be useful, but WITHOUT
11 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
12 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
13 * version 2 for more details (a copy is included in the LICENSE file that
14 * accompanied this code).
15 *
16 * You should have received a copy of the GNU General Public License version
17 * 2 along with this work; if not, write to the Free Software Foundation,
18 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
19 *
20 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
21 * or visit www.oracle.com if you need additional information or have any
22 * questions.
23 *
24 */
25
26 #include "precompiled.hpp"
27 #include "gc/shenandoah/shenandoahCollectorPolicy.hpp"
28 #include "gc/shenandoah/shenandoahConcurrentGC.hpp"
29 #include "gc/shenandoah/shenandoahControlThread.hpp"
30 #include "gc/shenandoah/shenandoahDegeneratedGC.hpp"
31 #include "gc/shenandoah/shenandoahEvacTracker.hpp"
32 #include "gc/shenandoah/shenandoahFreeSet.hpp"
33 #include "gc/shenandoah/shenandoahFullGC.hpp"
34 #include "gc/shenandoah/shenandoahGeneration.hpp"
35 #include "gc/shenandoah/shenandoahYoungGeneration.hpp"
36 #include "gc/shenandoah/shenandoahOldGeneration.hpp"
37 #include "gc/shenandoah/shenandoahPhaseTimings.hpp"
38 #include "gc/shenandoah/shenandoahHeap.inline.hpp"
39 #include "gc/shenandoah/shenandoahMark.inline.hpp"
40 #include "gc/shenandoah/shenandoahMonitoringSupport.hpp"
41 #include "gc/shenandoah/shenandoahOopClosures.inline.hpp"
42 #include "gc/shenandoah/shenandoahOldGC.hpp"
43 #include "gc/shenandoah/shenandoahRootProcessor.inline.hpp"
44 #include "gc/shenandoah/shenandoahUtils.hpp"
45 #include "gc/shenandoah/shenandoahVMOperations.hpp"
46 #include "gc/shenandoah/shenandoahWorkerPolicy.hpp"
47 #include "gc/shenandoah/heuristics/shenandoahHeuristics.hpp"
48 #include "gc/shenandoah/mode/shenandoahMode.hpp"
49 #include "memory/iterator.hpp"
50 #include "memory/metaspaceUtils.hpp"
51 #include "memory/metaspaceStats.hpp"
52 #include "memory/universe.hpp"
53 #include "runtime/atomic.hpp"
54
55 ShenandoahControlThread::ShenandoahControlThread() :
56 ConcurrentGCThread(),
57 _alloc_failure_waiters_lock(Mutex::safepoint - 2, "ShenandoahAllocFailureGC_lock", true),
58 _gc_waiters_lock(Mutex::safepoint - 2, "ShenandoahRequestedGC_lock", true),
59 _control_lock(Mutex::nosafepoint - 2, "ShenandoahControlGC_lock", true),
60 _regulator_lock(Mutex::nosafepoint - 2, "ShenandoahRegulatorGC_lock", true),
61 _periodic_task(this),
62 _requested_gc_cause(GCCause::_no_cause_specified),
63 _requested_generation(GenerationMode::GLOBAL),
64 _degen_point(ShenandoahGC::_degenerated_outside_cycle),
65 _degen_generation(nullptr),
66 _allocs_seen(0),
67 _mode(none) {
68 set_name("Shenandoah Control Thread");
69 reset_gc_id();
70 create_and_start();
71 _periodic_task.enroll();
72 if (ShenandoahPacing) {
73 _periodic_pacer_notify_task.enroll();
74 }
75 }
76
77 ShenandoahControlThread::~ShenandoahControlThread() {
78 // This is here so that super is called.
79 }
80
81 void ShenandoahPeriodicTask::task() {
82 _thread->handle_force_counters_update();
83 _thread->handle_counters_update();
84 }
85
86 void ShenandoahPeriodicPacerNotify::task() {
87 assert(ShenandoahPacing, "Should not be here otherwise");
88 ShenandoahHeap::heap()->pacer()->notify_waiters();
89 }
90
91 void ShenandoahControlThread::run_service() {
92 ShenandoahHeap* heap = ShenandoahHeap::heap();
93
94 GCMode default_mode = concurrent_normal;
95 GenerationMode generation = GLOBAL;
96 GCCause::Cause default_cause = GCCause::_shenandoah_concurrent_gc;
97
98 double last_shrink_time = os::elapsedTime();
99 uint age_period = 0;
100
101 // Shrink period avoids constantly polling regions for shrinking.
102 // Having a period 10x lower than the delay would mean we hit the
103 // shrinking with lag of less than 1/10-th of true delay.
104 // ShenandoahUncommitDelay is in msecs, but shrink_period is in seconds.
105 double shrink_period = (double)ShenandoahUncommitDelay / 1000 / 10;
106
107 ShenandoahCollectorPolicy* policy = heap->shenandoah_policy();
108
109 // Heuristics are notified of allocation failures here and other outcomes
110 // of the cycle. They're also used here to control whether the Nth consecutive
111 // degenerated cycle should be 'promoted' to a full cycle. The decision to
112 // trigger a cycle or not is evaluated on the regulator thread.
113 ShenandoahHeuristics* global_heuristics = heap->global_generation()->heuristics();
114 while (!in_graceful_shutdown() && !should_terminate()) {
115 // Figure out if we have pending requests.
116 bool alloc_failure_pending = _alloc_failure_gc.is_set();
117 bool is_gc_requested = _gc_requested.is_set();
118 GCCause::Cause requested_gc_cause = _requested_gc_cause;
119 bool explicit_gc_requested = is_gc_requested && is_explicit_gc(requested_gc_cause);
120 bool implicit_gc_requested = is_gc_requested && is_implicit_gc(requested_gc_cause);
121
122 // This control loop iteration have seen this much allocations.
123 size_t allocs_seen = Atomic::xchg(&_allocs_seen, (size_t)0, memory_order_relaxed);
124
125 // Check if we have seen a new target for soft max heap size.
126 bool soft_max_changed = check_soft_max_changed();
127
128 // Choose which GC mode to run in. The block below should select a single mode.
129 set_gc_mode(none);
130 GCCause::Cause cause = GCCause::_last_gc_cause;
131 ShenandoahGC::ShenandoahDegenPoint degen_point = ShenandoahGC::_degenerated_unset;
132
133 if (alloc_failure_pending) {
134 // Allocation failure takes precedence: we have to deal with it first thing
135 log_info(gc)("Trigger: Handle Allocation Failure");
136
137 cause = GCCause::_allocation_failure;
138
139 // Consume the degen point, and seed it with default value
140 degen_point = _degen_point;
141 _degen_point = ShenandoahGC::_degenerated_outside_cycle;
142
143 if (degen_point == ShenandoahGC::_degenerated_outside_cycle) {
144 _degen_generation = heap->mode()->is_generational() ? heap->young_generation() : heap->global_generation();
145 } else {
146 assert(_degen_generation != nullptr, "Need to know which generation to resume.");
147 }
148
149 ShenandoahHeuristics* heuristics = _degen_generation->heuristics();
150 generation = _degen_generation->generation_mode();
151 bool old_gen_evacuation_failed = heap->clear_old_evacuation_failure();
152
153 // Do not bother with degenerated cycle if old generation evacuation failed.
154 if (ShenandoahDegeneratedGC && heuristics->should_degenerate_cycle() && !old_gen_evacuation_failed) {
155 heuristics->record_allocation_failure_gc();
156 policy->record_alloc_failure_to_degenerated(degen_point);
157 set_gc_mode(stw_degenerated);
158 } else {
159 heuristics->record_allocation_failure_gc();
160 policy->record_alloc_failure_to_full();
161 generation = GLOBAL;
162 set_gc_mode(stw_full);
163 }
164 } else if (explicit_gc_requested) {
165 cause = requested_gc_cause;
166 generation = GLOBAL;
167 log_info(gc)("Trigger: Explicit GC request (%s)", GCCause::to_string(cause));
168
169 global_heuristics->record_requested_gc();
170
171 if (ExplicitGCInvokesConcurrent) {
172 policy->record_explicit_to_concurrent();
173 set_gc_mode(default_mode);
174 // Unload and clean up everything
175 heap->set_unload_classes(global_heuristics->can_unload_classes());
176 } else {
177 policy->record_explicit_to_full();
178 set_gc_mode(stw_full);
179 }
180 } else if (implicit_gc_requested) {
181 cause = requested_gc_cause;
182 generation = GLOBAL;
183 log_info(gc)("Trigger: Implicit GC request (%s)", GCCause::to_string(cause));
184
185 global_heuristics->record_requested_gc();
186
187 if (ShenandoahImplicitGCInvokesConcurrent) {
188 policy->record_implicit_to_concurrent();
189 set_gc_mode(default_mode);
190
191 // Unload and clean up everything
192 heap->set_unload_classes(global_heuristics->can_unload_classes());
193 } else {
194 policy->record_implicit_to_full();
195 set_gc_mode(stw_full);
196 }
197 } else {
198 // We should only be here if the regulator requested a cycle or if
199 // there is an old generation mark in progress.
200 if (_requested_gc_cause == GCCause::_shenandoah_concurrent_gc) {
201 if (_requested_generation == OLD && heap->doing_mixed_evacuations()) {
202 // If a request to start an old cycle arrived while an old cycle was running, but _before_
203 // it chose any regions for evacuation we don't want to start a new old cycle. Rather, we want
204 // the heuristic to run a young collection so that we can evacuate some old regions.
205 assert(!heap->is_concurrent_old_mark_in_progress(), "Should not be running mixed collections and concurrent marking.");
206 generation = YOUNG;
207 } else {
208 generation = _requested_generation;
209 }
210
211 // preemption was requested or this is a regular cycle
212 cause = GCCause::_shenandoah_concurrent_gc;
213 set_gc_mode(default_mode);
214
215 // Don't start a new old marking if there is one already in progress.
216 if (generation == OLD && heap->is_concurrent_old_mark_in_progress()) {
217 set_gc_mode(servicing_old);
218 }
219
220 if (generation == GLOBAL) {
221 heap->set_unload_classes(global_heuristics->should_unload_classes());
222 } else {
223 heap->set_unload_classes(false);
224 }
225
226 // Don't want to spin in this loop and start a cycle every time, so
227 // clear requested gc cause. This creates a race with callers of the
228 // blocking 'request_gc' method, but there it loops and resets the
229 // '_requested_gc_cause' until a full cycle is completed.
230 _requested_gc_cause = GCCause::_no_gc;
231 } else if (heap->is_concurrent_old_mark_in_progress() || heap->is_prepare_for_old_mark_in_progress()) {
232 // Nobody asked us to do anything, but we have an old-generation mark or old-generation preparation for
233 // mixed evacuation in progress, so resume working on that.
234 log_info(gc)("Resume old gc: marking=%s, preparing=%s",
235 BOOL_TO_STR(heap->is_concurrent_old_mark_in_progress()),
236 BOOL_TO_STR(heap->is_prepare_for_old_mark_in_progress()));
237
238 cause = GCCause::_shenandoah_concurrent_gc;
239 generation = OLD;
240 set_gc_mode(servicing_old);
241 }
242 }
243
244 // Blow all soft references on this cycle, if handling allocation failure,
245 // either implicit or explicit GC request, or we are requested to do so unconditionally.
246 if (generation == GLOBAL && (alloc_failure_pending || implicit_gc_requested || explicit_gc_requested || ShenandoahAlwaysClearSoftRefs)) {
247 heap->soft_ref_policy()->set_should_clear_all_soft_refs(true);
248 }
249
250 bool gc_requested = (_mode != none);
251 assert (!gc_requested || cause != GCCause::_last_gc_cause, "GC cause should be set");
252
253 if (gc_requested) {
254 // GC is starting, bump the internal ID
255 update_gc_id();
256
257 heap->reset_bytes_allocated_since_gc_start();
258
259 MetaspaceCombinedStats meta_sizes = MetaspaceUtils::get_combined_statistics();
260
261 // If GC was requested, we are sampling the counters even without actual triggers
262 // from allocation machinery. This captures GC phases more accurately.
263 set_forced_counters_update(true);
264
265 // If GC was requested, we better dump freeset data for performance debugging
266 {
267 ShenandoahHeapLocker locker(heap->lock());
268 heap->free_set()->log_status();
269 }
270 // In case this is a degenerated cycle, remember whether original cycle was aging.
271 bool was_aging_cycle = heap->is_aging_cycle();
272 heap->set_aging_cycle(false);
273 {
274 switch (_mode) {
275 case concurrent_normal: {
276 // At this point:
277 // if (generation == YOUNG), this is a normal YOUNG cycle
278 // if (generation == OLD), this is a bootstrap OLD cycle
279 // if (generation == GLOBAL), this is a GLOBAL cycle triggered by System.gc()
280 // In all three cases, we want to age old objects if this is an aging cycle
281 if (age_period-- == 0) {
282 heap->set_aging_cycle(true);
283 age_period = ShenandoahAgingCyclePeriod - 1;
284 }
285 service_concurrent_normal_cycle(heap, generation, cause);
286 break;
287 }
288 case stw_degenerated: {
289 heap->set_aging_cycle(was_aging_cycle);
290 if (!service_stw_degenerated_cycle(cause, degen_point)) {
291 // The degenerated GC was upgraded to a Full GC
292 generation = GLOBAL;
293 }
294 break;
295 }
296 case stw_full: {
297 if (age_period-- == 0) {
298 heap->set_aging_cycle(true);
299 age_period = ShenandoahAgingCyclePeriod - 1;
300 }
301 service_stw_full_cycle(cause);
302 break;
303 }
304 case servicing_old: {
305 assert(generation == OLD, "Expected old generation here");
306 GCIdMark gc_id_mark;
307 service_concurrent_old_cycle(heap, cause);
308 break;
309 }
310 default: {
311 ShouldNotReachHere();
312 }
313 }
314 }
315
316 // If this was the requested GC cycle, notify waiters about it
317 if (explicit_gc_requested || implicit_gc_requested) {
318 notify_gc_waiters();
319 }
320
321 // If this was the allocation failure GC cycle, notify waiters about it
322 if (alloc_failure_pending) {
323 notify_alloc_failure_waiters();
324 }
325
326 // Report current free set state at the end of cycle, whether
327 // it is a normal completion, or the abort.
328 {
329 ShenandoahHeapLocker locker(heap->lock());
330 heap->free_set()->log_status();
331
332 // Notify Universe about new heap usage. This has implications for
333 // global soft refs policy, and we better report it every time heap
334 // usage goes down.
335 Universe::heap()->update_capacity_and_used_at_gc();
336
337 // Signal that we have completed a visit to all live objects.
338 Universe::heap()->record_whole_heap_examined_timestamp();
339 }
340
341 // Disable forced counters update, and update counters one more time
342 // to capture the state at the end of GC session.
343 handle_force_counters_update();
344 set_forced_counters_update(false);
345
346 // Retract forceful part of soft refs policy
347 heap->soft_ref_policy()->set_should_clear_all_soft_refs(false);
348
349 // Clear metaspace oom flag, if current cycle unloaded classes
350 if (heap->unload_classes()) {
351 assert(generation == GLOBAL, "Only unload classes during GLOBAL cycle");
352 global_heuristics->clear_metaspace_oom();
353 }
354
355 process_phase_timings(heap);
356
357 // Print Metaspace change following GC (if logging is enabled).
358 MetaspaceUtils::print_metaspace_change(meta_sizes);
359
360 // GC is over, we are at idle now
361 if (ShenandoahPacing) {
362 heap->pacer()->setup_for_idle();
363 }
364 } else {
365 // Allow allocators to know we have seen this much regions
366 if (ShenandoahPacing && (allocs_seen > 0)) {
367 heap->pacer()->report_alloc(allocs_seen);
368 }
369 }
370
371 double current = os::elapsedTime();
372
373 if (ShenandoahUncommit && (explicit_gc_requested || soft_max_changed || (current - last_shrink_time > shrink_period))) {
374 // Explicit GC tries to uncommit everything down to min capacity.
375 // Soft max change tries to uncommit everything down to target capacity.
376 // Periodic uncommit tries to uncommit suitable regions down to min capacity.
377
378 double shrink_before = (explicit_gc_requested || soft_max_changed) ?
379 current :
380 current - (ShenandoahUncommitDelay / 1000.0);
381
382 size_t shrink_until = soft_max_changed ?
383 heap->soft_max_capacity() :
384 heap->min_capacity();
385
386 service_uncommit(shrink_before, shrink_until);
387 heap->phase_timings()->flush_cycle_to_global();
388 last_shrink_time = current;
389 }
390
391 // Don't wait around if there was an allocation failure - start the next cycle immediately.
392 if (!is_alloc_failure_gc()) {
393 // The timed wait is necessary because this thread has a responsibility to send
394 // 'alloc_words' to the pacer when it does not perform a GC.
395 MonitorLocker lock(&_control_lock, Mutex::_no_safepoint_check_flag);
396 lock.wait(ShenandoahControlIntervalMax);
397 }
398 }
399
400 // Wait for the actual stop(), can't leave run_service() earlier.
401 while (!should_terminate()) {
402 os::naked_short_sleep(ShenandoahControlIntervalMin);
403 }
404 }
405
406 void ShenandoahControlThread::process_phase_timings(const ShenandoahHeap* heap) {
407
408 // Commit worker statistics to cycle data
409 heap->phase_timings()->flush_par_workers_to_cycle();
410 if (ShenandoahPacing) {
411 heap->pacer()->flush_stats_to_cycle();
412 }
413
414 ShenandoahCycleStats evac_stats = heap->evac_tracker()->flush_cycle_to_global();
415
416 // Print GC stats for current cycle
417 {
418 LogTarget(Info, gc, stats) lt;
419 if (lt.is_enabled()) {
420 ResourceMark rm;
421 LogStream ls(lt);
422 heap->phase_timings()->print_cycle_on(&ls);
423 ShenandoahEvacuationTracker::print_evacuations_on(&ls, &evac_stats.workers,
424 &evac_stats.mutators);
425 if (ShenandoahPacing) {
426 heap->pacer()->print_cycle_on(&ls);
427 }
428 }
429 }
430
431 // Commit statistics to globals
432 heap->phase_timings()->flush_cycle_to_global();
433
434 }
435
436 // Young and old concurrent cycles are initiated by the regulator. Implicit
437 // and explicit GC requests are handled by the controller thread and always
438 // run a global cycle (which is concurrent by default, but may be overridden
439 // by command line options). Old cycles always degenerate to a global cycle.
440 // Young cycles are degenerated to complete the young cycle. Young
441 // and old degen may upgrade to Full GC. Full GC may also be
442 // triggered directly by a System.gc() invocation.
443 //
444 //
445 // +-----+ Idle +-----+-----------+---------------------+
446 // | + | | |
447 // | | | | |
448 // | | v | |
449 // | | Bootstrap Old +-- | ------------+ |
450 // | | + | | |
451 // | | | | | |
452 // | v v v v |
453 // | Resume Old <----------+ Young +--> Young Degen |
454 // | + + ^ + + |
455 // v | | | | | |
456 // Global <-+ | +----------------------------+ | |
457 // + | | |
458 // | v v |
459 // +---> Global Degen +--------------------> Full <----+
460 //
461 void ShenandoahControlThread::service_concurrent_normal_cycle(
462 const ShenandoahHeap* heap, const GenerationMode generation, GCCause::Cause cause) {
463 GCIdMark gc_id_mark;
464 switch (generation) {
465 case YOUNG: {
466 // Run a young cycle. This might or might not, have interrupted an ongoing
467 // concurrent mark in the old generation. We need to think about promotions
468 // in this case. Promoted objects should be above the TAMS in the old regions
469 // they end up in, but we have to be sure we don't promote into any regions
470 // that are in the cset.
471 log_info(gc, ergo)("Start GC cycle (YOUNG)");
472 service_concurrent_cycle(heap->young_generation(), cause, false);
473 break;
474 }
475 case GLOBAL: {
476 log_info(gc, ergo)("Start GC cycle (GLOBAL)");
477 service_concurrent_cycle(heap->global_generation(), cause, false);
478 break;
479 }
480 case OLD: {
481 log_info(gc, ergo)("Start GC cycle (OLD)");
482 service_concurrent_old_cycle(heap, cause);
483 break;
484 }
485 default:
486 ShouldNotReachHere();
487 }
488 const char* msg;
489 if (heap->mode()->is_generational()) {
490 if (heap->cancelled_gc()) {
491 msg = (generation == YOUNG)? "At end of Interrupted Concurrent Young GC": "At end of Interrupted Concurrent Bootstrap GC";
492 } else {
493 msg = (generation == YOUNG)? "At end of Concurrent Young GC": "At end of Concurrent Bootstrap GC";
494 }
495 } else {
496 msg = heap->cancelled_gc() ? "At end of cancelled GC" : "At end of GC";
497 }
498 heap->log_heap_status(msg);
499 }
500
501 void ShenandoahControlThread::service_concurrent_old_cycle(const ShenandoahHeap* heap, GCCause::Cause &cause) {
502
503 ShenandoahOldGeneration* old_generation = heap->old_generation();
504 ShenandoahYoungGeneration* young_generation = heap->young_generation();
505 ShenandoahOldGeneration::State original_state = old_generation->state();
506
507 TraceCollectorStats tcs(heap->monitoring_support()->concurrent_collection_counters());
508
509 switch (original_state) {
510 case ShenandoahOldGeneration::IDLE: {
511 assert(!heap->is_concurrent_old_mark_in_progress(), "Old already in progress.");
512 assert(old_generation->task_queues()->is_empty(), "Old mark queues should be empty.");
513 }
514 case ShenandoahOldGeneration::FILLING: {
515 _allow_old_preemption.set();
516 ShenandoahGCSession session(cause, old_generation);
517 old_generation->prepare_gc();
518 _allow_old_preemption.unset();
519
520 if (heap->is_prepare_for_old_mark_in_progress()) {
521 assert(old_generation->state() == ShenandoahOldGeneration::FILLING, "Prepare for mark should be in progress.");
522 return;
523 }
524
525 assert(old_generation->state() == ShenandoahOldGeneration::BOOTSTRAPPING, "Finished with filling, should be bootstrapping.");
526 }
527 case ShenandoahOldGeneration::BOOTSTRAPPING: {
528 // Configure the young generation's concurrent mark to put objects in
529 // old regions into the concurrent mark queues associated with the old
530 // generation. The young cycle will run as normal except that rather than
531 // ignore old references it will mark and enqueue them in the old concurrent
532 // task queues but it will not traverse them.
533 young_generation->set_old_gen_task_queues(old_generation->task_queues());
534 ShenandoahGCSession session(cause, young_generation);
535 service_concurrent_cycle(heap,young_generation, cause, true);
536 process_phase_timings(heap);
537 if (heap->cancelled_gc()) {
538 // Young generation bootstrap cycle has failed. Concurrent mark for old generation
539 // is going to resume after degenerated bootstrap cycle completes.
540 log_info(gc)("Bootstrap cycle for old generation was cancelled.");
541 return;
542 }
543
544 // Reset the degenerated point. Normally this would happen at the top
545 // of the control loop, but here we have just completed a young cycle
546 // which has bootstrapped the old concurrent marking.
547 _degen_point = ShenandoahGC::_degenerated_outside_cycle;
548
549 // From here we will 'resume' the old concurrent mark. This will skip reset
550 // and init mark for the concurrent mark. All of that work will have been
551 // done by the bootstrapping young cycle. In order to simplify the debugging
552 // effort, the old cycle will ONLY complete the mark phase. No actual
553 // collection of the old generation is happening here.
554 set_gc_mode(servicing_old);
555 old_generation->transition_to(ShenandoahOldGeneration::MARKING);
556 }
557 case ShenandoahOldGeneration::MARKING: {
558 ShenandoahGCSession session(cause, old_generation);
559 bool marking_complete = resume_concurrent_old_cycle(old_generation, cause);
560 if (marking_complete) {
561 assert(old_generation->state() != ShenandoahOldGeneration::MARKING, "Should not still be marking.");
562 if (original_state == ShenandoahOldGeneration::MARKING) {
563 heap->log_heap_status("At end of Concurrent Old Marking finishing increment");
564 }
565 } else if (original_state == ShenandoahOldGeneration::MARKING) {
566 heap->log_heap_status("At end of Concurrent Old Marking increment");
567 }
568 break;
569 }
570 default:
571 log_error(gc)("Unexpected state for old GC: %d", old_generation->state());
572 ShouldNotReachHere();
573 }
574 }
575
576 bool ShenandoahControlThread::resume_concurrent_old_cycle(ShenandoahGeneration* generation, GCCause::Cause cause) {
577
578 assert(ShenandoahHeap::heap()->is_concurrent_old_mark_in_progress(), "Old mark should be in progress");
579 log_debug(gc)("Resuming old generation with " UINT32_FORMAT " marking tasks queued.", generation->task_queues()->tasks());
580
581 ShenandoahHeap* heap = ShenandoahHeap::heap();
582
583 // We can only tolerate being cancelled during concurrent marking or during preparation for mixed
584 // evacuation. This flag here (passed by reference) is used to control precisely where the regulator
585 // is allowed to cancel a GC.
586 ShenandoahOldGC gc(generation, _allow_old_preemption);
587 if (gc.collect(cause)) {
588 generation->record_success_concurrent(false);
589 }
590
591 if (heap->cancelled_gc()) {
592 // It's possible the gc cycle was cancelled after the last time
593 // the collection checked for cancellation. In which case, the
594 // old gc cycle is still completed, and we have to deal with this
595 // cancellation. We set the degeneration point to be outside
596 // the cycle because if this is an allocation failure, that is
597 // what must be done (there is no degenerated old cycle). If the
598 // cancellation was due to a heuristic wanting to start a young
599 // cycle, then we are not actually going to a degenerated cycle,
600 // so the degenerated point doesn't matter here.
601 check_cancellation_or_degen(ShenandoahGC::_degenerated_outside_cycle);
602 if (_requested_gc_cause == GCCause::_shenandoah_concurrent_gc) {
603 heap->shenandoah_policy()->record_interrupted_old();
604 }
605 return false;
606 }
607 return true;
608 }
609
610 bool ShenandoahControlThread::check_soft_max_changed() const {
611 ShenandoahHeap* heap = ShenandoahHeap::heap();
612 size_t new_soft_max = Atomic::load(&SoftMaxHeapSize);
613 size_t old_soft_max = heap->soft_max_capacity();
614 if (new_soft_max != old_soft_max) {
615 new_soft_max = MAX2(heap->min_capacity(), new_soft_max);
616 new_soft_max = MIN2(heap->max_capacity(), new_soft_max);
617 if (new_soft_max != old_soft_max) {
618 log_info(gc)("Soft Max Heap Size: " SIZE_FORMAT "%s -> " SIZE_FORMAT "%s",
619 byte_size_in_proper_unit(old_soft_max), proper_unit_for_byte_size(old_soft_max),
620 byte_size_in_proper_unit(new_soft_max), proper_unit_for_byte_size(new_soft_max)
621 );
622 heap->set_soft_max_capacity(new_soft_max);
623 return true;
624 }
625 }
626 return false;
627 }
628
629 void ShenandoahControlThread::service_concurrent_cycle(ShenandoahGeneration* generation, GCCause::Cause cause, bool do_old_gc_bootstrap) {
630 // Normal cycle goes via all concurrent phases. If allocation failure (af) happens during
631 // any of the concurrent phases, it first degrades to Degenerated GC and completes GC there.
632 // If second allocation failure happens during Degenerated GC cycle (for example, when GC
633 // tries to evac something and no memory is available), cycle degrades to Full GC.
634 //
635 // There are also a shortcut through the normal cycle: immediate garbage shortcut, when
636 // heuristics says there are no regions to compact, and all the collection comes from immediately
637 // reclaimable regions.
638 //
639 // ................................................................................................
640 //
641 // (immediate garbage shortcut) Concurrent GC
642 // /-------------------------------------------\
643 // | |
644 // | |
645 // | |
646 // | v
647 // [START] ----> Conc Mark ----o----> Conc Evac --o--> Conc Update-Refs ---o----> [END]
648 // | | | ^
649 // | (af) | (af) | (af) |
650 // ..................|....................|.................|..............|.......................
651 // | | | |
652 // | | | | Degenerated GC
653 // v v v |
654 // STW Mark ----------> STW Evac ----> STW Update-Refs ----->o
655 // | | | ^
656 // | (af) | (af) | (af) |
657 // ..................|....................|.................|..............|.......................
658 // | | | |
659 // | v | | Full GC
660 // \------------------->o<----------------/ |
661 // | |
662 // v |
663 // Full GC --------------------------/
664 //
665 if (check_cancellation_or_degen(ShenandoahGC::_degenerated_outside_cycle)) return;
666
667 ShenandoahHeap* heap = ShenandoahHeap::heap();
668 ShenandoahGCSession session(cause, generation);
669 TraceCollectorStats tcs(heap->monitoring_support()->concurrent_collection_counters());
670
671 service_concurrent_cycle(heap, generation, cause, do_old_gc_bootstrap);
672 }
673
674 void ShenandoahControlThread::service_concurrent_cycle(const ShenandoahHeap* heap, ShenandoahGeneration* generation,
675 GCCause::Cause &cause, bool do_old_gc_bootstrap) {
676 ShenandoahConcurrentGC gc(generation, do_old_gc_bootstrap);
677 if (gc.collect(cause)) {
678 // Cycle is complete
679 generation->record_success_concurrent(gc.abbreviated());
680 } else {
681 assert(heap->cancelled_gc(), "Must have been cancelled");
682 check_cancellation_or_degen(gc.degen_point());
683 assert(generation->generation_mode() != OLD, "Old GC takes a different control path");
684 // Concurrent young-gen collection degenerates to young
685 // collection. Same for global collections.
686 _degen_generation = generation;
687 }
688 }
689
690 bool ShenandoahControlThread::check_cancellation_or_degen(ShenandoahGC::ShenandoahDegenPoint point) {
691 ShenandoahHeap* heap = ShenandoahHeap::heap();
692 if (!heap->cancelled_gc()) {
693 return false;
694 }
695
696 if (in_graceful_shutdown()) {
697 return true;
698 }
699
700 assert(_degen_point == ShenandoahGC::_degenerated_outside_cycle,
701 "Should not be set yet: %s", ShenandoahGC::degen_point_to_string(_degen_point));
702
703 if (is_alloc_failure_gc()) {
704 _degen_point = point;
705 return true;
706 }
707
708 if (_preemption_requested.is_set()) {
709 assert(_requested_generation == YOUNG, "Only young GCs may preempt old.");
710 _preemption_requested.unset();
711
712 // Old generation marking is only cancellable during concurrent marking.
713 // Once final mark is complete, the code does not check again for cancellation.
714 // If old generation was cancelled for an allocation failure, we wouldn't
715 // make it to this case. The calling code is responsible for forcing a
716 // cancellation due to allocation failure into a degenerated cycle.
717 _degen_point = point;
718 heap->clear_cancelled_gc(false /* clear oom handler */);
719 return true;
720 }
721
722 fatal("Cancel GC either for alloc failure GC, or gracefully exiting, or to pause old generation marking.");
723 return false;
724 }
725
726 void ShenandoahControlThread::stop_service() {
727 // Nothing to do here.
728 }
729
730 void ShenandoahControlThread::service_stw_full_cycle(GCCause::Cause cause) {
731 ShenandoahHeap* const heap = ShenandoahHeap::heap();
732
733 GCIdMark gc_id_mark;
734 ShenandoahGCSession session(cause, heap->global_generation());
735
736 ShenandoahFullGC gc;
737 gc.collect(cause);
738
739 heap->global_generation()->heuristics()->record_success_full();
740 heap->shenandoah_policy()->record_success_full();
741 }
742
743 bool ShenandoahControlThread::service_stw_degenerated_cycle(GCCause::Cause cause, ShenandoahGC::ShenandoahDegenPoint point) {
744 assert (point != ShenandoahGC::_degenerated_unset, "Degenerated point should be set");
745 ShenandoahHeap* const heap = ShenandoahHeap::heap();
746
747 GCIdMark gc_id_mark;
748 ShenandoahGCSession session(cause, _degen_generation);
749
750 ShenandoahDegenGC gc(point, _degen_generation);
751 gc.collect(cause);
752
753 assert(heap->young_generation()->task_queues()->is_empty(), "Unexpected young generation marking tasks");
754 if (_degen_generation->generation_mode() == GLOBAL) {
755 assert(heap->old_generation()->task_queues()->is_empty(), "Unexpected old generation marking tasks");
756 assert(heap->global_generation()->task_queues()->is_empty(), "Unexpected global generation marking tasks");
757 } else {
758 assert(_degen_generation->generation_mode() == YOUNG, "Expected degenerated young cycle, if not global.");
759 ShenandoahOldGeneration* old_generation = (ShenandoahOldGeneration*) heap->old_generation();
760 if (old_generation->state() == ShenandoahOldGeneration::BOOTSTRAPPING && !gc.upgraded_to_full()) {
761 old_generation->transition_to(ShenandoahOldGeneration::MARKING);
762 }
763 }
764
765 _degen_generation->heuristics()->record_success_degenerated();
766 heap->shenandoah_policy()->record_success_degenerated();
767 return !gc.upgraded_to_full();
768 }
769
770 void ShenandoahControlThread::service_uncommit(double shrink_before, size_t shrink_until) {
771 ShenandoahHeap* heap = ShenandoahHeap::heap();
772
773 // Determine if there is work to do. This avoids taking heap lock if there is
774 // no work available, avoids spamming logs with superfluous logging messages,
775 // and minimises the amount of work while locks are taken.
776
777 if (heap->committed() <= shrink_until) return;
778
779 bool has_work = false;
780 for (size_t i = 0; i < heap->num_regions(); i++) {
781 ShenandoahHeapRegion *r = heap->get_region(i);
782 if (r->is_empty_committed() && (r->empty_time() < shrink_before)) {
783 has_work = true;
784 break;
785 }
786 }
787
788 if (has_work) {
789 heap->entry_uncommit(shrink_before, shrink_until);
790 }
791 }
792
793 bool ShenandoahControlThread::is_explicit_gc(GCCause::Cause cause) const {
794 return GCCause::is_user_requested_gc(cause) ||
795 GCCause::is_serviceability_requested_gc(cause);
796 }
797
798 bool ShenandoahControlThread::is_implicit_gc(GCCause::Cause cause) const {
799 return !is_explicit_gc(cause) && cause != GCCause::_shenandoah_concurrent_gc;
800 }
801
802 void ShenandoahControlThread::request_gc(GCCause::Cause cause) {
803 assert(GCCause::is_user_requested_gc(cause) ||
804 GCCause::is_serviceability_requested_gc(cause) ||
805 cause == GCCause::_metadata_GC_clear_soft_refs ||
806 cause == GCCause::_codecache_GC_aggressive ||
807 cause == GCCause::_codecache_GC_threshold ||
808 cause == GCCause::_full_gc_alot ||
809 cause == GCCause::_wb_young_gc ||
810 cause == GCCause::_wb_full_gc ||
811 cause == GCCause::_wb_breakpoint ||
812 cause == GCCause::_scavenge_alot,
813 "only requested GCs here: %s", GCCause::to_string(cause));
814
815 if (is_explicit_gc(cause)) {
816 if (!DisableExplicitGC) {
817 handle_requested_gc(cause);
818 }
819 } else {
820 handle_requested_gc(cause);
821 }
822 }
823
824 bool ShenandoahControlThread::request_concurrent_gc(GenerationMode generation) {
825 if (_preemption_requested.is_set() || _gc_requested.is_set() || ShenandoahHeap::heap()->cancelled_gc()) {
826 // ignore subsequent requests from the heuristics
827 return false;
828 }
829
830 if (_mode == none) {
831 _requested_gc_cause = GCCause::_shenandoah_concurrent_gc;
832 _requested_generation = generation;
833 notify_control_thread();
834 MonitorLocker ml(&_regulator_lock, Mutex::_no_safepoint_check_flag);
835 ml.wait();
836 return true;
837 }
838
839 if (preempt_old_marking(generation)) {
840 log_info(gc)("Preempting old generation mark to allow %s GC.", generation_name(generation));
841 _requested_gc_cause = GCCause::_shenandoah_concurrent_gc;
842 _requested_generation = generation;
843 _preemption_requested.set();
844 ShenandoahHeap::heap()->cancel_gc(GCCause::_shenandoah_concurrent_gc);
845 notify_control_thread();
846
847 MonitorLocker ml(&_regulator_lock, Mutex::_no_safepoint_check_flag);
848 ml.wait();
849 return true;
850 }
851
852 return false;
853 }
854
855 void ShenandoahControlThread::notify_control_thread() {
856 MonitorLocker locker(&_control_lock, Mutex::_no_safepoint_check_flag);
857 _control_lock.notify();
858 }
859
860 bool ShenandoahControlThread::preempt_old_marking(GenerationMode generation) {
861 return generation == YOUNG && _allow_old_preemption.try_unset();
862 }
863
864 void ShenandoahControlThread::handle_requested_gc(GCCause::Cause cause) {
865 // Make sure we have at least one complete GC cycle before unblocking
866 // from the explicit GC request.
867 //
868 // This is especially important for weak references cleanup and/or native
869 // resources (e.g. DirectByteBuffers) machinery: when explicit GC request
870 // comes very late in the already running cycle, it would miss lots of new
871 // opportunities for cleanup that were made available before the caller
872 // requested the GC.
873
874 MonitorLocker ml(&_gc_waiters_lock);
875 size_t current_gc_id = get_gc_id();
876 size_t required_gc_id = current_gc_id + 1;
877 while (current_gc_id < required_gc_id) {
878 // Although setting gc request is under _gc_waiters_lock, but read side (run_service())
879 // does not take the lock. We need to enforce following order, so that read side sees
880 // latest requested gc cause when the flag is set.
881 _requested_gc_cause = cause;
882 _gc_requested.set();
883 notify_control_thread();
884 if (cause != GCCause::_wb_breakpoint) {
885 ml.wait();
886 }
887 current_gc_id = get_gc_id();
888 }
889 }
890
891 void ShenandoahControlThread::handle_alloc_failure(ShenandoahAllocRequest& req) {
892 ShenandoahHeap* heap = ShenandoahHeap::heap();
893
894 assert(current()->is_Java_thread(), "expect Java thread here");
895
896 if (try_set_alloc_failure_gc()) {
897 // Only report the first allocation failure
898 log_info(gc)("Failed to allocate %s, " SIZE_FORMAT "%s",
899 req.type_string(),
900 byte_size_in_proper_unit(req.size() * HeapWordSize), proper_unit_for_byte_size(req.size() * HeapWordSize));
901
902 // Now that alloc failure GC is scheduled, we can abort everything else
903 heap->cancel_gc(GCCause::_allocation_failure);
904 }
905
906 MonitorLocker ml(&_alloc_failure_waiters_lock);
907 while (is_alloc_failure_gc()) {
908 ml.wait();
909 }
910 }
911
912 void ShenandoahControlThread::handle_alloc_failure_evac(size_t words) {
913 ShenandoahHeap* heap = ShenandoahHeap::heap();
914
915 if (try_set_alloc_failure_gc()) {
916 // Only report the first allocation failure
917 log_info(gc)("Failed to allocate " SIZE_FORMAT "%s for evacuation",
918 byte_size_in_proper_unit(words * HeapWordSize), proper_unit_for_byte_size(words * HeapWordSize));
919 }
920
921 // Forcefully report allocation failure
922 heap->cancel_gc(GCCause::_shenandoah_allocation_failure_evac);
923 }
924
925 void ShenandoahControlThread::notify_alloc_failure_waiters() {
926 _alloc_failure_gc.unset();
927 MonitorLocker ml(&_alloc_failure_waiters_lock);
928 ml.notify_all();
929 }
930
931 bool ShenandoahControlThread::try_set_alloc_failure_gc() {
932 return _alloc_failure_gc.try_set();
933 }
934
935 bool ShenandoahControlThread::is_alloc_failure_gc() {
936 return _alloc_failure_gc.is_set();
937 }
938
939 void ShenandoahControlThread::notify_gc_waiters() {
940 _gc_requested.unset();
941 MonitorLocker ml(&_gc_waiters_lock);
942 ml.notify_all();
943 }
944
945 void ShenandoahControlThread::handle_counters_update() {
946 if (_do_counters_update.is_set()) {
947 _do_counters_update.unset();
948 ShenandoahHeap::heap()->monitoring_support()->update_counters();
949 }
950 }
951
952 void ShenandoahControlThread::handle_force_counters_update() {
953 if (_force_counters_update.is_set()) {
954 _do_counters_update.unset(); // reset these too, we do update now!
955 ShenandoahHeap::heap()->monitoring_support()->update_counters();
956 }
957 }
958
959 void ShenandoahControlThread::notify_heap_changed() {
960 // This is called from allocation path, and thus should be fast.
961
962 // Update monitoring counters when we took a new region. This amortizes the
963 // update costs on slow path.
964 if (_do_counters_update.is_unset()) {
965 _do_counters_update.set();
966 }
967 }
968
969 void ShenandoahControlThread::pacing_notify_alloc(size_t words) {
970 assert(ShenandoahPacing, "should only call when pacing is enabled");
971 Atomic::add(&_allocs_seen, words, memory_order_relaxed);
972 }
973
974 void ShenandoahControlThread::set_forced_counters_update(bool value) {
975 _force_counters_update.set_cond(value);
976 }
977
978 void ShenandoahControlThread::reset_gc_id() {
979 Atomic::store(&_gc_id, (size_t)0);
980 }
981
982 void ShenandoahControlThread::update_gc_id() {
983 Atomic::inc(&_gc_id);
984 }
985
986 size_t ShenandoahControlThread::get_gc_id() {
987 return Atomic::load(&_gc_id);
988 }
989
990 void ShenandoahControlThread::start() {
991 create_and_start();
992 }
993
994 void ShenandoahControlThread::prepare_for_graceful_shutdown() {
995 _graceful_shutdown.set();
996 }
997
998 bool ShenandoahControlThread::in_graceful_shutdown() {
999 return _graceful_shutdown.is_set();
1000 }
1001
1002 const char* ShenandoahControlThread::gc_mode_name(ShenandoahControlThread::GCMode mode) {
1003 switch (mode) {
1004 case none: return "idle";
1005 case concurrent_normal: return "normal";
1006 case stw_degenerated: return "degenerated";
1007 case stw_full: return "full";
1008 case servicing_old: return "old";
1009 default: return "unknown";
1010 }
1011 }
1012
1013 void ShenandoahControlThread::set_gc_mode(ShenandoahControlThread::GCMode new_mode) {
1014 if (_mode != new_mode) {
1015 log_info(gc)("Transition from: %s to: %s", gc_mode_name(_mode), gc_mode_name(new_mode));
1016 _mode = new_mode;
1017 MonitorLocker ml(&_regulator_lock, Mutex::_no_safepoint_check_flag);
1018 ml.notify_all();
1019 }
1020 }