1 /*
2 * Copyright (c) 2021, Red Hat, Inc. All rights reserved.
3 * Copyright Amazon.com Inc. or its affiliates. All Rights Reserved.
4 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
5 *
6 * This code is free software; you can redistribute it and/or modify it
7 * under the terms of the GNU General Public License version 2 only, as
8 * published by the Free Software Foundation.
9 *
10 * This code is distributed in the hope that it will be useful, but WITHOUT
11 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
12 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
13 * version 2 for more details (a copy is included in the LICENSE file that
14 * accompanied this code).
15 *
16 * You should have received a copy of the GNU General Public License version
17 * 2 along with this work; if not, write to the Free Software Foundation,
18 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
19 *
20 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
21 * or visit www.oracle.com if you need additional information or have any
22 * questions.
23 *
24 */
25
26
27 #include "gc/shared/collectorCounters.hpp"
28 #include "gc/shenandoah/shenandoahCollectorPolicy.hpp"
29 #include "gc/shenandoah/shenandoahConcurrentMark.hpp"
30 #include "gc/shenandoah/shenandoahDegeneratedGC.hpp"
31 #include "gc/shenandoah/shenandoahFullGC.hpp"
32 #include "gc/shenandoah/shenandoahGeneration.hpp"
33 #include "gc/shenandoah/shenandoahGenerationalHeap.hpp"
34 #include "gc/shenandoah/shenandoahHeap.inline.hpp"
35 #include "gc/shenandoah/shenandoahMetrics.hpp"
36 #include "gc/shenandoah/shenandoahMonitoringSupport.hpp"
37 #include "gc/shenandoah/shenandoahOldGeneration.hpp"
38 #include "gc/shenandoah/shenandoahRootProcessor.inline.hpp"
39 #include "gc/shenandoah/shenandoahStackWatermark.hpp"
40 #include "gc/shenandoah/shenandoahSTWMark.hpp"
41 #include "gc/shenandoah/shenandoahUtils.hpp"
42 #include "gc/shenandoah/shenandoahVerifier.hpp"
43 #include "gc/shenandoah/shenandoahVMOperations.hpp"
44 #include "gc/shenandoah/shenandoahWorkerPolicy.hpp"
45 #include "gc/shenandoah/shenandoahYoungGeneration.hpp"
46 #include "runtime/vmThread.hpp"
47 #include "utilities/events.hpp"
48
49 ShenandoahDegenGC::ShenandoahDegenGC(ShenandoahDegenPoint degen_point, ShenandoahGeneration* generation, bool do_old_gc_bootstrap) :
50 ShenandoahGC(generation),
51 _degen_point(degen_point),
52 _abbreviated(false),
53 _do_old_gc_bootstrap(do_old_gc_bootstrap) {
54 }
55
56 bool ShenandoahDegenGC::collect(GCCause::Cause cause) {
57 vmop_degenerated();
58 ShenandoahHeap* heap = ShenandoahHeap::heap();
59 if (heap->mode()->is_generational()) {
60 FormatBuffer<32> buf("Degenerated %s GC", _generation->name());
61 const char* msg = _do_old_gc_bootstrap ? "Degenerated Bootstrap Old GC" : buf.buffer();
62 heap->mmu_tracker()->record_degenerated(GCId::current(), msg);
63 heap->log_heap_status(FormatBuffer<64>("At end of %s", msg));
64 }
65 return true;
66 }
67
68 void ShenandoahDegenGC::vmop_degenerated() {
69 TraceCollectorStats tcs(ShenandoahHeap::heap()->monitoring_support()->full_stw_collection_counters());
70 ShenandoahTimingsTracker timing(ShenandoahPhaseTimings::degen_gc_gross);
71 VM_ShenandoahDegeneratedGC degenerated_gc(this);
72 VMThread::execute(°enerated_gc);
73 }
74
75 void ShenandoahDegenGC::entry_degenerated() {
76 const char* msg = degen_event_message(_degen_point);
77 ShenandoahPausePhase gc_phase(msg, ShenandoahPhaseTimings::degen_gc, true /* log_heap_usage */);
78 EventMark em("%s", msg);
79 ShenandoahHeap* const heap = ShenandoahHeap::heap();
80 ShenandoahWorkerScope scope(heap->workers(),
81 ShenandoahWorkerPolicy::calc_workers_for_stw_degenerated(),
82 "stw degenerated gc");
83
84 heap->set_degenerated_gc_in_progress(true);
85 op_degenerated();
86 heap->set_degenerated_gc_in_progress(false);
87 {
88 ShenandoahTimingsTracker timing(ShenandoahPhaseTimings::degen_gc_propagate_gc_state);
89 heap->propagate_gc_state_to_all_threads();
90 }
91 }
92
93 void ShenandoahDegenGC::op_degenerated() {
94 ShenandoahHeap* const heap = ShenandoahHeap::heap();
95 heap->release_injected_pins();
96 // Degenerated GC is STW, but it can also fail. Current mechanics communicates
97 // GC failure via cancelled_concgc() flag. So, if we detect the failure after
98 // some phase, we have to upgrade the Degenerate GC to Full GC.
99 heap->clear_cancelled_gc();
100
101 // If we degenerated from evacuation or update-refs, some objects in cset may
102 // have been self-forwarded by the failing thread. Clear those marks now so
103 // the remainder of this cycle (re-evac, update-refs, verification) sees a
104 // clean forwarding state.
105 if (_degen_point == ShenandoahDegenPoint::_degenerated_evac ||
106 _degen_point == ShenandoahDegenPoint::_degenerated_update_refs) {
107 ShenandoahGCPhase phase(ShenandoahPhaseTimings::degen_gc_un_self_forward);
108 heap->un_self_forward_cset_regions();
109 }
110
111 // If it's passive mode with ShenandoahCardBarrier turned on: clean the write table
112 // without swapping the tables since no scan happens in passive mode anyway
113 if (ShenandoahCardBarrier && !heap->mode()->is_generational()) {
114 heap->old_generation()->card_scan()->mark_write_table_as_clean();
115 }
116
117 if (heap->mode()->is_generational()) {
118 const ShenandoahOldGeneration* old_generation = heap->old_generation();
119 if (!heap->is_concurrent_old_mark_in_progress()) {
120 // If we are not marking the old generation, there should be nothing in the old mark queues
121 assert(old_generation->task_queues()->is_empty(), "Old gen task queues should be empty");
122 } else {
123 // This is still necessary for degenerated cycles because the degeneration point may occur
124 // after final mark of the young generation. See ShenandoahConcurrentGC::op_final_update_refs for
125 // a more detailed explanation.
126 old_generation->transfer_pointers_from_satb();
127 }
128
129 if (_generation->is_global()) {
130 // If we are in a global cycle, the old generation should not be marking. It is, however,
131 // allowed to be holding regions for evacuation or coalescing.
132 assert(old_generation->is_idle()
133 || old_generation->is_doing_mixed_evacuations()
134 || old_generation->is_preparing_for_mark(),
135 "Old generation cannot be in state: %s", old_generation->state_name());
136 }
137 }
138
139 ShenandoahMetricsSnapshot metrics(heap->free_set());
140
141 switch (_degen_point) {
142 // The cases below form the Duff's-like device: it describes the actual GC cycle,
143 // but enters it at different points, depending on which concurrent phase had
144 // degenerated.
145
146 case _degenerated_outside_cycle:
147 // We have degenerated from outside the cycle, which means something is bad with
148 // the heap, most probably heavy humongous fragmentation, or we are very low on free
149 // space. It makes little sense to wait for Full GC to reclaim as much as it can, when
150 // we can do the most aggressive degen cycle, which includes processing references and
151 // class unloading, unless those features are explicitly disabled.
152
153 // Note that we can only do this for "outside-cycle" degens, otherwise we would risk
154 // changing the cycle parameters mid-cycle during concurrent -> degenerated handover.
155 heap->set_unload_classes(_generation->heuristics()->can_unload_classes() &&
156 (!heap->mode()->is_generational() || _generation->is_global()));
157
158 if (heap->mode()->is_generational()) {
159 // Clean the read table before swapping it. The end goal here is to have a clean
160 // write table, and to have the read table updated with the previous write table.
161 heap->old_generation()->card_scan()->mark_read_table_as_clean();
162
163 if (_generation->is_young()) {
164 // Swap remembered sets for young
165 _generation->swap_card_tables();
166 }
167 }
168
169 case _degenerated_roots:
170 // Degenerated from concurrent root mark, reset the flag for STW mark
171 if (!heap->mode()->is_generational()) {
172 if (heap->is_concurrent_mark_in_progress()) {
173 heap->cancel_concurrent_mark();
174 }
175 } else {
176 if (_generation->is_concurrent_mark_in_progress()) {
177 // We want to allow old generation marking to be punctuated by young collections
178 // (even if they have degenerated). If this is a global cycle, we'd have cancelled
179 // the entire old gc before coming into this switch. Note that cancel_marking on
180 // the generation does NOT abandon incomplete SATB buffers as cancel_concurrent_mark does.
181 // We need to separate out the old pointers which is done below.
182 _generation->cancel_marking();
183 }
184
185 if (_degen_point == ShenandoahDegenPoint::_degenerated_roots) {
186 // We only need this if the concurrent cycle has already swapped the card tables.
187 // Marking will use the 'read' table, but interesting pointers may have been
188 // recorded in the 'write' table in the time between the cancelled concurrent cycle
189 // and this degenerated cycle. These pointers need to be included in the 'read' table
190 // used to scan the remembered set during the STW mark which follows here.
191 _generation->merge_write_table();
192 }
193 }
194
195 op_reset();
196
197 // STW mark
198 op_mark();
199
200 case _degenerated_mark:
201 // No fallthrough. Continue mark, handed over from concurrent mark if
202 // concurrent mark has yet completed
203 if (_degen_point == ShenandoahDegenPoint::_degenerated_mark && heap->is_concurrent_mark_in_progress()) {
204 assert(!ShenandoahBarrierSet::satb_mark_queue_set().get_filter_out_young(),
205 "Should not be filtering out young pointers when concurrent mark degenerates");
206 op_finish_mark();
207 }
208 assert(!heap->cancelled_gc(), "STW mark can not OOM");
209
210 /* Degen select Collection Set. etc. */
211 op_prepare_evacuation();
212
213 op_cleanup_early();
214
215 case _degenerated_evac:
216 // If heuristics thinks we should do the cycle, this flag would be set,
217 // and we can do evacuation. Otherwise, it would be the shortcut cycle.
218 if (heap->is_evacuation_in_progress()) {
219
220 if (_degen_point == _degenerated_evac) {
221 // Degeneration under oom-evac protocol allows the mutator LRB to expose
222 // references to from-space objects. This is okay, in theory, because we
223 // will come to the safepoint here to complete the evacuations and update
224 // the references. However, if the from-space reference is written to a
225 // region that was EC during final mark or was recycled after final mark
226 // it will not have TAMS or UWM updated. Such a region is effectively
227 // skipped during update references which can lead to crashes and corruption
228 // if the from-space reference is accessed.
229 if (UseTLAB) {
230 heap->labs_make_parsable();
231 }
232
233 for (size_t i = 0; i < heap->num_regions(); i++) {
234 ShenandoahHeapRegion* r = heap->get_region(i);
235 if (r->is_active() && r->top() > r->get_update_watermark()) {
236 r->set_update_watermark_at_safepoint(r->top());
237 }
238 }
239 }
240
241 // Degeneration under oom-evac protocol might have left some objects in
242 // collection set un-evacuated. Restart evacuation from the beginning to
243 // capture all objects. For all the objects that are already evacuated,
244 // it would be a simple check, which is supposed to be fast. This is also
245 // safe to do even without degeneration, as CSet iterator is at beginning
246 // in preparation for evacuation anyway.
247 //
248 // Before doing that, we need to make sure we never had any cset-pinned
249 // regions. This may happen if allocation failure happened when evacuating
250 // the about-to-be-pinned object, oom-evac protocol left the object in
251 // the collection set, and then the pin reached the cset region. If we continue
252 // the cycle here, we would trash the cset and alive objects in it. To avoid
253 // it, we fail degeneration right away and slide into Full GC to recover.
254
255 {
256 heap->sync_pinned_region_status();
257 heap->collection_set()->clear_current_index();
258 ShenandoahHeapRegion* r;
259 while ((r = heap->collection_set()->next()) != nullptr) {
260 if (r->is_pinned()) {
261 op_degenerated_fail();
262 return;
263 }
264 }
265
266 heap->collection_set()->clear_current_index();
267 }
268 op_evacuate();
269 if (heap->cancelled_gc()) {
270 op_degenerated_fail();
271 return;
272 }
273 } else if (has_in_place_promotions(heap)) {
274 // We have nothing to evacuate, but there are still regions to promote in place.
275 ShenandoahGCPhase phase(ShenandoahPhaseTimings::degen_gc_promote_regions);
276 ShenandoahGenerationalHeap::heap()->promote_regions_in_place(_generation, false /* concurrent*/);
277 }
278
279 // Update collector state regardless of whether there are forwarded objects
280 heap->set_evacuation_in_progress(false);
281 heap->set_concurrent_weak_root_in_progress(false);
282 heap->set_concurrent_strong_root_in_progress(false);
283
284 // If heuristics thinks we should do the cycle, this flag would be set,
285 // and we need to do update-refs. Otherwise, it would be the shortcut cycle.
286 if (heap->has_forwarded_objects()) {
287 op_init_update_refs();
288 assert(!heap->cancelled_gc(), "STW reference update can not OOM");
289 } else {
290 _abbreviated = true;
291 }
292
293 // labs are retired, walk the old regions and update remembered set
294 if (ShenandoahHeap::heap()->mode()->is_generational()) {
295 ShenandoahGenerationalHeap::heap()->old_generation()->update_card_table();
296 }
297
298 case _degenerated_update_refs:
299 if (heap->has_forwarded_objects()) {
300 op_update_refs();
301 op_update_roots();
302 assert(!heap->cancelled_gc(), "STW reference update can not OOM");
303 }
304
305 op_cleanup_complete();
306
307 if (heap->mode()->is_generational()) {
308 ShenandoahGenerationalHeap::heap()->complete_degenerated_cycle();
309 }
310
311 break;
312 default:
313 ShouldNotReachHere();
314 }
315
316 DEBUG_ONLY(heap->assert_no_self_forwards());
317
318 // Leaving degenerated GC, we need to flip barriers back to idle.
319 ShenandoahCodeRoots::arm_nmethods();
320 ShenandoahStackWatermark::change_epoch_id();
321
322 if (ShenandoahVerify) {
323 heap->verifier()->verify_after_degenerated(_generation);
324 }
325
326 if (VerifyAfterGC) {
327 Universe::verify();
328 }
329
330 // Decide if this cycle made good progress, and, if not, should it upgrade to a full GC.
331 const bool progress = metrics.is_good_progress();
332 ShenandoahCollectorPolicy* policy = heap->shenandoah_policy();
333 policy->record_degenerated(_generation->is_young(), _abbreviated, progress);
334 if (progress) {
335 heap->notify_gc_progress();
336 _generation->heuristics()->record_degenerated(heap->mode()->is_generational() && _generation->is_global());
337 heap->start_idle_span();
338 } else if (policy->should_upgrade_degenerated_gc()) {
339 // Upgrade to full GC, register full-GC impact on heuristics.
340 op_degenerated_futile();
341 } else {
342 _generation->heuristics()->record_degenerated(heap->mode()->is_generational() && _generation->is_global());
343 }
344 }
345
346 void ShenandoahDegenGC::op_reset() {
347 _generation->prepare_gc();
348 }
349
350 void ShenandoahDegenGC::op_mark() {
351 assert(!_generation->is_concurrent_mark_in_progress(), "Should be reset");
352 ShenandoahGCPhase phase(ShenandoahPhaseTimings::degen_gc_mark);
353 ShenandoahSTWMark mark(_generation, false /*full gc*/);
354 mark.mark();
355 }
356
357 void ShenandoahDegenGC::op_finish_mark() {
358 ShenandoahConcurrentMark mark(_generation);
359 mark.finish_mark();
360 }
361
362 void ShenandoahDegenGC::op_prepare_evacuation() {
363 ShenandoahHeap* const heap = ShenandoahHeap::heap();
364 if (ShenandoahVerify) {
365 heap->verifier()->verify_roots_no_forwarded(_generation);
366 }
367
368 // STW cleanup weak roots and unload classes
369 heap->parallel_cleaning(_generation, false /*full gc*/);
370
371 // Prepare regions and collection set
372 _generation->prepare_regions_and_collection_set(false /*concurrent*/);
373
374 // Retire the TLABs, which will force threads to reacquire their TLABs after the pause.
375 // This is needed for two reasons. Strong one: new allocations would be with new freeset,
376 // which would be outside the collection set, so no cset writes would happen there.
377 // Weaker one: new allocations would happen past update watermark, and so less work would
378 // be needed for reference updates (would update the large filler instead).
379 if (UseTLAB) {
380 ShenandoahGCPhase phase(ShenandoahPhaseTimings::degen_gc_final_manage_labs);
381 heap->tlabs_retire(false);
382 }
383
384 if (!heap->collection_set()->is_empty()) {
385 if (ShenandoahVerify) {
386 heap->verifier()->verify_before_evacuation(_generation);
387 }
388
389 heap->set_evacuation_in_progress(true);
390 heap->set_has_forwarded_objects(true);
391 } else {
392 if (ShenandoahVerify) {
393 if (has_in_place_promotions(heap)) {
394 heap->verifier()->verify_after_concmark_with_promotions(_generation);
395 } else {
396 heap->verifier()->verify_after_concmark(_generation);
397 }
398 }
399
400 if (VerifyAfterGC) {
401 Universe::verify();
402 }
403 }
404 }
405
406 bool ShenandoahDegenGC::has_in_place_promotions(const ShenandoahHeap* heap) const {
407 return heap->mode()->is_generational() && heap->old_generation()->has_in_place_promotions();
408 }
409
410 void ShenandoahDegenGC::op_cleanup_early() {
411 ShenandoahHeap::heap()->recycle_trash();
412 }
413
414 void ShenandoahDegenGC::op_evacuate() {
415 ShenandoahGCPhase phase(ShenandoahPhaseTimings::degen_gc_evac);
416 ShenandoahHeap::heap()->evacuate_collection_set(_generation, false /* concurrent*/);
417 }
418
419 void ShenandoahDegenGC::op_init_update_refs() {
420 // Evacuation has completed
421 ShenandoahHeap* const heap = ShenandoahHeap::heap();
422 heap->prepare_update_heap_references();
423 heap->set_update_refs_in_progress(true);
424 }
425
426 void ShenandoahDegenGC::op_update_refs() {
427 ShenandoahHeap* const heap = ShenandoahHeap::heap();
428 ShenandoahGCPhase phase(ShenandoahPhaseTimings::degen_gc_update_refs);
429 // Handed over from concurrent update references phase
430 heap->update_heap_references(_generation, false /*concurrent*/);
431
432 heap->set_update_refs_in_progress(false);
433 heap->set_has_forwarded_objects(false);
434 }
435
436 void ShenandoahDegenGC::op_update_roots() {
437 ShenandoahHeap* const heap = ShenandoahHeap::heap();
438
439 update_roots(false /*full_gc*/);
440
441 heap->update_heap_region_states(false /*concurrent*/);
442
443 if (ShenandoahVerify) {
444 heap->verifier()->verify_after_update_refs(_generation);
445 }
446
447 if (VerifyAfterGC) {
448 Universe::verify();
449 }
450
451 heap->rebuild_free_set(false /*concurrent*/);
452 }
453
454 void ShenandoahDegenGC::op_cleanup_complete() {
455 ShenandoahGCPhase phase(ShenandoahPhaseTimings::degen_gc_cleanup_complete);
456 ShenandoahHeap::heap()->recycle_trash();
457 }
458
459 void ShenandoahDegenGC::op_degenerated_fail() {
460 upgrade_to_full();
461 }
462
463 void ShenandoahDegenGC::op_degenerated_futile() {
464 upgrade_to_full();
465 }
466
467 const char* ShenandoahDegenGC::degen_event_message(ShenandoahDegenPoint point) const {
468 switch (point) {
469 case _degenerated_unset:
470 SHENANDOAH_RETURN_EVENT_MESSAGE(_generation->type(), "Pause Degenerated GC", " (<UNSET>)");
471 case _degenerated_outside_cycle:
472 SHENANDOAH_RETURN_EVENT_MESSAGE(_generation->type(), "Pause Degenerated GC", " (Outside of Cycle)");
473 case _degenerated_roots:
474 SHENANDOAH_RETURN_EVENT_MESSAGE(_generation->type(), "Pause Degenerated GC", " (Roots)");
475 case _degenerated_mark:
476 SHENANDOAH_RETURN_EVENT_MESSAGE(_generation->type(), "Pause Degenerated GC", " (Mark)");
477 case _degenerated_evac:
478 SHENANDOAH_RETURN_EVENT_MESSAGE(_generation->type(), "Pause Degenerated GC", " (Evacuation)");
479 case _degenerated_update_refs:
480 SHENANDOAH_RETURN_EVENT_MESSAGE(_generation->type(), "Pause Degenerated GC", " (Update Refs)");
481 default:
482 ShouldNotReachHere();
483 SHENANDOAH_RETURN_EVENT_MESSAGE(_generation->type(), "Pause Degenerated GC", " (?)");
484 }
485 }
486
487 void ShenandoahDegenGC::upgrade_to_full() {
488 log_info(gc)("Degenerated GC upgrading to Full GC");
489 ShenandoahHeap* heap = ShenandoahHeap::heap();
490 heap->cancel_gc(GCCause::_shenandoah_upgrade_to_full_gc);
491 heap->increment_total_collections(true);
492 heap->shenandoah_policy()->record_degenerated_upgrade_to_full();
493 ShenandoahFullGC full_gc;
494 full_gc.op_full(GCCause::_shenandoah_upgrade_to_full_gc);
495 }