1 /*
  2  * Copyright (c) 2021, Red Hat, Inc. All rights reserved.
  3  * Copyright Amazon.com Inc. or its affiliates. All Rights Reserved.
  4  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  5  *
  6  * This code is free software; you can redistribute it and/or modify it
  7  * under the terms of the GNU General Public License version 2 only, as
  8  * published by the Free Software Foundation.
  9  *
 10  * This code is distributed in the hope that it will be useful, but WITHOUT
 11  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
 12  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
 13  * version 2 for more details (a copy is included in the LICENSE file that
 14  * accompanied this code).
 15  *
 16  * You should have received a copy of the GNU General Public License version
 17  * 2 along with this work; if not, write to the Free Software Foundation,
 18  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
 19  *
 20  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
 21  * or visit www.oracle.com if you need additional information or have any
 22  * questions.
 23  *
 24  */
 25 
 26 #include "precompiled.hpp"
 27 
 28 #include "gc/shared/collectorCounters.hpp"
 29 #include "gc/shenandoah/shenandoahCollectorPolicy.hpp"
 30 #include "gc/shenandoah/shenandoahConcurrentMark.hpp"
 31 #include "gc/shenandoah/shenandoahDegeneratedGC.hpp"
 32 #include "gc/shenandoah/shenandoahFullGC.hpp"
 33 #include "gc/shenandoah/shenandoahGeneration.hpp"
 34 #include "gc/shenandoah/shenandoahGenerationalHeap.hpp"
 35 #include "gc/shenandoah/shenandoahHeap.inline.hpp"
 36 #include "gc/shenandoah/shenandoahMetrics.hpp"
 37 #include "gc/shenandoah/shenandoahMonitoringSupport.hpp"
 38 #include "gc/shenandoah/shenandoahOldGeneration.hpp"
 39 #include "gc/shenandoah/shenandoahOopClosures.inline.hpp"
 40 #include "gc/shenandoah/shenandoahRootProcessor.inline.hpp"
 41 #include "gc/shenandoah/shenandoahSTWMark.hpp"
 42 #include "gc/shenandoah/shenandoahUtils.hpp"
 43 #include "gc/shenandoah/shenandoahVerifier.hpp"
 44 #include "gc/shenandoah/shenandoahYoungGeneration.hpp"
 45 #include "gc/shenandoah/shenandoahWorkerPolicy.hpp"
 46 #include "gc/shenandoah/shenandoahVMOperations.hpp"
 47 #include "runtime/vmThread.hpp"
 48 #include "utilities/events.hpp"
 49 
 50 ShenandoahDegenGC::ShenandoahDegenGC(ShenandoahDegenPoint degen_point, ShenandoahGeneration* generation) :
 51   ShenandoahGC(),
 52   _degen_point(degen_point),
 53   _generation(generation),
 54   _abbreviated(false) {
 55 }
 56 
 57 bool ShenandoahDegenGC::collect(GCCause::Cause cause) {
 58   vmop_degenerated();
 59   ShenandoahHeap* heap = ShenandoahHeap::heap();
 60   if (heap->mode()->is_generational()) {
 61     bool is_bootstrap_gc = heap->old_generation()->is_bootstrapping();
 62     heap->mmu_tracker()->record_degenerated(GCId::current(), is_bootstrap_gc);
 63     const char* msg = is_bootstrap_gc? "At end of Degenerated Bootstrap Old GC": "At end of Degenerated Young GC";
 64     heap->log_heap_status(msg);
 65   }
 66   return true;
 67 }
 68 
 69 void ShenandoahDegenGC::vmop_degenerated() {
 70   TraceCollectorStats tcs(ShenandoahHeap::heap()->monitoring_support()->full_stw_collection_counters());
 71   ShenandoahTimingsTracker timing(ShenandoahPhaseTimings::degen_gc_gross);
 72   VM_ShenandoahDegeneratedGC degenerated_gc(this);
 73   VMThread::execute(&degenerated_gc);
 74 }
 75 
 76 void ShenandoahDegenGC::entry_degenerated() {
 77   const char* msg = degen_event_message(_degen_point);
 78   ShenandoahPausePhase gc_phase(msg, ShenandoahPhaseTimings::degen_gc, true /* log_heap_usage */);
 79   EventMark em("%s", msg);
 80   ShenandoahHeap* const heap = ShenandoahHeap::heap();
 81   ShenandoahWorkerScope scope(heap->workers(),
 82                               ShenandoahWorkerPolicy::calc_workers_for_stw_degenerated(),
 83                               "stw degenerated gc");
 84 
 85   heap->set_degenerated_gc_in_progress(true);
 86   op_degenerated();
 87   heap->set_degenerated_gc_in_progress(false);
 88   {
 89     ShenandoahTimingsTracker timing(ShenandoahPhaseTimings::degen_gc_propagate_gc_state);
 90     heap->propagate_gc_state_to_all_threads();
 91   }
 92 }
 93 
 94 void ShenandoahDegenGC::op_degenerated() {
 95   ShenandoahHeap* const heap = ShenandoahHeap::heap();
 96   // Degenerated GC is STW, but it can also fail. Current mechanics communicates
 97   // GC failure via cancelled_concgc() flag. So, if we detect the failure after
 98   // some phase, we have to upgrade the Degenerate GC to Full GC.
 99   heap->clear_cancelled_gc(true /* clear oom handler */);
100 
101 #ifdef ASSERT
102   if (heap->mode()->is_generational()) {
103     ShenandoahOldGeneration* old_generation = heap->old_generation();
104     if (!heap->is_concurrent_old_mark_in_progress()) {
105       // If we are not marking the old generation, there should be nothing in the old mark queues
106       assert(old_generation->task_queues()->is_empty(), "Old gen task queues should be empty");
107     }
108 
109     if (_generation->is_global()) {
110       // If we are in a global cycle, the old generation should not be marking. It is, however,
111       // allowed to be holding regions for evacuation or coalescing.
112       assert(old_generation->is_idle()
113              || old_generation->is_doing_mixed_evacuations()
114              || old_generation->is_preparing_for_mark(),
115              "Old generation cannot be in state: %s", old_generation->state_name());
116     }
117   }
118 #endif
119 
120   ShenandoahMetricsSnapshot metrics;
121   metrics.snap_before();
122 
123   switch (_degen_point) {
124     // The cases below form the Duff's-like device: it describes the actual GC cycle,
125     // but enters it at different points, depending on which concurrent phase had
126     // degenerated.
127 
128     case _degenerated_outside_cycle:
129       // We have degenerated from outside the cycle, which means something is bad with
130       // the heap, most probably heavy humongous fragmentation, or we are very low on free
131       // space. It makes little sense to wait for Full GC to reclaim as much as it can, when
132       // we can do the most aggressive degen cycle, which includes processing references and
133       // class unloading, unless those features are explicitly disabled.
134 
135       // Note that we can only do this for "outside-cycle" degens, otherwise we would risk
136       // changing the cycle parameters mid-cycle during concurrent -> degenerated handover.
137       heap->set_unload_classes(_generation->heuristics()->can_unload_classes() &&
138                                 (!heap->mode()->is_generational() || _generation->is_global()));
139 
140       if (heap->mode()->is_generational()) {
141         // Clean the read table before swapping it. The end goal here is to have a clean
142         // write table, and to have the read table updated with the previous write table.
143         heap->old_generation()->card_scan()->mark_read_table_as_clean();
144 
145         if (_generation->is_young()) {
146           // Swap remembered sets for young
147           _generation->swap_card_tables();
148         }
149       }
150 
151     case _degenerated_roots:
152       // Degenerated from concurrent root mark, reset the flag for STW mark
153       if (!heap->mode()->is_generational()) {
154         if (heap->is_concurrent_mark_in_progress()) {
155           heap->cancel_concurrent_mark();
156         }
157       } else {
158         if (_generation->is_concurrent_mark_in_progress()) {
159           // We want to allow old generation marking to be punctuated by young collections
160           // (even if they have degenerated). If this is a global cycle, we'd have cancelled
161           // the entire old gc before coming into this switch. Note that cancel_marking on
162           // the generation does NOT abandon incomplete SATB buffers as cancel_concurrent_mark does.
163           // We need to separate out the old pointers which is done below.
164           _generation->cancel_marking();
165         }
166 
167         if (heap->is_concurrent_mark_in_progress()) {
168           // If either old or young marking is in progress, the SATB barrier will be enabled.
169           // The SATB buffer may hold a mix of old and young pointers. The old pointers need to be
170           // transferred to the old generation mark queues and the young pointers are NOT part
171           // of this snapshot, so they must be dropped here. It is safe to drop them here because
172           // we will rescan the roots on this safepoint.
173           heap->old_generation()->transfer_pointers_from_satb();
174         }
175 
176         if (_degen_point == ShenandoahDegenPoint::_degenerated_roots) {
177           // We only need this if the concurrent cycle has already swapped the card tables.
178           // Marking will use the 'read' table, but interesting pointers may have been
179           // recorded in the 'write' table in the time between the cancelled concurrent cycle
180           // and this degenerated cycle. These pointers need to be included in the 'read' table
181           // used to scan the remembered set during the STW mark which follows here.
182           _generation->merge_write_table();
183         }
184       }
185 
186       op_reset();
187 
188       // STW mark
189       op_mark();
190 
191     case _degenerated_mark:
192       // No fallthrough. Continue mark, handed over from concurrent mark if
193       // concurrent mark has yet completed
194       if (_degen_point == ShenandoahDegenPoint::_degenerated_mark &&
195           heap->is_concurrent_mark_in_progress()) {
196         op_finish_mark();
197       }
198       assert(!heap->cancelled_gc(), "STW mark can not OOM");
199 
200       /* Degen select Collection Set. etc. */
201       op_prepare_evacuation();
202 
203       op_cleanup_early();
204 
205     case _degenerated_evac:
206       // If heuristics thinks we should do the cycle, this flag would be set,
207       // and we can do evacuation. Otherwise, it would be the shortcut cycle.
208       if (heap->is_evacuation_in_progress()) {
209 
210         if (_degen_point == _degenerated_evac) {
211           // Degeneration under oom-evac protocol allows the mutator LRB to expose
212           // references to from-space objects. This is okay, in theory, because we
213           // will come to the safepoint here to complete the evacuations and update
214           // the references. However, if the from-space reference is written to a
215           // region that was EC during final mark or was recycled after final mark
216           // it will not have TAMS or UWM updated. Such a region is effectively
217           // skipped during update references which can lead to crashes and corruption
218           // if the from-space reference is accessed.
219           if (UseTLAB) {
220             heap->labs_make_parsable();
221           }
222 
223           for (size_t i = 0; i < heap->num_regions(); i++) {
224             ShenandoahHeapRegion* r = heap->get_region(i);
225             if (r->is_active() && r->top() > r->get_update_watermark()) {
226               r->set_update_watermark_at_safepoint(r->top());
227             }
228           }
229         }
230 
231         // Degeneration under oom-evac protocol might have left some objects in
232         // collection set un-evacuated. Restart evacuation from the beginning to
233         // capture all objects. For all the objects that are already evacuated,
234         // it would be a simple check, which is supposed to be fast. This is also
235         // safe to do even without degeneration, as CSet iterator is at beginning
236         // in preparation for evacuation anyway.
237         //
238         // Before doing that, we need to make sure we never had any cset-pinned
239         // regions. This may happen if allocation failure happened when evacuating
240         // the about-to-be-pinned object, oom-evac protocol left the object in
241         // the collection set, and then the pin reached the cset region. If we continue
242         // the cycle here, we would trash the cset and alive objects in it. To avoid
243         // it, we fail degeneration right away and slide into Full GC to recover.
244 
245         {
246           heap->sync_pinned_region_status();
247           heap->collection_set()->clear_current_index();
248           ShenandoahHeapRegion* r;
249           while ((r = heap->collection_set()->next()) != nullptr) {
250             if (r->is_pinned()) {
251               op_degenerated_fail();
252               return;
253             }
254           }
255 
256           heap->collection_set()->clear_current_index();
257         }
258         op_evacuate();
259         if (heap->cancelled_gc()) {
260           op_degenerated_fail();
261           return;
262         }
263       } else if (has_in_place_promotions(heap)) {
264         // We have nothing to evacuate, but there are still regions to promote in place.
265         ShenandoahGCPhase phase(ShenandoahPhaseTimings::degen_gc_promote_regions);
266         ShenandoahGenerationalHeap::heap()->promote_regions_in_place(false /* concurrent*/);
267       }
268 
269       // Update collector state regardless of whether there are forwarded objects
270       heap->set_evacuation_in_progress(false);
271       heap->set_concurrent_weak_root_in_progress(false);
272       heap->set_concurrent_strong_root_in_progress(false);
273 
274       // If heuristics thinks we should do the cycle, this flag would be set,
275       // and we need to do update-refs. Otherwise, it would be the shortcut cycle.
276       if (heap->has_forwarded_objects()) {
277         op_init_update_refs();
278         assert(!heap->cancelled_gc(), "STW reference update can not OOM");
279       } else {
280         _abbreviated = true;
281       }
282 
283     case _degenerated_update_refs:
284       if (heap->has_forwarded_objects()) {
285         op_update_refs();
286         op_update_roots();
287         assert(!heap->cancelled_gc(), "STW reference update can not OOM");
288       }
289 
290       // Disarm nmethods that armed in concurrent cycle.
291       // In above case, update roots should disarm them
292       ShenandoahCodeRoots::disarm_nmethods();
293 
294       op_cleanup_complete();
295 
296       if (heap->mode()->is_generational()) {
297         ShenandoahGenerationalHeap::heap()->complete_degenerated_cycle();
298       }
299 
300       break;
301     default:
302       ShouldNotReachHere();
303   }
304 
305   if (ShenandoahVerify) {
306     heap->verifier()->verify_after_degenerated();
307   }
308 
309   if (VerifyAfterGC) {
310     Universe::verify();
311   }
312 
313   metrics.snap_after();
314 
315   // Decide if this cycle made good progress, and, if not, should it upgrade to a full GC.
316   const bool progress = metrics.is_good_progress(_generation);
317   ShenandoahCollectorPolicy* policy = heap->shenandoah_policy();
318   policy->record_degenerated(_generation->is_young(), _abbreviated, progress);
319   if (progress) {
320     heap->notify_gc_progress();
321   } else if (!heap->mode()->is_generational() || policy->generational_should_upgrade_degenerated_gc()) {
322     op_degenerated_futile();
323   }
324 }
325 
326 void ShenandoahDegenGC::op_reset() {
327   _generation->prepare_gc();
328 }
329 
330 void ShenandoahDegenGC::op_mark() {
331   assert(!_generation->is_concurrent_mark_in_progress(), "Should be reset");
332   ShenandoahGCPhase phase(ShenandoahPhaseTimings::degen_gc_stw_mark);
333   ShenandoahSTWMark mark(_generation, false /*full gc*/);
334   mark.mark();
335 }
336 
337 void ShenandoahDegenGC::op_finish_mark() {
338   ShenandoahConcurrentMark mark(_generation);
339   mark.finish_mark();
340 }
341 
342 void ShenandoahDegenGC::op_prepare_evacuation() {
343   ShenandoahHeap* const heap = ShenandoahHeap::heap();
344   if (ShenandoahVerify) {
345     heap->verifier()->verify_roots_no_forwarded();
346   }
347 
348   // STW cleanup weak roots and unload classes
349   heap->parallel_cleaning(false /*full gc*/);
350 
351   // Prepare regions and collection set
352   _generation->prepare_regions_and_collection_set(false /*concurrent*/);
353 
354   // Retire the TLABs, which will force threads to reacquire their TLABs after the pause.
355   // This is needed for two reasons. Strong one: new allocations would be with new freeset,
356   // which would be outside the collection set, so no cset writes would happen there.
357   // Weaker one: new allocations would happen past update watermark, and so less work would
358   // be needed for reference updates (would update the large filler instead).
359   if (UseTLAB) {
360     ShenandoahGCPhase phase(ShenandoahPhaseTimings::degen_gc_final_manage_labs);
361     heap->tlabs_retire(false);
362   }
363 
364   if (!heap->collection_set()->is_empty()) {
365     if (ShenandoahVerify) {
366       heap->verifier()->verify_before_evacuation();
367     }
368 
369     heap->set_evacuation_in_progress(true);
370 
371     heap->set_has_forwarded_objects(true);
372   } else {
373     if (ShenandoahVerify) {
374       if (has_in_place_promotions(heap)) {
375         heap->verifier()->verify_after_concmark_with_promotions();
376       } else {
377         heap->verifier()->verify_after_concmark();
378       }
379     }
380 
381     if (VerifyAfterGC) {
382       Universe::verify();
383     }
384   }
385 }
386 
387 bool ShenandoahDegenGC::has_in_place_promotions(const ShenandoahHeap* heap) const {
388   return heap->mode()->is_generational() && heap->old_generation()->has_in_place_promotions();
389 }
390 
391 void ShenandoahDegenGC::op_cleanup_early() {
392   ShenandoahHeap::heap()->recycle_trash();
393 }
394 
395 void ShenandoahDegenGC::op_evacuate() {
396   ShenandoahGCPhase phase(ShenandoahPhaseTimings::degen_gc_stw_evac);
397   ShenandoahHeap::heap()->evacuate_collection_set(false /* concurrent*/);
398 }
399 
400 void ShenandoahDegenGC::op_init_update_refs() {
401   // Evacuation has completed
402   ShenandoahHeap* const heap = ShenandoahHeap::heap();
403   heap->prepare_update_heap_references();
404   heap->set_update_refs_in_progress(true);
405 }
406 
407 void ShenandoahDegenGC::op_update_refs() {
408   ShenandoahHeap* const heap = ShenandoahHeap::heap();
409   ShenandoahGCPhase phase(ShenandoahPhaseTimings::degen_gc_update_refs);
410   // Handed over from concurrent update references phase
411   heap->update_heap_references(false /*concurrent*/);
412 
413   heap->set_update_refs_in_progress(false);
414   heap->set_has_forwarded_objects(false);
415 }
416 
417 void ShenandoahDegenGC::op_update_roots() {
418   ShenandoahHeap* const heap = ShenandoahHeap::heap();
419 
420   update_roots(false /*full_gc*/);
421 
422   heap->update_heap_region_states(false /*concurrent*/);
423 
424   if (ShenandoahVerify) {
425     heap->verifier()->verify_after_update_refs();
426   }
427 
428   if (VerifyAfterGC) {
429     Universe::verify();
430   }
431 
432   heap->rebuild_free_set(false /*concurrent*/);
433 }
434 
435 void ShenandoahDegenGC::op_cleanup_complete() {
436   ShenandoahGCPhase phase(ShenandoahPhaseTimings::degen_gc_cleanup_complete);
437   ShenandoahHeap::heap()->recycle_trash();
438 }
439 
440 void ShenandoahDegenGC::op_degenerated_fail() {
441   upgrade_to_full();
442 }
443 
444 void ShenandoahDegenGC::op_degenerated_futile() {
445   upgrade_to_full();
446 }
447 
448 const char* ShenandoahDegenGC::degen_event_message(ShenandoahDegenPoint point) const {
449   switch (point) {
450     case _degenerated_unset:
451       SHENANDOAH_RETURN_EVENT_MESSAGE(_generation->type(), "Pause Degenerated GC", " (<UNSET>)");
452     case _degenerated_outside_cycle:
453       SHENANDOAH_RETURN_EVENT_MESSAGE(_generation->type(), "Pause Degenerated GC", " (Outside of Cycle)");
454     case _degenerated_roots:
455       SHENANDOAH_RETURN_EVENT_MESSAGE(_generation->type(), "Pause Degenerated GC", " (Roots)");
456     case _degenerated_mark:
457       SHENANDOAH_RETURN_EVENT_MESSAGE(_generation->type(), "Pause Degenerated GC", " (Mark)");
458     case _degenerated_evac:
459       SHENANDOAH_RETURN_EVENT_MESSAGE(_generation->type(), "Pause Degenerated GC", " (Evacuation)");
460     case _degenerated_update_refs:
461       SHENANDOAH_RETURN_EVENT_MESSAGE(_generation->type(), "Pause Degenerated GC", " (Update Refs)");
462     default:
463       ShouldNotReachHere();
464       SHENANDOAH_RETURN_EVENT_MESSAGE(_generation->type(), "Pause Degenerated GC", " (?)");
465   }
466 }
467 
468 void ShenandoahDegenGC::upgrade_to_full() {
469   log_info(gc)("Degenerated GC upgrading to Full GC");
470   ShenandoahHeap* heap = ShenandoahHeap::heap();
471   heap->cancel_gc(GCCause::_shenandoah_upgrade_to_full_gc);
472   heap->shenandoah_policy()->record_degenerated_upgrade_to_full();
473   ShenandoahFullGC full_gc;
474   full_gc.op_full(GCCause::_shenandoah_upgrade_to_full_gc);
475 }