1 /*
  2  * Copyright Amazon.com Inc. or its affiliates. All Rights Reserved.
  3  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  4  *
  5  * This code is free software; you can redistribute it and/or modify it
  6  * under the terms of the GNU General Public License version 2 only, as
  7  * published by the Free Software Foundation.
  8  *
  9  * This code is distributed in the hope that it will be useful, but WITHOUT
 10  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
 11  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
 12  * version 2 for more details (a copy is included in the LICENSE file that
 13  * accompanied this code).
 14  *
 15  * You should have received a copy of the GNU General Public License version
 16  * 2 along with this work; if not, write to the Free Software Foundation,
 17  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
 18  *
 19  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
 20  * or visit www.oracle.com if you need additional information or have any
 21  * questions.
 22  *
 23  */
 24 
 25 #include "precompiled.hpp"
 26 
 27 #include "gc/shared/preservedMarks.inline.hpp"
 28 #include "gc/shenandoah/shenandoahGenerationalFullGC.hpp"
 29 #include "gc/shenandoah/shenandoahGenerationalHeap.hpp"
 30 #include "gc/shenandoah/shenandoahGeneration.hpp"
 31 #include "gc/shenandoah/shenandoahHeap.inline.hpp"
 32 #include "gc/shenandoah/shenandoahHeapRegion.hpp"
 33 #include "gc/shenandoah/shenandoahYoungGeneration.hpp"
 34 #include "gc/shenandoah/shenandoahOldGeneration.hpp"
 35 #include "gc/shenandoah/shenandoahUtils.hpp"
 36 
 37 #ifdef ASSERT
 38 void assert_regions_used_not_more_than_capacity(ShenandoahGeneration* generation) {
 39   assert(generation->used_regions_size() <= generation->max_capacity(),
 40          "%s generation affiliated regions must be less than capacity", generation->name());
 41 }
 42 
 43 void assert_usage_not_more_than_regions_used(ShenandoahGeneration* generation) {
 44   assert(generation->used_including_humongous_waste() <= generation->used_regions_size(),
 45          "%s consumed can be no larger than span of affiliated regions", generation->name());
 46 }
 47 #else
 48 void assert_regions_used_not_more_than_capacity(ShenandoahGeneration* generation) {}
 49 void assert_usage_not_more_than_regions_used(ShenandoahGeneration* generation) {}
 50 #endif
 51 
 52 
 53 void ShenandoahGenerationalFullGC::prepare() {
 54   auto heap = ShenandoahGenerationalHeap::heap();
 55   // Since we may arrive here from degenerated GC failure of either young or old, establish generation as GLOBAL.
 56   heap->set_gc_generation(heap->global_generation());
 57 
 58   // No need for old_gen->increase_used() as this was done when plabs were allocated.
 59   heap->reset_generation_reserves();
 60 
 61   // Full GC supersedes any marking or coalescing in old generation.
 62   heap->cancel_old_gc();
 63 }
 64 
 65 void ShenandoahGenerationalFullGC::handle_completion(ShenandoahHeap* heap) {
 66   // Full GC should reset time since last gc for young and old heuristics
 67   ShenandoahYoungGeneration* young = heap->young_generation();
 68   ShenandoahOldGeneration* old = heap->old_generation();
 69   young->heuristics()->record_cycle_end();
 70   old->heuristics()->record_cycle_end();
 71 
 72   heap->mmu_tracker()->record_full(GCId::current());
 73   heap->log_heap_status("At end of Full GC");
 74 
 75   assert(old->state() == ShenandoahOldGeneration::WAITING_FOR_BOOTSTRAP,
 76          "After full GC, old generation should be waiting for bootstrap.");
 77 
 78   // Since we allow temporary violation of these constraints during Full GC, we want to enforce that the assertions are
 79   // made valid by the time Full GC completes.
 80   assert_regions_used_not_more_than_capacity(old);
 81   assert_regions_used_not_more_than_capacity(young);
 82   assert_usage_not_more_than_regions_used(old);
 83   assert_usage_not_more_than_regions_used(young);
 84 
 85   // Establish baseline for next old-has-grown trigger.
 86   old->set_live_bytes_after_last_mark(old->used_including_humongous_waste());
 87 }
 88 
 89 void ShenandoahGenerationalFullGC::rebuild_remembered_set(ShenandoahHeap* heap) {
 90   ShenandoahGCPhase phase(ShenandoahPhaseTimings::full_gc_reconstruct_remembered_set);
 91   ShenandoahRegionIterator regions;
 92   ShenandoahReconstructRememberedSetTask task(&regions);
 93   heap->workers()->run_task(&task);
 94 }
 95 
 96 void ShenandoahGenerationalFullGC::balance_generations_after_gc(ShenandoahHeap* heap) {
 97   size_t old_usage = heap->old_generation()->used_regions_size();
 98   size_t old_capacity = heap->old_generation()->max_capacity();
 99 
100   assert(old_usage % ShenandoahHeapRegion::region_size_bytes() == 0, "Old usage must align with region size");
101   assert(old_capacity % ShenandoahHeapRegion::region_size_bytes() == 0, "Old capacity must align with region size");
102 
103   if (old_capacity > old_usage) {
104     size_t excess_old_regions = (old_capacity - old_usage) / ShenandoahHeapRegion::region_size_bytes();
105     heap->generation_sizer()->transfer_to_young(excess_old_regions);
106   } else if (old_capacity < old_usage) {
107     size_t old_regions_deficit = (old_usage - old_capacity) / ShenandoahHeapRegion::region_size_bytes();
108     heap->generation_sizer()->force_transfer_to_old(old_regions_deficit);
109   }
110 
111   log_info(gc)("FullGC done: young usage: " PROPERFMT ", old usage: " PROPERFMT,
112                PROPERFMTARGS(heap->young_generation()->used()),
113                PROPERFMTARGS(heap->old_generation()->used()));
114 }
115 
116 void ShenandoahGenerationalFullGC::balance_generations_after_rebuilding_free_set() {
117   auto result = ShenandoahGenerationalHeap::heap()->balance_generations();
118   LogTarget(Info, gc, ergo) lt;
119   if (lt.is_enabled()) {
120     LogStream ls(lt);
121     result.print_on("Full GC", &ls);
122   }
123 }
124 
125 void ShenandoahGenerationalFullGC::log_live_in_old(ShenandoahHeap* heap) {
126   LogTarget(Info, gc) lt;
127   if (lt.is_enabled()) {
128     size_t live_bytes_in_old = 0;
129     for (size_t i = 0; i < heap->num_regions(); i++) {
130       ShenandoahHeapRegion* r = heap->get_region(i);
131       if (r->is_old()) {
132         live_bytes_in_old += r->get_live_data_bytes();
133       }
134     }
135     log_info(gc)("Live bytes in old after STW mark: " PROPERFMT, PROPERFMTARGS(live_bytes_in_old));
136   }
137 }
138 
139 void ShenandoahGenerationalFullGC::restore_top_before_promote(ShenandoahHeap* heap) {
140   for (size_t i = 0; i < heap->num_regions(); i++) {
141     ShenandoahHeapRegion* r = heap->get_region(i);
142     if (r->get_top_before_promote() != nullptr) {
143       r->restore_top_before_promote();
144     }
145   }
146 }
147 
148 void ShenandoahGenerationalFullGC::account_for_region(ShenandoahHeapRegion* r, size_t &region_count, size_t &region_usage, size_t &humongous_waste) {
149   region_count++;
150   region_usage += r->used();
151   if (r->is_humongous_start()) {
152     // For each humongous object, we take this path once regardless of how many regions it spans.
153     HeapWord* obj_addr = r->bottom();
154     oop obj = cast_to_oop(obj_addr);
155     size_t word_size = obj->size();
156     size_t region_size_words = ShenandoahHeapRegion::region_size_words();
157     size_t overreach = word_size % region_size_words;
158     if (overreach != 0) {
159       humongous_waste += (region_size_words - overreach) * HeapWordSize;
160     }
161     // else, this humongous object aligns exactly on region size, so no waste.
162   }
163 }
164 
165 void ShenandoahGenerationalFullGC::maybe_coalesce_and_fill_region(ShenandoahHeapRegion* r) {
166   if (r->is_pinned() && r->is_old() && r->is_active() && !r->is_humongous()) {
167     r->begin_preemptible_coalesce_and_fill();
168     r->oop_fill_and_coalesce_without_cancel();
169   }
170 }
171 
172 void ShenandoahGenerationalFullGC::compute_balances() {
173   auto heap = ShenandoahGenerationalHeap::heap();
174 
175   // In case this Full GC resulted from degeneration, clear the tally on anticipated promotion.
176   heap->old_generation()->set_promotion_potential(0);
177   // Invoke this in case we are able to transfer memory from OLD to YOUNG.
178   heap->compute_old_generation_balance(0, 0);
179 }
180 
181 ShenandoahPrepareForGenerationalCompactionObjectClosure::ShenandoahPrepareForGenerationalCompactionObjectClosure(PreservedMarks* preserved_marks,
182                                                           GrowableArray<ShenandoahHeapRegion*>& empty_regions,
183                                                           ShenandoahHeapRegion* from_region, uint worker_id) :
184         _preserved_marks(preserved_marks),
185         _heap(ShenandoahHeap::heap()),
186         _tenuring_threshold(0),
187         _empty_regions(empty_regions),
188         _empty_regions_pos(0),
189         _old_to_region(nullptr),
190         _young_to_region(nullptr),
191         _from_region(nullptr),
192         _from_affiliation(ShenandoahAffiliation::FREE),
193         _old_compact_point(nullptr),
194         _young_compact_point(nullptr),
195         _worker_id(worker_id) {
196   assert(from_region != nullptr, "Worker needs from_region");
197   // assert from_region has live?
198   if (from_region->is_old()) {
199     _old_to_region = from_region;
200     _old_compact_point = from_region->bottom();
201   } else if (from_region->is_young()) {
202     _young_to_region = from_region;
203     _young_compact_point = from_region->bottom();
204   }
205 
206   _tenuring_threshold = _heap->age_census()->tenuring_threshold();
207 }
208 
209 void ShenandoahPrepareForGenerationalCompactionObjectClosure::set_from_region(ShenandoahHeapRegion* from_region) {
210   log_debug(gc)("Worker %u compacting %s Region " SIZE_FORMAT " which had used " SIZE_FORMAT " and %s live",
211                 _worker_id, from_region->affiliation_name(),
212                 from_region->index(), from_region->used(), from_region->has_live()? "has": "does not have");
213 
214   _from_region = from_region;
215   _from_affiliation = from_region->affiliation();
216   if (_from_region->has_live()) {
217     if (_from_affiliation == ShenandoahAffiliation::OLD_GENERATION) {
218       if (_old_to_region == nullptr) {
219         _old_to_region = from_region;
220         _old_compact_point = from_region->bottom();
221       }
222     } else {
223       assert(_from_affiliation == ShenandoahAffiliation::YOUNG_GENERATION, "from_region must be OLD or YOUNG");
224       if (_young_to_region == nullptr) {
225         _young_to_region = from_region;
226         _young_compact_point = from_region->bottom();
227       }
228     }
229   } // else, we won't iterate over this _from_region so we don't need to set up to region to hold copies
230 }
231 
232 void ShenandoahPrepareForGenerationalCompactionObjectClosure::finish() {
233   finish_old_region();
234   finish_young_region();
235 }
236 
237 void ShenandoahPrepareForGenerationalCompactionObjectClosure::finish_old_region() {
238   if (_old_to_region != nullptr) {
239     log_debug(gc)("Planned compaction into Old Region " SIZE_FORMAT ", used: " SIZE_FORMAT " tabulated by worker %u",
240             _old_to_region->index(), _old_compact_point - _old_to_region->bottom(), _worker_id);
241     _old_to_region->set_new_top(_old_compact_point);
242     _old_to_region = nullptr;
243   }
244 }
245 
246 void ShenandoahPrepareForGenerationalCompactionObjectClosure::finish_young_region() {
247   if (_young_to_region != nullptr) {
248     log_debug(gc)("Worker %u planned compaction into Young Region " SIZE_FORMAT ", used: " SIZE_FORMAT,
249             _worker_id, _young_to_region->index(), _young_compact_point - _young_to_region->bottom());
250     _young_to_region->set_new_top(_young_compact_point);
251     _young_to_region = nullptr;
252   }
253 }
254 
255 bool ShenandoahPrepareForGenerationalCompactionObjectClosure::is_compact_same_region() {
256   return (_from_region == _old_to_region) || (_from_region == _young_to_region);
257 }
258 
259 void ShenandoahPrepareForGenerationalCompactionObjectClosure::do_object(oop p) {
260   assert(_from_region != nullptr, "must set before work");
261   assert((_from_region->bottom() <= cast_from_oop<HeapWord*>(p)) && (cast_from_oop<HeapWord*>(p) < _from_region->top()),
262          "Object must reside in _from_region");
263   assert(_heap->complete_marking_context()->is_marked(p), "must be marked");
264   assert(!_heap->complete_marking_context()->allocated_after_mark_start(p), "must be truly marked");
265 
266   size_t obj_size = p->size();
267   uint from_region_age = _from_region->age();
268   uint object_age = p->age();
269 
270   bool promote_object = false;
271   if ((_from_affiliation == ShenandoahAffiliation::YOUNG_GENERATION) &&
272       (from_region_age + object_age >= _tenuring_threshold)) {
273     if ((_old_to_region != nullptr) && (_old_compact_point + obj_size > _old_to_region->end())) {
274       finish_old_region();
275       _old_to_region = nullptr;
276     }
277     if (_old_to_region == nullptr) {
278       if (_empty_regions_pos < _empty_regions.length()) {
279         ShenandoahHeapRegion* new_to_region = _empty_regions.at(_empty_regions_pos);
280         _empty_regions_pos++;
281         new_to_region->set_affiliation(OLD_GENERATION);
282         _old_to_region = new_to_region;
283         _old_compact_point = _old_to_region->bottom();
284         promote_object = true;
285       }
286       // Else this worker thread does not yet have any empty regions into which this aged object can be promoted so
287       // we leave promote_object as false, deferring the promotion.
288     } else {
289       promote_object = true;
290     }
291   }
292 
293   if (promote_object || (_from_affiliation == ShenandoahAffiliation::OLD_GENERATION)) {
294     assert(_old_to_region != nullptr, "_old_to_region should not be nullptr when evacuating to OLD region");
295     if (_old_compact_point + obj_size > _old_to_region->end()) {
296       ShenandoahHeapRegion* new_to_region;
297 
298       log_debug(gc)("Worker %u finishing old region " SIZE_FORMAT ", compact_point: " PTR_FORMAT ", obj_size: " SIZE_FORMAT
299       ", &compact_point[obj_size]: " PTR_FORMAT ", region end: " PTR_FORMAT,  _worker_id, _old_to_region->index(),
300               p2i(_old_compact_point), obj_size, p2i(_old_compact_point + obj_size), p2i(_old_to_region->end()));
301 
302       // Object does not fit.  Get a new _old_to_region.
303       finish_old_region();
304       if (_empty_regions_pos < _empty_regions.length()) {
305         new_to_region = _empty_regions.at(_empty_regions_pos);
306         _empty_regions_pos++;
307         new_to_region->set_affiliation(OLD_GENERATION);
308       } else {
309         // If we've exhausted the previously selected _old_to_region, we know that the _old_to_region is distinct
310         // from _from_region.  That's because there is always room for _from_region to be compacted into itself.
311         // Since we're out of empty regions, let's use _from_region to hold the results of its own compaction.
312         new_to_region = _from_region;
313       }
314 
315       assert(new_to_region != _old_to_region, "must not reuse same OLD to-region");
316       assert(new_to_region != nullptr, "must not be nullptr");
317       _old_to_region = new_to_region;
318       _old_compact_point = _old_to_region->bottom();
319     }
320 
321     // Object fits into current region, record new location, if object does not move:
322     assert(_old_compact_point + obj_size <= _old_to_region->end(), "must fit");
323     shenandoah_assert_not_forwarded(nullptr, p);
324     if (_old_compact_point != cast_from_oop<HeapWord*>(p)) {
325       _preserved_marks->push_if_necessary(p, p->mark());
326       p->forward_to(cast_to_oop(_old_compact_point));
327     }
328     _old_compact_point += obj_size;
329   } else {
330     assert(_from_affiliation == ShenandoahAffiliation::YOUNG_GENERATION,
331            "_from_region must be OLD_GENERATION or YOUNG_GENERATION");
332     assert(_young_to_region != nullptr, "_young_to_region should not be nullptr when compacting YOUNG _from_region");
333 
334     // After full gc compaction, all regions have age 0.  Embed the region's age into the object's age in order to preserve
335     // tenuring progress.
336     if (_heap->is_aging_cycle()) {
337       ShenandoahHeap::increase_object_age(p, from_region_age + 1);
338     } else {
339       ShenandoahHeap::increase_object_age(p, from_region_age);
340     }
341 
342     if (_young_compact_point + obj_size > _young_to_region->end()) {
343       ShenandoahHeapRegion* new_to_region;
344 
345       log_debug(gc)("Worker %u finishing young region " SIZE_FORMAT ", compact_point: " PTR_FORMAT ", obj_size: " SIZE_FORMAT
346       ", &compact_point[obj_size]: " PTR_FORMAT ", region end: " PTR_FORMAT,  _worker_id, _young_to_region->index(),
347               p2i(_young_compact_point), obj_size, p2i(_young_compact_point + obj_size), p2i(_young_to_region->end()));
348 
349       // Object does not fit.  Get a new _young_to_region.
350       finish_young_region();
351       if (_empty_regions_pos < _empty_regions.length()) {
352         new_to_region = _empty_regions.at(_empty_regions_pos);
353         _empty_regions_pos++;
354         new_to_region->set_affiliation(YOUNG_GENERATION);
355       } else {
356         // If we've exhausted the previously selected _young_to_region, we know that the _young_to_region is distinct
357         // from _from_region.  That's because there is always room for _from_region to be compacted into itself.
358         // Since we're out of empty regions, let's use _from_region to hold the results of its own compaction.
359         new_to_region = _from_region;
360       }
361 
362       assert(new_to_region != _young_to_region, "must not reuse same OLD to-region");
363       assert(new_to_region != nullptr, "must not be nullptr");
364       _young_to_region = new_to_region;
365       _young_compact_point = _young_to_region->bottom();
366     }
367 
368     // Object fits into current region, record new location, if object does not move:
369     assert(_young_compact_point + obj_size <= _young_to_region->end(), "must fit");
370     shenandoah_assert_not_forwarded(nullptr, p);
371 
372     if (_young_compact_point != cast_from_oop<HeapWord*>(p)) {
373       _preserved_marks->push_if_necessary(p, p->mark());
374       p->forward_to(cast_to_oop(_young_compact_point));
375     }
376     _young_compact_point += obj_size;
377   }
378 }