1 /* 2 * Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. 3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. 4 * 5 * This code is free software; you can redistribute it and/or modify it 6 * under the terms of the GNU General Public License version 2 only, as 7 * published by the Free Software Foundation. 8 * 9 * This code is distributed in the hope that it will be useful, but WITHOUT 10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License 12 * version 2 for more details (a copy is included in the LICENSE file that 13 * accompanied this code). 14 * 15 * You should have received a copy of the GNU General Public License version 16 * 2 along with this work; if not, write to the Free Software Foundation, 17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. 18 * 19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA 20 * or visit www.oracle.com if you need additional information or have any 21 * questions. 22 * 23 */ 24 25 #include "precompiled.hpp" 26 27 #include "gc/shared/preservedMarks.inline.hpp" 28 #include "gc/shenandoah/shenandoahGenerationalFullGC.hpp" 29 #include "gc/shenandoah/shenandoahGenerationalHeap.hpp" 30 #include "gc/shenandoah/shenandoahGeneration.hpp" 31 #include "gc/shenandoah/shenandoahHeap.inline.hpp" 32 #include "gc/shenandoah/shenandoahHeapRegion.hpp" 33 #include "gc/shenandoah/shenandoahYoungGeneration.hpp" 34 #include "gc/shenandoah/shenandoahOldGeneration.hpp" 35 #include "gc/shenandoah/shenandoahUtils.hpp" 36 37 #ifdef ASSERT 38 void assert_regions_used_not_more_than_capacity(ShenandoahGeneration* generation) { 39 assert(generation->used_regions_size() <= generation->max_capacity(), 40 "%s generation affiliated regions must be less than capacity", generation->name()); 41 } 42 43 void assert_usage_not_more_than_regions_used(ShenandoahGeneration* generation) { 44 assert(generation->used_including_humongous_waste() <= generation->used_regions_size(), 45 "%s consumed can be no larger than span of affiliated regions", generation->name()); 46 } 47 #else 48 void assert_regions_used_not_more_than_capacity(ShenandoahGeneration* generation) {} 49 void assert_usage_not_more_than_regions_used(ShenandoahGeneration* generation) {} 50 #endif 51 52 53 void ShenandoahGenerationalFullGC::prepare() { 54 auto heap = ShenandoahGenerationalHeap::heap(); 55 // Since we may arrive here from degenerated GC failure of either young or old, establish generation as GLOBAL. 56 heap->set_gc_generation(heap->global_generation()); 57 58 // No need for old_gen->increase_used() as this was done when plabs were allocated. 59 heap->reset_generation_reserves(); 60 61 // Full GC supersedes any marking or coalescing in old generation. 62 heap->cancel_old_gc(); 63 } 64 65 void ShenandoahGenerationalFullGC::handle_completion(ShenandoahHeap* heap) { 66 // Full GC should reset time since last gc for young and old heuristics 67 ShenandoahYoungGeneration* young = heap->young_generation(); 68 ShenandoahOldGeneration* old = heap->old_generation(); 69 young->heuristics()->record_cycle_end(); 70 old->heuristics()->record_cycle_end(); 71 72 heap->mmu_tracker()->record_full(GCId::current()); 73 heap->log_heap_status("At end of Full GC"); 74 75 assert(old->is_idle(), "After full GC, old generation should be idle."); 76 77 // Since we allow temporary violation of these constraints during Full GC, we want to enforce that the assertions are 78 // made valid by the time Full GC completes. 79 assert_regions_used_not_more_than_capacity(old); 80 assert_regions_used_not_more_than_capacity(young); 81 assert_usage_not_more_than_regions_used(old); 82 assert_usage_not_more_than_regions_used(young); 83 84 // Establish baseline for next old-has-grown trigger. 85 old->set_live_bytes_after_last_mark(old->used_including_humongous_waste()); 86 } 87 88 void ShenandoahGenerationalFullGC::rebuild_remembered_set(ShenandoahHeap* heap) { 89 ShenandoahGCPhase phase(ShenandoahPhaseTimings::full_gc_reconstruct_remembered_set); 90 ShenandoahRegionIterator regions; 91 ShenandoahReconstructRememberedSetTask task(®ions); 92 heap->workers()->run_task(&task); 93 94 // Rebuilding the remembered set recomputes all the card offsets for objects. 95 // The adjust pointers phase coalesces and fills all necessary regions. In case 96 // we came to the full GC from an incomplete global cycle, we need to indicate 97 // that the old regions are parseable. 98 heap->old_generation()->set_parseable(true); 99 } 100 101 void ShenandoahGenerationalFullGC::balance_generations_after_gc(ShenandoahHeap* heap) { 102 size_t old_usage = heap->old_generation()->used_regions_size(); 103 size_t old_capacity = heap->old_generation()->max_capacity(); 104 105 assert(old_usage % ShenandoahHeapRegion::region_size_bytes() == 0, "Old usage must align with region size"); 106 assert(old_capacity % ShenandoahHeapRegion::region_size_bytes() == 0, "Old capacity must align with region size"); 107 108 if (old_capacity > old_usage) { 109 size_t excess_old_regions = (old_capacity - old_usage) / ShenandoahHeapRegion::region_size_bytes(); 110 heap->generation_sizer()->transfer_to_young(excess_old_regions); 111 } else if (old_capacity < old_usage) { 112 size_t old_regions_deficit = (old_usage - old_capacity) / ShenandoahHeapRegion::region_size_bytes(); 113 heap->generation_sizer()->force_transfer_to_old(old_regions_deficit); 114 } 115 116 log_info(gc)("FullGC done: young usage: " PROPERFMT ", old usage: " PROPERFMT, 117 PROPERFMTARGS(heap->young_generation()->used()), 118 PROPERFMTARGS(heap->old_generation()->used())); 119 } 120 121 void ShenandoahGenerationalFullGC::balance_generations_after_rebuilding_free_set() { 122 auto result = ShenandoahGenerationalHeap::heap()->balance_generations(); 123 LogTarget(Info, gc, ergo) lt; 124 if (lt.is_enabled()) { 125 LogStream ls(lt); 126 result.print_on("Full GC", &ls); 127 } 128 } 129 130 void ShenandoahGenerationalFullGC::log_live_in_old(ShenandoahHeap* heap) { 131 LogTarget(Info, gc) lt; 132 if (lt.is_enabled()) { 133 size_t live_bytes_in_old = 0; 134 for (size_t i = 0; i < heap->num_regions(); i++) { 135 ShenandoahHeapRegion* r = heap->get_region(i); 136 if (r->is_old()) { 137 live_bytes_in_old += r->get_live_data_bytes(); 138 } 139 } 140 log_info(gc)("Live bytes in old after STW mark: " PROPERFMT, PROPERFMTARGS(live_bytes_in_old)); 141 } 142 } 143 144 void ShenandoahGenerationalFullGC::restore_top_before_promote(ShenandoahHeap* heap) { 145 for (size_t i = 0; i < heap->num_regions(); i++) { 146 ShenandoahHeapRegion* r = heap->get_region(i); 147 if (r->get_top_before_promote() != nullptr) { 148 r->restore_top_before_promote(); 149 } 150 } 151 } 152 153 void ShenandoahGenerationalFullGC::account_for_region(ShenandoahHeapRegion* r, size_t ®ion_count, size_t ®ion_usage, size_t &humongous_waste) { 154 region_count++; 155 region_usage += r->used(); 156 if (r->is_humongous_start()) { 157 // For each humongous object, we take this path once regardless of how many regions it spans. 158 HeapWord* obj_addr = r->bottom(); 159 oop obj = cast_to_oop(obj_addr); 160 size_t word_size = obj->size(); 161 size_t region_size_words = ShenandoahHeapRegion::region_size_words(); 162 size_t overreach = word_size % region_size_words; 163 if (overreach != 0) { 164 humongous_waste += (region_size_words - overreach) * HeapWordSize; 165 } 166 // else, this humongous object aligns exactly on region size, so no waste. 167 } 168 } 169 170 void ShenandoahGenerationalFullGC::maybe_coalesce_and_fill_region(ShenandoahHeapRegion* r) { 171 if (r->is_pinned() && r->is_old() && r->is_active() && !r->is_humongous()) { 172 r->begin_preemptible_coalesce_and_fill(); 173 r->oop_coalesce_and_fill(false); 174 } 175 } 176 177 void ShenandoahGenerationalFullGC::compute_balances() { 178 auto heap = ShenandoahGenerationalHeap::heap(); 179 180 // In case this Full GC resulted from degeneration, clear the tally on anticipated promotion. 181 heap->old_generation()->set_promotion_potential(0); 182 // Invoke this in case we are able to transfer memory from OLD to YOUNG. 183 heap->compute_old_generation_balance(0, 0); 184 } 185 186 ShenandoahPrepareForGenerationalCompactionObjectClosure::ShenandoahPrepareForGenerationalCompactionObjectClosure(PreservedMarks* preserved_marks, 187 GrowableArray<ShenandoahHeapRegion*>& empty_regions, 188 ShenandoahHeapRegion* from_region, uint worker_id) : 189 _preserved_marks(preserved_marks), 190 _heap(ShenandoahHeap::heap()), 191 _tenuring_threshold(0), 192 _empty_regions(empty_regions), 193 _empty_regions_pos(0), 194 _old_to_region(nullptr), 195 _young_to_region(nullptr), 196 _from_region(nullptr), 197 _from_affiliation(ShenandoahAffiliation::FREE), 198 _old_compact_point(nullptr), 199 _young_compact_point(nullptr), 200 _worker_id(worker_id) { 201 assert(from_region != nullptr, "Worker needs from_region"); 202 // assert from_region has live? 203 if (from_region->is_old()) { 204 _old_to_region = from_region; 205 _old_compact_point = from_region->bottom(); 206 } else if (from_region->is_young()) { 207 _young_to_region = from_region; 208 _young_compact_point = from_region->bottom(); 209 } 210 211 _tenuring_threshold = _heap->age_census()->tenuring_threshold(); 212 } 213 214 void ShenandoahPrepareForGenerationalCompactionObjectClosure::set_from_region(ShenandoahHeapRegion* from_region) { 215 log_debug(gc)("Worker %u compacting %s Region " SIZE_FORMAT " which had used " SIZE_FORMAT " and %s live", 216 _worker_id, from_region->affiliation_name(), 217 from_region->index(), from_region->used(), from_region->has_live()? "has": "does not have"); 218 219 _from_region = from_region; 220 _from_affiliation = from_region->affiliation(); 221 if (_from_region->has_live()) { 222 if (_from_affiliation == ShenandoahAffiliation::OLD_GENERATION) { 223 if (_old_to_region == nullptr) { 224 _old_to_region = from_region; 225 _old_compact_point = from_region->bottom(); 226 } 227 } else { 228 assert(_from_affiliation == ShenandoahAffiliation::YOUNG_GENERATION, "from_region must be OLD or YOUNG"); 229 if (_young_to_region == nullptr) { 230 _young_to_region = from_region; 231 _young_compact_point = from_region->bottom(); 232 } 233 } 234 } // else, we won't iterate over this _from_region so we don't need to set up to region to hold copies 235 } 236 237 void ShenandoahPrepareForGenerationalCompactionObjectClosure::finish() { 238 finish_old_region(); 239 finish_young_region(); 240 } 241 242 void ShenandoahPrepareForGenerationalCompactionObjectClosure::finish_old_region() { 243 if (_old_to_region != nullptr) { 244 log_debug(gc)("Planned compaction into Old Region " SIZE_FORMAT ", used: " SIZE_FORMAT " tabulated by worker %u", 245 _old_to_region->index(), _old_compact_point - _old_to_region->bottom(), _worker_id); 246 _old_to_region->set_new_top(_old_compact_point); 247 _old_to_region = nullptr; 248 } 249 } 250 251 void ShenandoahPrepareForGenerationalCompactionObjectClosure::finish_young_region() { 252 if (_young_to_region != nullptr) { 253 log_debug(gc)("Worker %u planned compaction into Young Region " SIZE_FORMAT ", used: " SIZE_FORMAT, 254 _worker_id, _young_to_region->index(), _young_compact_point - _young_to_region->bottom()); 255 _young_to_region->set_new_top(_young_compact_point); 256 _young_to_region = nullptr; 257 } 258 } 259 260 bool ShenandoahPrepareForGenerationalCompactionObjectClosure::is_compact_same_region() { 261 return (_from_region == _old_to_region) || (_from_region == _young_to_region); 262 } 263 264 void ShenandoahPrepareForGenerationalCompactionObjectClosure::do_object(oop p) { 265 assert(_from_region != nullptr, "must set before work"); 266 assert((_from_region->bottom() <= cast_from_oop<HeapWord*>(p)) && (cast_from_oop<HeapWord*>(p) < _from_region->top()), 267 "Object must reside in _from_region"); 268 assert(_heap->complete_marking_context()->is_marked(p), "must be marked"); 269 assert(!_heap->complete_marking_context()->allocated_after_mark_start(p), "must be truly marked"); 270 271 size_t obj_size = p->size(); 272 uint from_region_age = _from_region->age(); 273 uint object_age = p->age(); 274 275 bool promote_object = false; 276 if ((_from_affiliation == ShenandoahAffiliation::YOUNG_GENERATION) && 277 (from_region_age + object_age >= _tenuring_threshold)) { 278 if ((_old_to_region != nullptr) && (_old_compact_point + obj_size > _old_to_region->end())) { 279 finish_old_region(); 280 _old_to_region = nullptr; 281 } 282 if (_old_to_region == nullptr) { 283 if (_empty_regions_pos < _empty_regions.length()) { 284 ShenandoahHeapRegion* new_to_region = _empty_regions.at(_empty_regions_pos); 285 _empty_regions_pos++; 286 new_to_region->set_affiliation(OLD_GENERATION); 287 _old_to_region = new_to_region; 288 _old_compact_point = _old_to_region->bottom(); 289 promote_object = true; 290 } 291 // Else this worker thread does not yet have any empty regions into which this aged object can be promoted so 292 // we leave promote_object as false, deferring the promotion. 293 } else { 294 promote_object = true; 295 } 296 } 297 298 if (promote_object || (_from_affiliation == ShenandoahAffiliation::OLD_GENERATION)) { 299 assert(_old_to_region != nullptr, "_old_to_region should not be nullptr when evacuating to OLD region"); 300 if (_old_compact_point + obj_size > _old_to_region->end()) { 301 ShenandoahHeapRegion* new_to_region; 302 303 log_debug(gc)("Worker %u finishing old region " SIZE_FORMAT ", compact_point: " PTR_FORMAT ", obj_size: " SIZE_FORMAT 304 ", &compact_point[obj_size]: " PTR_FORMAT ", region end: " PTR_FORMAT, _worker_id, _old_to_region->index(), 305 p2i(_old_compact_point), obj_size, p2i(_old_compact_point + obj_size), p2i(_old_to_region->end())); 306 307 // Object does not fit. Get a new _old_to_region. 308 finish_old_region(); 309 if (_empty_regions_pos < _empty_regions.length()) { 310 new_to_region = _empty_regions.at(_empty_regions_pos); 311 _empty_regions_pos++; 312 new_to_region->set_affiliation(OLD_GENERATION); 313 } else { 314 // If we've exhausted the previously selected _old_to_region, we know that the _old_to_region is distinct 315 // from _from_region. That's because there is always room for _from_region to be compacted into itself. 316 // Since we're out of empty regions, let's use _from_region to hold the results of its own compaction. 317 new_to_region = _from_region; 318 } 319 320 assert(new_to_region != _old_to_region, "must not reuse same OLD to-region"); 321 assert(new_to_region != nullptr, "must not be nullptr"); 322 _old_to_region = new_to_region; 323 _old_compact_point = _old_to_region->bottom(); 324 } 325 326 // Object fits into current region, record new location, if object does not move: 327 assert(_old_compact_point + obj_size <= _old_to_region->end(), "must fit"); 328 shenandoah_assert_not_forwarded(nullptr, p); 329 if (_old_compact_point != cast_from_oop<HeapWord*>(p)) { 330 _preserved_marks->push_if_necessary(p, p->mark()); 331 p->forward_to(cast_to_oop(_old_compact_point)); 332 } 333 _old_compact_point += obj_size; 334 } else { 335 assert(_from_affiliation == ShenandoahAffiliation::YOUNG_GENERATION, 336 "_from_region must be OLD_GENERATION or YOUNG_GENERATION"); 337 assert(_young_to_region != nullptr, "_young_to_region should not be nullptr when compacting YOUNG _from_region"); 338 339 // After full gc compaction, all regions have age 0. Embed the region's age into the object's age in order to preserve 340 // tenuring progress. 341 if (_heap->is_aging_cycle()) { 342 ShenandoahHeap::increase_object_age(p, from_region_age + 1); 343 } else { 344 ShenandoahHeap::increase_object_age(p, from_region_age); 345 } 346 347 if (_young_compact_point + obj_size > _young_to_region->end()) { 348 ShenandoahHeapRegion* new_to_region; 349 350 log_debug(gc)("Worker %u finishing young region " SIZE_FORMAT ", compact_point: " PTR_FORMAT ", obj_size: " SIZE_FORMAT 351 ", &compact_point[obj_size]: " PTR_FORMAT ", region end: " PTR_FORMAT, _worker_id, _young_to_region->index(), 352 p2i(_young_compact_point), obj_size, p2i(_young_compact_point + obj_size), p2i(_young_to_region->end())); 353 354 // Object does not fit. Get a new _young_to_region. 355 finish_young_region(); 356 if (_empty_regions_pos < _empty_regions.length()) { 357 new_to_region = _empty_regions.at(_empty_regions_pos); 358 _empty_regions_pos++; 359 new_to_region->set_affiliation(YOUNG_GENERATION); 360 } else { 361 // If we've exhausted the previously selected _young_to_region, we know that the _young_to_region is distinct 362 // from _from_region. That's because there is always room for _from_region to be compacted into itself. 363 // Since we're out of empty regions, let's use _from_region to hold the results of its own compaction. 364 new_to_region = _from_region; 365 } 366 367 assert(new_to_region != _young_to_region, "must not reuse same OLD to-region"); 368 assert(new_to_region != nullptr, "must not be nullptr"); 369 _young_to_region = new_to_region; 370 _young_compact_point = _young_to_region->bottom(); 371 } 372 373 // Object fits into current region, record new location, if object does not move: 374 assert(_young_compact_point + obj_size <= _young_to_region->end(), "must fit"); 375 shenandoah_assert_not_forwarded(nullptr, p); 376 377 if (_young_compact_point != cast_from_oop<HeapWord*>(p)) { 378 _preserved_marks->push_if_necessary(p, p->mark()); 379 p->forward_to(cast_to_oop(_young_compact_point)); 380 } 381 _young_compact_point += obj_size; 382 } 383 }