1 /* 2 * Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. 3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. 4 * 5 * This code is free software; you can redistribute it and/or modify it 6 * under the terms of the GNU General Public License version 2 only, as 7 * published by the Free Software Foundation. 8 * 9 * This code is distributed in the hope that it will be useful, but WITHOUT 10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License 12 * version 2 for more details (a copy is included in the LICENSE file that 13 * accompanied this code). 14 * 15 * You should have received a copy of the GNU General Public License version 16 * 2 along with this work; if not, write to the Free Software Foundation, 17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. 18 * 19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA 20 * or visit www.oracle.com if you need additional information or have any 21 * questions. 22 * 23 */ 24 25 #include "precompiled.hpp" 26 #include "gc/shenandoah/shenandoahCollectorPolicy.hpp" 27 #include "gc/shenandoah/shenandoahCollectionSetPreselector.hpp" 28 #include "gc/shenandoah/shenandoahFreeSet.hpp" 29 #include "gc/shenandoah/shenandoahGeneration.hpp" 30 #include "gc/shenandoah/shenandoahGenerationalHeap.hpp" 31 #include "gc/shenandoah/shenandoahMarkClosures.hpp" 32 #include "gc/shenandoah/shenandoahMonitoringSupport.hpp" 33 #include "gc/shenandoah/shenandoahOldGeneration.hpp" 34 #include "gc/shenandoah/shenandoahReferenceProcessor.hpp" 35 #include "gc/shenandoah/shenandoahScanRemembered.inline.hpp" 36 #include "gc/shenandoah/shenandoahTaskqueue.inline.hpp" 37 #include "gc/shenandoah/shenandoahUtils.hpp" 38 #include "gc/shenandoah/shenandoahVerifier.hpp" 39 #include "gc/shenandoah/shenandoahYoungGeneration.hpp" 40 #include "gc/shenandoah/heuristics/shenandoahHeuristics.hpp" 41 42 #include "utilities/quickSort.hpp" 43 44 45 class ShenandoahResetUpdateRegionStateClosure : public ShenandoahHeapRegionClosure { 46 private: 47 ShenandoahHeap* _heap; 48 ShenandoahMarkingContext* const _ctx; 49 public: 50 ShenandoahResetUpdateRegionStateClosure() : 51 _heap(ShenandoahHeap::heap()), 52 _ctx(_heap->marking_context()) {} 53 54 void heap_region_do(ShenandoahHeapRegion* r) override { 55 if (_heap->is_bitmap_slice_committed(r)) { 56 _ctx->clear_bitmap(r); 57 } 58 59 if (r->is_active()) { 60 // Reset live data and set TAMS optimistically. We would recheck these under the pause 61 // anyway to capture any updates that happened since now. 62 _ctx->capture_top_at_mark_start(r); 63 r->clear_live_data(); 64 } 65 } 66 67 bool is_thread_safe() override { return true; } 68 }; 69 70 class ShenandoahResetBitmapTask : public ShenandoahHeapRegionClosure { 71 private: 72 ShenandoahHeap* _heap; 73 ShenandoahMarkingContext* const _ctx; 74 public: 75 ShenandoahResetBitmapTask() : 76 _heap(ShenandoahHeap::heap()), 77 _ctx(_heap->marking_context()) {} 78 79 void heap_region_do(ShenandoahHeapRegion* region) { 80 if (_heap->is_bitmap_slice_committed(region)) { 81 _ctx->clear_bitmap(region); 82 } 83 } 84 85 bool is_thread_safe() { return true; } 86 }; 87 88 // Copy the write-version of the card-table into the read-version, clearing the 89 // write-copy. 90 class ShenandoahMergeWriteTable: public ShenandoahHeapRegionClosure { 91 private: 92 RememberedScanner* _scanner; 93 public: 94 ShenandoahMergeWriteTable(RememberedScanner* scanner) : _scanner(scanner) {} 95 96 void heap_region_do(ShenandoahHeapRegion* r) override { 97 assert(r->is_old(), "Don't waste time doing this for non-old regions"); 98 _scanner->merge_write_table(r->bottom(), ShenandoahHeapRegion::region_size_words()); 99 } 100 101 bool is_thread_safe() override { 102 return true; 103 } 104 }; 105 106 class ShenandoahCopyWriteCardTableToRead: public ShenandoahHeapRegionClosure { 107 private: 108 RememberedScanner* _scanner; 109 public: 110 ShenandoahCopyWriteCardTableToRead(RememberedScanner* scanner) : _scanner(scanner) {} 111 112 void heap_region_do(ShenandoahHeapRegion* region) override { 113 assert(region->is_old(), "Don't waste time doing this for non-old regions"); 114 _scanner->reset_remset(region->bottom(), ShenandoahHeapRegion::region_size_words()); 115 } 116 117 bool is_thread_safe() override { return true; } 118 }; 119 120 void ShenandoahGeneration::confirm_heuristics_mode() { 121 if (_heuristics->is_diagnostic() && !UnlockDiagnosticVMOptions) { 122 vm_exit_during_initialization( 123 err_msg("Heuristics \"%s\" is diagnostic, and must be enabled via -XX:+UnlockDiagnosticVMOptions.", 124 _heuristics->name())); 125 } 126 if (_heuristics->is_experimental() && !UnlockExperimentalVMOptions) { 127 vm_exit_during_initialization( 128 err_msg("Heuristics \"%s\" is experimental, and must be enabled via -XX:+UnlockExperimentalVMOptions.", 129 _heuristics->name())); 130 } 131 } 132 133 ShenandoahHeuristics* ShenandoahGeneration::initialize_heuristics(ShenandoahMode* gc_mode) { 134 _heuristics = gc_mode->initialize_heuristics(this); 135 _heuristics->set_guaranteed_gc_interval(ShenandoahGuaranteedGCInterval); 136 confirm_heuristics_mode(); 137 return _heuristics; 138 } 139 140 size_t ShenandoahGeneration::bytes_allocated_since_gc_start() const { 141 return Atomic::load(&_bytes_allocated_since_gc_start); 142 } 143 144 void ShenandoahGeneration::reset_bytes_allocated_since_gc_start() { 145 Atomic::store(&_bytes_allocated_since_gc_start, (size_t)0); 146 } 147 148 void ShenandoahGeneration::increase_allocated(size_t bytes) { 149 Atomic::add(&_bytes_allocated_since_gc_start, bytes, memory_order_relaxed); 150 } 151 152 void ShenandoahGeneration::set_evacuation_reserve(size_t new_val) { 153 _evacuation_reserve = new_val; 154 } 155 156 size_t ShenandoahGeneration::get_evacuation_reserve() const { 157 return _evacuation_reserve; 158 } 159 160 void ShenandoahGeneration::augment_evacuation_reserve(size_t increment) { 161 _evacuation_reserve += increment; 162 } 163 164 void ShenandoahGeneration::log_status(const char *msg) const { 165 typedef LogTarget(Info, gc, ergo) LogGcInfo; 166 167 if (!LogGcInfo::is_enabled()) { 168 return; 169 } 170 171 // Not under a lock here, so read each of these once to make sure 172 // byte size in proper unit and proper unit for byte size are consistent. 173 size_t v_used = used(); 174 size_t v_used_regions = used_regions_size(); 175 size_t v_soft_max_capacity = soft_max_capacity(); 176 size_t v_max_capacity = max_capacity(); 177 size_t v_available = available(); 178 size_t v_humongous_waste = get_humongous_waste(); 179 LogGcInfo::print("%s: %s generation used: " SIZE_FORMAT "%s, used regions: " SIZE_FORMAT "%s, " 180 "humongous waste: " SIZE_FORMAT "%s, soft capacity: " SIZE_FORMAT "%s, max capacity: " SIZE_FORMAT "%s, " 181 "available: " SIZE_FORMAT "%s", msg, name(), 182 byte_size_in_proper_unit(v_used), proper_unit_for_byte_size(v_used), 183 byte_size_in_proper_unit(v_used_regions), proper_unit_for_byte_size(v_used_regions), 184 byte_size_in_proper_unit(v_humongous_waste), proper_unit_for_byte_size(v_humongous_waste), 185 byte_size_in_proper_unit(v_soft_max_capacity), proper_unit_for_byte_size(v_soft_max_capacity), 186 byte_size_in_proper_unit(v_max_capacity), proper_unit_for_byte_size(v_max_capacity), 187 byte_size_in_proper_unit(v_available), proper_unit_for_byte_size(v_available)); 188 } 189 190 void ShenandoahGeneration::reset_mark_bitmap() { 191 ShenandoahHeap* heap = ShenandoahHeap::heap(); 192 heap->assert_gc_workers(heap->workers()->active_workers()); 193 194 set_mark_incomplete(); 195 196 ShenandoahResetBitmapTask task; 197 parallel_heap_region_iterate(&task); 198 } 199 200 // The ideal is to swap the remembered set so the safepoint effort is no more than a few pointer manipulations. 201 // However, limitations in the implementation of the mutator write-barrier make it difficult to simply change the 202 // location of the card table. So the interim implementation of swap_remembered_set will copy the write-table 203 // onto the read-table and will then clear the write-table. 204 void ShenandoahGeneration::swap_remembered_set() { 205 // Must be sure that marking is complete before we swap remembered set. 206 ShenandoahGenerationalHeap* heap = ShenandoahGenerationalHeap::heap(); 207 heap->assert_gc_workers(heap->workers()->active_workers()); 208 shenandoah_assert_safepoint(); 209 210 // TODO: Eventually, we want to replace this with a constant-time exchange of pointers. 211 ShenandoahOldGeneration* old_generation = heap->old_generation(); 212 ShenandoahCopyWriteCardTableToRead task(old_generation->card_scan()); 213 old_generation->parallel_heap_region_iterate(&task); 214 } 215 216 // Copy the write-version of the card-table into the read-version, clearing the 217 // write-version. The work is done at a safepoint and in parallel by the GC 218 // worker threads. 219 void ShenandoahGeneration::merge_write_table() { 220 // This should only happen for degenerated cycles 221 ShenandoahGenerationalHeap* heap = ShenandoahGenerationalHeap::heap(); 222 heap->assert_gc_workers(heap->workers()->active_workers()); 223 shenandoah_assert_safepoint(); 224 225 ShenandoahOldGeneration* old_generation = heap->old_generation(); 226 ShenandoahMergeWriteTable task(old_generation->card_scan()); 227 old_generation->parallel_heap_region_iterate(&task); 228 } 229 230 void ShenandoahGeneration::prepare_gc() { 231 // Invalidate the marking context 232 set_mark_incomplete(); 233 234 // Capture Top At Mark Start for this generation (typically young) and reset mark bitmap. 235 ShenandoahResetUpdateRegionStateClosure cl; 236 parallel_region_iterate_free(&cl); 237 } 238 239 void ShenandoahGeneration::parallel_region_iterate_free(ShenandoahHeapRegionClosure* cl) { 240 ShenandoahHeap::heap()->parallel_heap_region_iterate(cl); 241 } 242 243 void ShenandoahGeneration::compute_evacuation_budgets(ShenandoahHeap* const heap) { 244 shenandoah_assert_generational(); 245 246 ShenandoahOldGeneration* const old_generation = heap->old_generation(); 247 ShenandoahYoungGeneration* const young_generation = heap->young_generation(); 248 249 // During initialization and phase changes, it is more likely that fewer objects die young and old-gen 250 // memory is not yet full (or is in the process of being replaced). During these times especially, it 251 // is beneficial to loan memory from old-gen to young-gen during the evacuation and update-refs phases 252 // of execution. 253 254 // Calculate EvacuationReserve before PromotionReserve. Evacuation is more critical than promotion. 255 // If we cannot evacuate old-gen, we will not be able to reclaim old-gen memory. Promotions are less 256 // critical. If we cannot promote, there may be degradation of young-gen memory because old objects 257 // accumulate there until they can be promoted. This increases the young-gen marking and evacuation work. 258 259 // First priority is to reclaim the easy garbage out of young-gen. 260 261 // maximum_young_evacuation_reserve is upper bound on memory to be evacuated out of young 262 const size_t maximum_young_evacuation_reserve = (young_generation->max_capacity() * ShenandoahEvacReserve) / 100; 263 const size_t young_evacuation_reserve = MIN2(maximum_young_evacuation_reserve, young_generation->available_with_reserve()); 264 265 // maximum_old_evacuation_reserve is an upper bound on memory evacuated from old and evacuated to old (promoted), 266 // clamped by the old generation space available. 267 // 268 // Here's the algebra. 269 // Let SOEP = ShenandoahOldEvacRatioPercent, 270 // OE = old evac, 271 // YE = young evac, and 272 // TE = total evac = OE + YE 273 // By definition: 274 // SOEP/100 = OE/TE 275 // = OE/(OE+YE) 276 // => SOEP/(100-SOEP) = OE/((OE+YE)-OE) // componendo-dividendo: If a/b = c/d, then a/(b-a) = c/(d-c) 277 // = OE/YE 278 // => OE = YE*SOEP/(100-SOEP) 279 280 // We have to be careful in the event that SOEP is set to 100 by the user. 281 assert(ShenandoahOldEvacRatioPercent <= 100, "Error"); 282 const size_t old_available = old_generation->available(); 283 const size_t maximum_old_evacuation_reserve = (ShenandoahOldEvacRatioPercent == 100) ? 284 old_available : MIN2((maximum_young_evacuation_reserve * ShenandoahOldEvacRatioPercent) / (100 - ShenandoahOldEvacRatioPercent), 285 old_available); 286 287 288 // Second priority is to reclaim garbage out of old-gen if there are old-gen collection candidates. Third priority 289 // is to promote as much as we have room to promote. However, if old-gen memory is in short supply, this means young 290 // GC is operating under "duress" and was unable to transfer the memory that we would normally expect. In this case, 291 // old-gen will refrain from compacting itself in order to allow a quicker young-gen cycle (by avoiding the update-refs 292 // through ALL of old-gen). If there is some memory available in old-gen, we will use this for promotions as promotions 293 // do not add to the update-refs burden of GC. 294 295 size_t old_evacuation_reserve, old_promo_reserve; 296 if (is_global()) { 297 // Global GC is typically triggered by user invocation of System.gc(), and typically indicates that there is lots 298 // of garbage to be reclaimed because we are starting a new phase of execution. Marking for global GC may take 299 // significantly longer than typical young marking because we must mark through all old objects. To expedite 300 // evacuation and update-refs, we give emphasis to reclaiming garbage first, wherever that garbage is found. 301 // Global GC will adjust generation sizes to accommodate the collection set it chooses. 302 303 // Set old_promo_reserve to enforce that no regions are preselected for promotion. Such regions typically 304 // have relatively high memory utilization. We still call select_aged_regions() because this will prepare for 305 // promotions in place, if relevant. 306 old_promo_reserve = 0; 307 308 // Dedicate all available old memory to old_evacuation reserve. This may be small, because old-gen is only 309 // expanded based on an existing mixed evacuation workload at the end of the previous GC cycle. We'll expand 310 // the budget for evacuation of old during GLOBAL cset selection. 311 old_evacuation_reserve = maximum_old_evacuation_reserve; 312 } else if (old_generation->has_unprocessed_collection_candidates()) { 313 // We reserved all old-gen memory at end of previous GC to hold anticipated evacuations to old-gen. If this is 314 // mixed evacuation, reserve all of this memory for compaction of old-gen and do not promote. Prioritize compaction 315 // over promotion in order to defragment OLD so that it will be better prepared to efficiently receive promoted memory. 316 old_evacuation_reserve = maximum_old_evacuation_reserve; 317 old_promo_reserve = 0; 318 } else { 319 // Make all old-evacuation memory for promotion, but if we can't use it all for promotion, we'll allow some evacuation. 320 old_evacuation_reserve = 0; 321 old_promo_reserve = maximum_old_evacuation_reserve; 322 } 323 assert(old_evacuation_reserve <= old_available, "Error"); 324 325 // We see too many old-evacuation failures if we force ourselves to evacuate into regions that are not initially empty. 326 // So we limit the old-evacuation reserve to unfragmented memory. Even so, old-evacuation is free to fill in nooks and 327 // crannies within existing partially used regions and it generally tries to do so. 328 const size_t old_free_unfragmented = old_generation->free_unaffiliated_regions() * ShenandoahHeapRegion::region_size_bytes(); 329 if (old_evacuation_reserve > old_free_unfragmented) { 330 const size_t delta = old_evacuation_reserve - old_free_unfragmented; 331 old_evacuation_reserve -= delta; 332 // Let promo consume fragments of old-gen memory if not global 333 if (!is_global()) { 334 old_promo_reserve += delta; 335 } 336 } 337 338 // Preselect regions for promotion by evacuation (obtaining the live data to seed promoted_reserve), 339 // and identify regions that will promote in place. These use the tenuring threshold. 340 const size_t consumed_by_advance_promotion = select_aged_regions(old_promo_reserve); 341 assert(consumed_by_advance_promotion <= maximum_old_evacuation_reserve, "Cannot promote more than available old-gen memory"); 342 343 // Note that unused old_promo_reserve might not be entirely consumed_by_advance_promotion. Do not transfer this 344 // to old_evacuation_reserve because this memory is likely very fragmented, and we do not want to increase the likelihood 345 // of old evacuation failure. 346 young_generation->set_evacuation_reserve(young_evacuation_reserve); 347 old_generation->set_evacuation_reserve(old_evacuation_reserve); 348 old_generation->set_promoted_reserve(consumed_by_advance_promotion); 349 350 // There is no need to expand OLD because all memory used here was set aside at end of previous GC, except in the 351 // case of a GLOBAL gc. During choose_collection_set() of GLOBAL, old will be expanded on demand. 352 } 353 354 // Having chosen the collection set, adjust the budgets for generational mode based on its composition. Note 355 // that young_generation->available() now knows about recently discovered immediate garbage. 356 // 357 void ShenandoahGeneration::adjust_evacuation_budgets(ShenandoahHeap* const heap, ShenandoahCollectionSet* const collection_set) { 358 shenandoah_assert_generational(); 359 // We may find that old_evacuation_reserve and/or loaned_for_young_evacuation are not fully consumed, in which case we may 360 // be able to increase regions_available_to_loan 361 362 // The role of adjust_evacuation_budgets() is to compute the correct value of regions_available_to_loan and to make 363 // effective use of this memory, including the remnant memory within these regions that may result from rounding loan to 364 // integral number of regions. Excess memory that is available to be loaned is applied to an allocation supplement, 365 // which allows mutators to allocate memory beyond the current capacity of young-gen on the promise that the loan 366 // will be repaid as soon as we finish updating references for the recently evacuated collection set. 367 368 // We cannot recalculate regions_available_to_loan by simply dividing old_generation->available() by region_size_bytes 369 // because the available memory may be distributed between many partially occupied regions that are already holding old-gen 370 // objects. Memory in partially occupied regions is not "available" to be loaned. Note that an increase in old-gen 371 // available that results from a decrease in memory consumed by old evacuation is not necessarily available to be loaned 372 // to young-gen. 373 374 size_t region_size_bytes = ShenandoahHeapRegion::region_size_bytes(); 375 ShenandoahOldGeneration* const old_generation = heap->old_generation(); 376 ShenandoahYoungGeneration* const young_generation = heap->young_generation(); 377 378 size_t old_evacuated = collection_set->get_old_bytes_reserved_for_evacuation(); 379 size_t old_evacuated_committed = (size_t) (ShenandoahOldEvacWaste * old_evacuated); 380 size_t old_evacuation_reserve = old_generation->get_evacuation_reserve(); 381 382 if (old_evacuated_committed > old_evacuation_reserve) { 383 // This should only happen due to round-off errors when enforcing ShenandoahOldEvacWaste 384 assert(old_evacuated_committed <= (33 * old_evacuation_reserve) / 32, 385 "Round-off errors should be less than 3.125%%, committed: " SIZE_FORMAT ", reserved: " SIZE_FORMAT, 386 old_evacuated_committed, old_evacuation_reserve); 387 old_evacuated_committed = old_evacuation_reserve; 388 // Leave old_evac_reserve as previously configured 389 } else if (old_evacuated_committed < old_evacuation_reserve) { 390 // This happens if the old-gen collection consumes less than full budget. 391 old_evacuation_reserve = old_evacuated_committed; 392 old_generation->set_evacuation_reserve(old_evacuation_reserve); 393 } 394 395 size_t young_advance_promoted = collection_set->get_young_bytes_to_be_promoted(); 396 size_t young_advance_promoted_reserve_used = (size_t) (ShenandoahPromoEvacWaste * young_advance_promoted); 397 398 size_t young_evacuated = collection_set->get_young_bytes_reserved_for_evacuation(); 399 size_t young_evacuated_reserve_used = (size_t) (ShenandoahEvacWaste * young_evacuated); 400 401 size_t total_young_available = young_generation->available_with_reserve(); 402 assert(young_evacuated_reserve_used <= total_young_available, "Cannot evacuate more than is available in young"); 403 young_generation->set_evacuation_reserve(young_evacuated_reserve_used); 404 405 size_t old_available = old_generation->available(); 406 // Now that we've established the collection set, we know how much memory is really required by old-gen for evacuation 407 // and promotion reserves. Try shrinking OLD now in case that gives us a bit more runway for mutator allocations during 408 // evac and update phases. 409 size_t old_consumed = old_evacuated_committed + young_advance_promoted_reserve_used; 410 411 if (old_available < old_consumed) { 412 // This can happen due to round-off errors when adding the results of truncated integer arithmetic. 413 // We've already truncated old_evacuated_committed. Truncate young_advance_promoted_reserve_used here. 414 assert(young_advance_promoted_reserve_used <= (33 * (old_available - old_evacuated_committed)) / 32, 415 "Round-off errors should be less than 3.125%%, committed: " SIZE_FORMAT ", reserved: " SIZE_FORMAT, 416 young_advance_promoted_reserve_used, old_available - old_evacuated_committed); 417 young_advance_promoted_reserve_used = old_available - old_evacuated_committed; 418 old_consumed = old_evacuated_committed + young_advance_promoted_reserve_used; 419 } 420 421 assert(old_available >= old_consumed, "Cannot consume (" SIZE_FORMAT ") more than is available (" SIZE_FORMAT ")", 422 old_consumed, old_available); 423 size_t excess_old = old_available - old_consumed; 424 size_t unaffiliated_old_regions = old_generation->free_unaffiliated_regions(); 425 size_t unaffiliated_old = unaffiliated_old_regions * region_size_bytes; 426 assert(old_available >= unaffiliated_old, "Unaffiliated old is a subset of old available"); 427 428 // Make sure old_evac_committed is unaffiliated 429 if (old_evacuated_committed > 0) { 430 if (unaffiliated_old > old_evacuated_committed) { 431 size_t giveaway = unaffiliated_old - old_evacuated_committed; 432 size_t giveaway_regions = giveaway / region_size_bytes; // round down 433 if (giveaway_regions > 0) { 434 excess_old = MIN2(excess_old, giveaway_regions * region_size_bytes); 435 } else { 436 excess_old = 0; 437 } 438 } else { 439 excess_old = 0; 440 } 441 } 442 443 // If we find that OLD has excess regions, give them back to YOUNG now to reduce likelihood we run out of allocation 444 // runway during evacuation and update-refs. 445 size_t regions_to_xfer = 0; 446 if (excess_old > unaffiliated_old) { 447 // we can give back unaffiliated_old (all of unaffiliated is excess) 448 if (unaffiliated_old_regions > 0) { 449 regions_to_xfer = unaffiliated_old_regions; 450 } 451 } else if (unaffiliated_old_regions > 0) { 452 // excess_old < unaffiliated old: we can give back MIN(excess_old/region_size_bytes, unaffiliated_old_regions) 453 size_t excess_regions = excess_old / region_size_bytes; 454 size_t regions_to_xfer = MIN2(excess_regions, unaffiliated_old_regions); 455 } 456 457 if (regions_to_xfer > 0) { 458 bool result = ShenandoahGenerationalHeap::cast(heap)->generation_sizer()->transfer_to_young(regions_to_xfer); 459 assert(excess_old > regions_to_xfer * region_size_bytes, "Cannot xfer more than excess old"); 460 excess_old -= regions_to_xfer * region_size_bytes; 461 log_info(gc, ergo)("%s transferred " SIZE_FORMAT " excess regions to young before start of evacuation", 462 result? "Successfully": "Unsuccessfully", regions_to_xfer); 463 } 464 465 // Add in the excess_old memory to hold unanticipated promotions, if any. If there are more unanticipated 466 // promotions than fit in reserved memory, they will be deferred until a future GC pass. 467 size_t total_promotion_reserve = young_advance_promoted_reserve_used + excess_old; 468 old_generation->set_promoted_reserve(total_promotion_reserve); 469 old_generation->reset_promoted_expended(); 470 } 471 472 typedef struct { 473 ShenandoahHeapRegion* _region; 474 size_t _live_data; 475 } AgedRegionData; 476 477 static int compare_by_aged_live(AgedRegionData a, AgedRegionData b) { 478 if (a._live_data < b._live_data) 479 return -1; 480 else if (a._live_data > b._live_data) 481 return 1; 482 else return 0; 483 } 484 485 inline void assert_no_in_place_promotions() { 486 #ifdef ASSERT 487 class ShenandoahNoInPlacePromotions : public ShenandoahHeapRegionClosure { 488 public: 489 void heap_region_do(ShenandoahHeapRegion *r) override { 490 assert(r->get_top_before_promote() == nullptr, 491 "Region " SIZE_FORMAT " should not be ready for in-place promotion", r->index()); 492 } 493 } cl; 494 ShenandoahHeap::heap()->heap_region_iterate(&cl); 495 #endif 496 } 497 498 // Preselect for inclusion into the collection set regions whose age is at or above tenure age which contain more than 499 // ShenandoahOldGarbageThreshold amounts of garbage. We identify these regions by setting the appropriate entry of 500 // the collection set's preselected regions array to true. All entries are initialized to false before calling this 501 // function. 502 // 503 // During the subsequent selection of the collection set, we give priority to these promotion set candidates. 504 // Without this prioritization, we found that the aged regions tend to be ignored because they typically have 505 // much less garbage and much more live data than the recently allocated "eden" regions. When aged regions are 506 // repeatedly excluded from the collection set, the amount of live memory within the young generation tends to 507 // accumulate and this has the undesirable side effect of causing young-generation collections to require much more 508 // CPU and wall-clock time. 509 // 510 // A second benefit of treating aged regions differently than other regions during collection set selection is 511 // that this allows us to more accurately budget memory to hold the results of evacuation. Memory for evacuation 512 // of aged regions must be reserved in the old generation. Memory for evacuation of all other regions must be 513 // reserved in the young generation. 514 size_t ShenandoahGeneration::select_aged_regions(size_t old_available) { 515 516 // There should be no regions configured for subsequent in-place-promotions carried over from the previous cycle. 517 assert_no_in_place_promotions(); 518 519 auto const heap = ShenandoahGenerationalHeap::heap(); 520 bool* const candidate_regions_for_promotion_by_copy = heap->collection_set()->preselected_regions(); 521 ShenandoahMarkingContext* const ctx = heap->marking_context(); 522 523 const uint tenuring_threshold = heap->age_census()->tenuring_threshold(); 524 const size_t old_garbage_threshold = (ShenandoahHeapRegion::region_size_bytes() * ShenandoahOldGarbageThreshold) / 100; 525 526 size_t old_consumed = 0; 527 size_t promo_potential = 0; 528 size_t candidates = 0; 529 530 // Tracks the padding of space above top in regions eligible for promotion in place 531 size_t promote_in_place_pad = 0; 532 533 // Sort the promotion-eligible regions in order of increasing live-data-bytes so that we can first reclaim regions that require 534 // less evacuation effort. This prioritizes garbage first, expanding the allocation pool early before we reclaim regions that 535 // have more live data. 536 const size_t num_regions = heap->num_regions(); 537 538 ResourceMark rm; 539 AgedRegionData* sorted_regions = NEW_RESOURCE_ARRAY(AgedRegionData, num_regions); 540 541 for (size_t i = 0; i < num_regions; i++) { 542 ShenandoahHeapRegion* const r = heap->get_region(i); 543 if (r->is_empty() || !r->has_live() || !r->is_young() || !r->is_regular()) { 544 // skip over regions that aren't regular young with some live data 545 continue; 546 } 547 if (r->age() >= tenuring_threshold) { 548 if ((r->garbage() < old_garbage_threshold)) { 549 // This tenure-worthy region has too little garbage, so we do not want to expend the copying effort to 550 // reclaim the garbage; instead this region may be eligible for promotion-in-place to the 551 // old generation. 552 HeapWord* tams = ctx->top_at_mark_start(r); 553 HeapWord* original_top = r->top(); 554 if (!heap->is_concurrent_old_mark_in_progress() && tams == original_top) { 555 // No allocations from this region have been made during concurrent mark. It meets all the criteria 556 // for in-place-promotion. Though we only need the value of top when we fill the end of the region, 557 // we use this field to indicate that this region should be promoted in place during the evacuation 558 // phase. 559 r->save_top_before_promote(); 560 561 size_t remnant_size = r->free() / HeapWordSize; 562 if (remnant_size > ShenandoahHeap::min_fill_size()) { 563 ShenandoahHeap::fill_with_object(original_top, remnant_size); 564 // Fill the remnant memory within this region to assure no allocations prior to promote in place. Otherwise, 565 // newly allocated objects will not be parsable when promote in place tries to register them. Furthermore, any 566 // new allocations would not necessarily be eligible for promotion. This addresses both issues. 567 r->set_top(r->end()); 568 promote_in_place_pad += remnant_size * HeapWordSize; 569 } else { 570 // Since the remnant is so small that it cannot be filled, we don't have to worry about any accidental 571 // allocations occurring within this region before the region is promoted in place. 572 } 573 } 574 // Else, we do not promote this region (either in place or by copy) because it has received new allocations. 575 576 // During evacuation, we exclude from promotion regions for which age > tenure threshold, garbage < garbage-threshold, 577 // and get_top_before_promote() != tams 578 } else { 579 // Record this promotion-eligible candidate region. After sorting and selecting the best candidates below, 580 // we may still decide to exclude this promotion-eligible region from the current collection set. If this 581 // happens, we will consider this region as part of the anticipated promotion potential for the next GC 582 // pass; see further below. 583 sorted_regions[candidates]._region = r; 584 sorted_regions[candidates++]._live_data = r->get_live_data_bytes(); 585 } 586 } else { 587 // We only evacuate & promote objects from regular regions whose garbage() is above old-garbage-threshold. 588 // Objects in tenure-worthy regions with less garbage are promoted in place. These take a different path to 589 // old-gen. Regions excluded from promotion because their garbage content is too low (causing us to anticipate that 590 // the region would be promoted in place) may be eligible for evacuation promotion by the time promotion takes 591 // place during a subsequent GC pass because more garbage is found within the region between now and then. This 592 // should not happen if we are properly adapting the tenure age. The theory behind adaptive tenuring threshold 593 // is to choose the youngest age that demonstrates no "significant" further loss of population since the previous 594 // age. If not this, we expect the tenure age to demonstrate linear population decay for at least two population 595 // samples, whereas we expect to observe exponential population decay for ages younger than the tenure age. 596 // 597 // In the case that certain regions which were anticipated to be promoted in place need to be promoted by 598 // evacuation, it may be the case that there is not sufficient reserve within old-gen to hold evacuation of 599 // these regions. The likely outcome is that these regions will not be selected for evacuation or promotion 600 // in the current cycle and we will anticipate that they will be promoted in the next cycle. This will cause 601 // us to reserve more old-gen memory so that these objects can be promoted in the subsequent cycle. 602 // 603 // TODO: 604 // If we are auto-tuning the tenure age and regions that were anticipated to be promoted in place end up 605 // being promoted by evacuation, this event should feed into the tenure-age-selection heuristic so that 606 // the tenure age can be increased. 607 if (heap->is_aging_cycle() && (r->age() + 1 == tenuring_threshold)) { 608 if (r->garbage() >= old_garbage_threshold) { 609 promo_potential += r->get_live_data_bytes(); 610 } 611 } 612 } 613 // Note that we keep going even if one region is excluded from selection. 614 // Subsequent regions may be selected if they have smaller live data. 615 } 616 // Sort in increasing order according to live data bytes. Note that candidates represents the number of regions 617 // that qualify to be promoted by evacuation. 618 if (candidates > 0) { 619 size_t selected_regions = 0; 620 size_t selected_live = 0; 621 QuickSort::sort<AgedRegionData>(sorted_regions, candidates, compare_by_aged_live, false); 622 for (size_t i = 0; i < candidates; i++) { 623 ShenandoahHeapRegion* const region = sorted_regions[i]._region; 624 size_t region_live_data = sorted_regions[i]._live_data; 625 size_t promotion_need = (size_t) (region_live_data * ShenandoahPromoEvacWaste); 626 if (old_consumed + promotion_need <= old_available) { 627 old_consumed += promotion_need; 628 candidate_regions_for_promotion_by_copy[region->index()] = true; 629 selected_regions++; 630 selected_live += region_live_data; 631 } else { 632 // We rejected this promotable region from the collection set because we had no room to hold its copy. 633 // Add this region to promo potential for next GC. 634 promo_potential += region_live_data; 635 assert(!candidate_regions_for_promotion_by_copy[region->index()], "Shouldn't be selected"); 636 } 637 // We keep going even if one region is excluded from selection because we need to accumulate all eligible 638 // regions that are not preselected into promo_potential 639 } 640 log_info(gc)("Preselected " SIZE_FORMAT " regions containing " SIZE_FORMAT " live bytes," 641 " consuming: " SIZE_FORMAT " of budgeted: " SIZE_FORMAT, 642 selected_regions, selected_live, old_consumed, old_available); 643 } 644 645 heap->old_generation()->set_pad_for_promote_in_place(promote_in_place_pad); 646 heap->old_generation()->set_promotion_potential(promo_potential); 647 return old_consumed; 648 } 649 650 void ShenandoahGeneration::prepare_regions_and_collection_set(bool concurrent) { 651 ShenandoahHeap* heap = ShenandoahHeap::heap(); 652 ShenandoahCollectionSet* collection_set = heap->collection_set(); 653 bool is_generational = heap->mode()->is_generational(); 654 655 assert(!heap->is_full_gc_in_progress(), "Only for concurrent and degenerated GC"); 656 assert(!is_old(), "Only YOUNG and GLOBAL GC perform evacuations"); 657 { 658 ShenandoahGCPhase phase(concurrent ? ShenandoahPhaseTimings::final_update_region_states : 659 ShenandoahPhaseTimings::degen_gc_final_update_region_states); 660 ShenandoahFinalMarkUpdateRegionStateClosure cl(complete_marking_context()); 661 parallel_heap_region_iterate(&cl); 662 663 if (is_young()) { 664 // We always need to update the watermark for old regions. If there 665 // are mixed collections pending, we also need to synchronize the 666 // pinned status for old regions. Since we are already visiting every 667 // old region here, go ahead and sync the pin status too. 668 ShenandoahFinalMarkUpdateRegionStateClosure old_cl(nullptr); 669 heap->old_generation()->parallel_heap_region_iterate(&old_cl); 670 } 671 } 672 673 // Tally the census counts and compute the adaptive tenuring threshold 674 if (is_generational && ShenandoahGenerationalAdaptiveTenuring && !ShenandoahGenerationalCensusAtEvac) { 675 // Objects above TAMS weren't included in the age census. Since they were all 676 // allocated in this cycle they belong in the age 0 cohort. We walk over all 677 // young regions and sum the volume of objects between TAMS and top. 678 ShenandoahUpdateCensusZeroCohortClosure age0_cl(complete_marking_context()); 679 heap->young_generation()->heap_region_iterate(&age0_cl); 680 size_t age0_pop = age0_cl.get_age0_population(); 681 682 // Update the global census, including the missed age 0 cohort above, 683 // along with the census done during marking, and compute the tenuring threshold. 684 ShenandoahAgeCensus* census = ShenandoahGenerationalHeap::heap()->age_census(); 685 census->update_census(age0_pop); 686 #ifndef PRODUCT 687 size_t total_pop = age0_cl.get_total_population(); 688 size_t total_census = census->get_total(); 689 // Usually total_pop > total_census, but not by too much. 690 // We use integer division so anything up to just less than 2 is considered 691 // reasonable, and the "+1" is to avoid divide-by-zero. 692 assert((total_pop+1)/(total_census+1) == 1, "Extreme divergence: " 693 SIZE_FORMAT "/" SIZE_FORMAT, total_pop, total_census); 694 #endif 695 } 696 697 { 698 ShenandoahGCPhase phase(concurrent ? ShenandoahPhaseTimings::choose_cset : 699 ShenandoahPhaseTimings::degen_gc_choose_cset); 700 701 collection_set->clear(); 702 ShenandoahHeapLocker locker(heap->lock()); 703 if (is_generational) { 704 // Seed the collection set with resource area-allocated 705 // preselected regions, which are removed when we exit this scope. 706 ResourceMark rm; 707 ShenandoahCollectionSetPreselector preselector(collection_set, heap->num_regions()); 708 709 // TODO: young_available can include available (between top() and end()) within each young region that is not 710 // part of the collection set. Making this memory available to the young_evacuation_reserve allows a larger 711 // young collection set to be chosen when available memory is under extreme pressure. Implementing this "improvement" 712 // is tricky, because the incremental construction of the collection set actually changes the amount of memory 713 // available to hold evacuated young-gen objects. As currently implemented, the memory that is available within 714 // non-empty regions that are not selected as part of the collection set can be allocated by the mutator while 715 // GC is evacuating and updating references. 716 717 // Find the amount that will be promoted, regions that will be promoted in 718 // place, and preselect older regions that will be promoted by evacuation. 719 compute_evacuation_budgets(heap); 720 721 // Choose the collection set, including the regions preselected above for 722 // promotion into the old generation. 723 _heuristics->choose_collection_set(collection_set); 724 if (!collection_set->is_empty()) { 725 // only make use of evacuation budgets when we are evacuating 726 adjust_evacuation_budgets(heap, collection_set); 727 } 728 729 if (is_global()) { 730 // We have just chosen a collection set for a global cycle. The mark bitmap covering old regions is complete, so 731 // the remembered set scan can use that to avoid walking into garbage. When the next old mark begins, we will 732 // use the mark bitmap to make the old regions parsable by coalescing and filling any unmarked objects. Thus, 733 // we prepare for old collections by remembering which regions are old at this time. Note that any objects 734 // promoted into old regions will be above TAMS, and so will be considered marked. However, free regions that 735 // become old after this point will not be covered correctly by the mark bitmap, so we must be careful not to 736 // coalesce those regions. Only the old regions which are not part of the collection set at this point are 737 // eligible for coalescing. As implemented now, this has the side effect of possibly initiating mixed-evacuations 738 // after a global cycle for old regions that were not included in this collection set. 739 heap->old_generation()->prepare_for_mixed_collections_after_global_gc(); 740 } 741 } else { 742 _heuristics->choose_collection_set(collection_set); 743 } 744 } 745 746 747 { 748 ShenandoahGCPhase phase(concurrent ? ShenandoahPhaseTimings::final_rebuild_freeset : 749 ShenandoahPhaseTimings::degen_gc_final_rebuild_freeset); 750 ShenandoahHeapLocker locker(heap->lock()); 751 size_t young_cset_regions, old_cset_regions; 752 753 // We are preparing for evacuation. At this time, we ignore cset region tallies. 754 size_t first_old, last_old, num_old; 755 heap->free_set()->prepare_to_rebuild(young_cset_regions, old_cset_regions, first_old, last_old, num_old); 756 // Free set construction uses reserve quantities, because they are known to be valid here 757 heap->free_set()->finish_rebuild(young_cset_regions, old_cset_regions, num_old, true); 758 } 759 } 760 761 bool ShenandoahGeneration::is_bitmap_clear() { 762 ShenandoahHeap* heap = ShenandoahHeap::heap(); 763 ShenandoahMarkingContext* context = heap->marking_context(); 764 const size_t num_regions = heap->num_regions(); 765 for (size_t idx = 0; idx < num_regions; idx++) { 766 ShenandoahHeapRegion* r = heap->get_region(idx); 767 if (contains(r) && r->is_affiliated()) { 768 if (heap->is_bitmap_slice_committed(r) && (context->top_at_mark_start(r) > r->bottom()) && 769 !context->is_bitmap_clear_range(r->bottom(), r->end())) { 770 return false; 771 } 772 } 773 } 774 return true; 775 } 776 777 bool ShenandoahGeneration::is_mark_complete() { 778 return _is_marking_complete.is_set(); 779 } 780 781 void ShenandoahGeneration::set_mark_complete() { 782 _is_marking_complete.set(); 783 } 784 785 void ShenandoahGeneration::set_mark_incomplete() { 786 _is_marking_complete.unset(); 787 } 788 789 ShenandoahMarkingContext* ShenandoahGeneration::complete_marking_context() { 790 assert(is_mark_complete(), "Marking must be completed."); 791 return ShenandoahHeap::heap()->marking_context(); 792 } 793 794 void ShenandoahGeneration::cancel_marking() { 795 log_info(gc)("Cancel marking: %s", name()); 796 if (is_concurrent_mark_in_progress()) { 797 set_mark_incomplete(); 798 } 799 _task_queues->clear(); 800 ref_processor()->abandon_partial_discovery(); 801 set_concurrent_mark_in_progress(false); 802 } 803 804 ShenandoahGeneration::ShenandoahGeneration(ShenandoahGenerationType type, 805 uint max_workers, 806 size_t max_capacity, 807 size_t soft_max_capacity) : 808 _type(type), 809 _task_queues(new ShenandoahObjToScanQueueSet(max_workers)), 810 _ref_processor(new ShenandoahReferenceProcessor(MAX2(max_workers, 1U))), 811 _affiliated_region_count(0), _humongous_waste(0), _evacuation_reserve(0), 812 _used(0), _bytes_allocated_since_gc_start(0), 813 _max_capacity(max_capacity), _soft_max_capacity(soft_max_capacity), 814 _heuristics(nullptr) 815 { 816 _is_marking_complete.set(); 817 assert(max_workers > 0, "At least one queue"); 818 for (uint i = 0; i < max_workers; ++i) { 819 ShenandoahObjToScanQueue* task_queue = new ShenandoahObjToScanQueue(); 820 _task_queues->register_queue(i, task_queue); 821 } 822 } 823 824 ShenandoahGeneration::~ShenandoahGeneration() { 825 for (uint i = 0; i < _task_queues->size(); ++i) { 826 ShenandoahObjToScanQueue* q = _task_queues->queue(i); 827 delete q; 828 } 829 delete _task_queues; 830 } 831 832 void ShenandoahGeneration::reserve_task_queues(uint workers) { 833 _task_queues->reserve(workers); 834 } 835 836 ShenandoahObjToScanQueueSet* ShenandoahGeneration::old_gen_task_queues() const { 837 return nullptr; 838 } 839 840 void ShenandoahGeneration::scan_remembered_set(bool is_concurrent) { 841 assert(is_young(), "Should only scan remembered set for young generation."); 842 843 ShenandoahGenerationalHeap* const heap = ShenandoahGenerationalHeap::heap(); 844 uint nworkers = heap->workers()->active_workers(); 845 reserve_task_queues(nworkers); 846 847 ShenandoahReferenceProcessor* rp = ref_processor(); 848 ShenandoahRegionChunkIterator work_list(nworkers); 849 ShenandoahScanRememberedTask task(task_queues(), old_gen_task_queues(), rp, &work_list, is_concurrent); 850 heap->assert_gc_workers(nworkers); 851 heap->workers()->run_task(&task); 852 if (ShenandoahEnableCardStats) { 853 RememberedScanner* scanner = heap->old_generation()->card_scan(); 854 assert(scanner != nullptr, "Not generational"); 855 scanner->log_card_stats(nworkers, CARD_STAT_SCAN_RS); 856 } 857 } 858 859 size_t ShenandoahGeneration::increment_affiliated_region_count() { 860 shenandoah_assert_heaplocked_or_fullgc_safepoint(); 861 // During full gc, multiple GC worker threads may change region affiliations without a lock. No lock is enforced 862 // on read and write of _affiliated_region_count. At the end of full gc, a single thread overwrites the count with 863 // a coherent value. 864 _affiliated_region_count++; 865 return _affiliated_region_count; 866 } 867 868 size_t ShenandoahGeneration::decrement_affiliated_region_count() { 869 shenandoah_assert_heaplocked_or_fullgc_safepoint(); 870 // During full gc, multiple GC worker threads may change region affiliations without a lock. No lock is enforced 871 // on read and write of _affiliated_region_count. At the end of full gc, a single thread overwrites the count with 872 // a coherent value. 873 _affiliated_region_count--; 874 // TODO: REMOVE IS_GLOBAL() QUALIFIER AFTER WE FIX GLOBAL AFFILIATED REGION ACCOUNTING 875 assert(is_global() || ShenandoahHeap::heap()->is_full_gc_in_progress() || 876 (_used + _humongous_waste <= _affiliated_region_count * ShenandoahHeapRegion::region_size_bytes()), 877 "used + humongous cannot exceed regions"); 878 return _affiliated_region_count; 879 } 880 881 size_t ShenandoahGeneration::increase_affiliated_region_count(size_t delta) { 882 shenandoah_assert_heaplocked_or_fullgc_safepoint(); 883 _affiliated_region_count += delta; 884 return _affiliated_region_count; 885 } 886 887 size_t ShenandoahGeneration::decrease_affiliated_region_count(size_t delta) { 888 shenandoah_assert_heaplocked_or_fullgc_safepoint(); 889 assert(_affiliated_region_count >= delta, "Affiliated region count cannot be negative"); 890 891 _affiliated_region_count -= delta; 892 // TODO: REMOVE IS_GLOBAL() QUALIFIER AFTER WE FIX GLOBAL AFFILIATED REGION ACCOUNTING 893 assert(is_global() || ShenandoahHeap::heap()->is_full_gc_in_progress() || 894 (_used + _humongous_waste <= _affiliated_region_count * ShenandoahHeapRegion::region_size_bytes()), 895 "used + humongous cannot exceed regions"); 896 return _affiliated_region_count; 897 } 898 899 void ShenandoahGeneration::establish_usage(size_t num_regions, size_t num_bytes, size_t humongous_waste) { 900 assert(ShenandoahSafepoint::is_at_shenandoah_safepoint(), "must be at a safepoint"); 901 _affiliated_region_count = num_regions; 902 _used = num_bytes; 903 _humongous_waste = humongous_waste; 904 } 905 906 void ShenandoahGeneration::increase_used(size_t bytes) { 907 Atomic::add(&_used, bytes); 908 } 909 910 void ShenandoahGeneration::increase_humongous_waste(size_t bytes) { 911 if (bytes > 0) { 912 Atomic::add(&_humongous_waste, bytes); 913 } 914 } 915 916 void ShenandoahGeneration::decrease_humongous_waste(size_t bytes) { 917 if (bytes > 0) { 918 assert(ShenandoahHeap::heap()->is_full_gc_in_progress() || (_humongous_waste >= bytes), 919 "Waste (" SIZE_FORMAT ") cannot be negative (after subtracting " SIZE_FORMAT ")", _humongous_waste, bytes); 920 Atomic::sub(&_humongous_waste, bytes); 921 } 922 } 923 924 void ShenandoahGeneration::decrease_used(size_t bytes) { 925 // TODO: REMOVE IS_GLOBAL() QUALIFIER AFTER WE FIX GLOBAL AFFILIATED REGION ACCOUNTING 926 assert(is_global() || ShenandoahHeap::heap()->is_full_gc_in_progress() || 927 (_used >= bytes), "cannot reduce bytes used by generation below zero"); 928 Atomic::sub(&_used, bytes); 929 } 930 931 size_t ShenandoahGeneration::used_regions() const { 932 return _affiliated_region_count; 933 } 934 935 size_t ShenandoahGeneration::free_unaffiliated_regions() const { 936 size_t result = max_capacity() / ShenandoahHeapRegion::region_size_bytes(); 937 if (_affiliated_region_count > result) { 938 result = 0; 939 } else { 940 result -= _affiliated_region_count; 941 } 942 return result; 943 } 944 945 size_t ShenandoahGeneration::used_regions_size() const { 946 return _affiliated_region_count * ShenandoahHeapRegion::region_size_bytes(); 947 } 948 949 size_t ShenandoahGeneration::available() const { 950 return available(max_capacity()); 951 } 952 953 // For ShenandoahYoungGeneration, Include the young available that may have been reserved for the Collector. 954 size_t ShenandoahGeneration::available_with_reserve() const { 955 return available(max_capacity()); 956 } 957 958 size_t ShenandoahGeneration::soft_available() const { 959 return available(soft_max_capacity()); 960 } 961 962 size_t ShenandoahGeneration::available(size_t capacity) const { 963 size_t in_use = used() + get_humongous_waste(); 964 return in_use > capacity ? 0 : capacity - in_use; 965 } 966 967 size_t ShenandoahGeneration::increase_capacity(size_t increment) { 968 shenandoah_assert_heaplocked_or_safepoint(); 969 970 // We do not enforce that new capacity >= heap->max_size_for(this). The maximum generation size is treated as a rule of thumb 971 // which may be violated during certain transitions, such as when we are forcing transfers for the purpose of promoting regions 972 // in place. 973 // TODO: REMOVE IS_GLOBAL() QUALIFIER AFTER WE FIX GLOBAL AFFILIATED REGION ACCOUNTING 974 assert(is_global() || ShenandoahHeap::heap()->is_full_gc_in_progress() || 975 (_max_capacity + increment <= ShenandoahHeap::heap()->max_capacity()), "Generation cannot be larger than heap size"); 976 assert(increment % ShenandoahHeapRegion::region_size_bytes() == 0, "Generation capacity must be multiple of region size"); 977 _max_capacity += increment; 978 979 // This detects arithmetic wraparound on _used 980 // TODO: REMOVE IS_GLOBAL() QUALIFIER AFTER WE FIX GLOBAL AFFILIATED REGION ACCOUNTING 981 assert(is_global() || ShenandoahHeap::heap()->is_full_gc_in_progress() || 982 (_affiliated_region_count * ShenandoahHeapRegion::region_size_bytes() >= _used), 983 "Affiliated regions must hold more than what is currently used"); 984 return _max_capacity; 985 } 986 987 size_t ShenandoahGeneration::set_capacity(size_t byte_size) { 988 shenandoah_assert_heaplocked_or_safepoint(); 989 _max_capacity = byte_size; 990 return _max_capacity; 991 } 992 993 size_t ShenandoahGeneration::decrease_capacity(size_t decrement) { 994 shenandoah_assert_heaplocked_or_safepoint(); 995 996 // We do not enforce that new capacity >= heap->min_size_for(this). The minimum generation size is treated as a rule of thumb 997 // which may be violated during certain transitions, such as when we are forcing transfers for the purpose of promoting regions 998 // in place. 999 assert(decrement % ShenandoahHeapRegion::region_size_bytes() == 0, "Generation capacity must be multiple of region size"); 1000 assert(_max_capacity >= decrement, "Generation capacity cannot be negative"); 1001 1002 _max_capacity -= decrement; 1003 1004 // This detects arithmetic wraparound on _used 1005 // TODO: REMOVE IS_GLOBAL() QUALIFIER AFTER WE FIX GLOBAL AFFILIATED REGION ACCOUNTING 1006 assert(is_global() || ShenandoahHeap::heap()->is_full_gc_in_progress() || 1007 (_affiliated_region_count * ShenandoahHeapRegion::region_size_bytes() >= _used), 1008 "Affiliated regions must hold more than what is currently used"); 1009 // TODO: REMOVE IS_GLOBAL() QUALIFIER AFTER WE FIX GLOBAL AFFILIATED REGION ACCOUNTING 1010 assert(is_global() || ShenandoahHeap::heap()->is_full_gc_in_progress() || 1011 (_used <= _max_capacity), "Cannot use more than capacity"); 1012 // TODO: REMOVE IS_GLOBAL() QUALIFIER AFTER WE FIX GLOBAL AFFILIATED REGION ACCOUNTING 1013 assert(is_global() || ShenandoahHeap::heap()->is_full_gc_in_progress() || 1014 (_affiliated_region_count * ShenandoahHeapRegion::region_size_bytes() <= _max_capacity), 1015 "Cannot use more than capacity"); 1016 return _max_capacity; 1017 } 1018 1019 void ShenandoahGeneration::record_success_concurrent(bool abbreviated) { 1020 heuristics()->record_success_concurrent(); 1021 ShenandoahHeap::heap()->shenandoah_policy()->record_success_concurrent(is_young(), abbreviated); 1022 }