1 /*
   2  * Copyright Amazon.com Inc. or its affiliates. All Rights Reserved.
   3  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
   4  *
   5  * This code is free software; you can redistribute it and/or modify it
   6  * under the terms of the GNU General Public License version 2 only, as
   7  * published by the Free Software Foundation.
   8  *
   9  * This code is distributed in the hope that it will be useful, but WITHOUT
  10  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  11  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  12  * version 2 for more details (a copy is included in the LICENSE file that
  13  * accompanied this code).
  14  *
  15  * You should have received a copy of the GNU General Public License version
  16  * 2 along with this work; if not, write to the Free Software Foundation,
  17  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
  18  *
  19  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
  20  * or visit www.oracle.com if you need additional information or have any
  21  * questions.
  22  *
  23  */
  24 
  25 #include "precompiled.hpp"
  26 #include "gc/shenandoah/shenandoahCollectorPolicy.hpp"
  27 #include "gc/shenandoah/shenandoahCollectionSetPreselector.hpp"
  28 #include "gc/shenandoah/shenandoahFreeSet.hpp"
  29 #include "gc/shenandoah/shenandoahGeneration.hpp"
  30 #include "gc/shenandoah/shenandoahGenerationalHeap.hpp"
  31 #include "gc/shenandoah/shenandoahMarkClosures.hpp"
  32 #include "gc/shenandoah/shenandoahMonitoringSupport.hpp"
  33 #include "gc/shenandoah/shenandoahOldGeneration.hpp"
  34 #include "gc/shenandoah/shenandoahReferenceProcessor.hpp"
  35 #include "gc/shenandoah/shenandoahScanRemembered.inline.hpp"
  36 #include "gc/shenandoah/shenandoahTaskqueue.inline.hpp"
  37 #include "gc/shenandoah/shenandoahUtils.hpp"
  38 #include "gc/shenandoah/shenandoahVerifier.hpp"
  39 #include "gc/shenandoah/shenandoahYoungGeneration.hpp"
  40 #include "gc/shenandoah/heuristics/shenandoahHeuristics.hpp"
  41 
  42 #include "utilities/quickSort.hpp"
  43 
  44 
  45 class ShenandoahResetUpdateRegionStateClosure : public ShenandoahHeapRegionClosure {
  46  private:
  47   ShenandoahHeap* _heap;
  48   ShenandoahMarkingContext* const _ctx;
  49  public:
  50   ShenandoahResetUpdateRegionStateClosure() :
  51     _heap(ShenandoahHeap::heap()),
  52     _ctx(_heap->marking_context()) {}
  53 
  54   void heap_region_do(ShenandoahHeapRegion* r) override {
  55     if (_heap->is_bitmap_slice_committed(r)) {
  56       _ctx->clear_bitmap(r);
  57     }
  58 
  59     if (r->is_active()) {
  60       // Reset live data and set TAMS optimistically. We would recheck these under the pause
  61       // anyway to capture any updates that happened since now.
  62       _ctx->capture_top_at_mark_start(r);
  63       r->clear_live_data();
  64     }
  65   }
  66 
  67   bool is_thread_safe() override { return true; }
  68 };
  69 
  70 class ShenandoahResetBitmapTask : public ShenandoahHeapRegionClosure {
  71  private:
  72   ShenandoahHeap* _heap;
  73   ShenandoahMarkingContext* const _ctx;
  74  public:
  75   ShenandoahResetBitmapTask() :
  76     _heap(ShenandoahHeap::heap()),
  77     _ctx(_heap->marking_context()) {}
  78 
  79   void heap_region_do(ShenandoahHeapRegion* region) {
  80     if (_heap->is_bitmap_slice_committed(region)) {
  81       _ctx->clear_bitmap(region);
  82     }
  83   }
  84 
  85   bool is_thread_safe() { return true; }
  86 };
  87 
  88 // Copy the write-version of the card-table into the read-version, clearing the
  89 // write-copy.
  90 class ShenandoahMergeWriteTable: public ShenandoahHeapRegionClosure {
  91  private:
  92   ShenandoahHeap* _heap;
  93   RememberedScanner* _scanner;
  94  public:
  95   ShenandoahMergeWriteTable() : _heap(ShenandoahHeap::heap()), _scanner(_heap->card_scan()) {}
  96 
  97   virtual void heap_region_do(ShenandoahHeapRegion* r) override {
  98     assert(r->is_old(), "Don't waste time doing this for non-old regions");
  99     _scanner->merge_write_table(r->bottom(), ShenandoahHeapRegion::region_size_words());
 100   }
 101 
 102   virtual bool is_thread_safe() override {
 103     return true;
 104   }
 105 };
 106 
 107 class ShenandoahSquirrelAwayCardTable: public ShenandoahHeapRegionClosure {
 108  private:
 109   ShenandoahHeap* _heap;
 110   RememberedScanner* _scanner;
 111  public:
 112   ShenandoahSquirrelAwayCardTable() :
 113     _heap(ShenandoahHeap::heap()),
 114     _scanner(_heap->card_scan()) {}
 115 
 116   void heap_region_do(ShenandoahHeapRegion* region) {
 117     assert(region->is_old(), "Don't waste time doing this for non-old regions");
 118     _scanner->reset_remset(region->bottom(), ShenandoahHeapRegion::region_size_words());
 119   }
 120 
 121   bool is_thread_safe() { return true; }
 122 };
 123 
 124 void ShenandoahGeneration::confirm_heuristics_mode() {
 125   if (_heuristics->is_diagnostic() && !UnlockDiagnosticVMOptions) {
 126     vm_exit_during_initialization(
 127             err_msg("Heuristics \"%s\" is diagnostic, and must be enabled via -XX:+UnlockDiagnosticVMOptions.",
 128                     _heuristics->name()));
 129   }
 130   if (_heuristics->is_experimental() && !UnlockExperimentalVMOptions) {
 131     vm_exit_during_initialization(
 132             err_msg("Heuristics \"%s\" is experimental, and must be enabled via -XX:+UnlockExperimentalVMOptions.",
 133                     _heuristics->name()));
 134   }
 135 }
 136 
 137 ShenandoahHeuristics* ShenandoahGeneration::initialize_heuristics(ShenandoahMode* gc_mode) {
 138   _heuristics = gc_mode->initialize_heuristics(this);
 139   _heuristics->set_guaranteed_gc_interval(ShenandoahGuaranteedGCInterval);
 140   confirm_heuristics_mode();
 141   return _heuristics;
 142 }
 143 
 144 size_t ShenandoahGeneration::bytes_allocated_since_gc_start() const {
 145   return Atomic::load(&_bytes_allocated_since_gc_start);
 146 }
 147 
 148 void ShenandoahGeneration::reset_bytes_allocated_since_gc_start() {
 149   Atomic::store(&_bytes_allocated_since_gc_start, (size_t)0);
 150 }
 151 
 152 void ShenandoahGeneration::increase_allocated(size_t bytes) {
 153   Atomic::add(&_bytes_allocated_since_gc_start, bytes, memory_order_relaxed);
 154 }
 155 
 156 void ShenandoahGeneration::set_evacuation_reserve(size_t new_val) {
 157   _evacuation_reserve = new_val;
 158 }
 159 
 160 size_t ShenandoahGeneration::get_evacuation_reserve() const {
 161   return _evacuation_reserve;
 162 }
 163 
 164 void ShenandoahGeneration::augment_evacuation_reserve(size_t increment) {
 165   _evacuation_reserve += increment;
 166 }
 167 
 168 void ShenandoahGeneration::log_status(const char *msg) const {
 169   typedef LogTarget(Info, gc, ergo) LogGcInfo;
 170 
 171   if (!LogGcInfo::is_enabled()) {
 172     return;
 173   }
 174 
 175   // Not under a lock here, so read each of these once to make sure
 176   // byte size in proper unit and proper unit for byte size are consistent.
 177   size_t v_used = used();
 178   size_t v_used_regions = used_regions_size();
 179   size_t v_soft_max_capacity = soft_max_capacity();
 180   size_t v_max_capacity = max_capacity();
 181   size_t v_available = available();
 182   size_t v_humongous_waste = get_humongous_waste();
 183   LogGcInfo::print("%s: %s generation used: " SIZE_FORMAT "%s, used regions: " SIZE_FORMAT "%s, "
 184                    "humongous waste: " SIZE_FORMAT "%s, soft capacity: " SIZE_FORMAT "%s, max capacity: " SIZE_FORMAT "%s, "
 185                    "available: " SIZE_FORMAT "%s", msg, name(),
 186                    byte_size_in_proper_unit(v_used),              proper_unit_for_byte_size(v_used),
 187                    byte_size_in_proper_unit(v_used_regions),      proper_unit_for_byte_size(v_used_regions),
 188                    byte_size_in_proper_unit(v_humongous_waste),   proper_unit_for_byte_size(v_humongous_waste),
 189                    byte_size_in_proper_unit(v_soft_max_capacity), proper_unit_for_byte_size(v_soft_max_capacity),
 190                    byte_size_in_proper_unit(v_max_capacity),      proper_unit_for_byte_size(v_max_capacity),
 191                    byte_size_in_proper_unit(v_available),         proper_unit_for_byte_size(v_available));
 192 }
 193 
 194 void ShenandoahGeneration::reset_mark_bitmap() {
 195   ShenandoahHeap* heap = ShenandoahHeap::heap();
 196   heap->assert_gc_workers(heap->workers()->active_workers());
 197 
 198   set_mark_incomplete();
 199 
 200   ShenandoahResetBitmapTask task;
 201   parallel_heap_region_iterate(&task);
 202 }
 203 
 204 // The ideal is to swap the remembered set so the safepoint effort is no more than a few pointer manipulations.
 205 // However, limitations in the implementation of the mutator write-barrier make it difficult to simply change the
 206 // location of the card table.  So the interim implementation of swap_remembered_set will copy the write-table
 207 // onto the read-table and will then clear the write-table.
 208 void ShenandoahGeneration::swap_remembered_set() {
 209   // Must be sure that marking is complete before we swap remembered set.
 210   ShenandoahHeap* heap = ShenandoahHeap::heap();
 211   heap->assert_gc_workers(heap->workers()->active_workers());
 212   shenandoah_assert_safepoint();
 213 
 214   // TODO: Eventually, we want replace this with a constant-time exchange of pointers.
 215   ShenandoahSquirrelAwayCardTable task;
 216   heap->old_generation()->parallel_heap_region_iterate(&task);
 217 }
 218 
 219 // Copy the write-version of the card-table into the read-version, clearing the
 220 // write-version. The work is done at a safepoint and in parallel by the GC
 221 // worker threads.
 222 void ShenandoahGeneration::merge_write_table() {
 223   // This should only happen for degenerated cycles
 224   ShenandoahHeap* heap = ShenandoahHeap::heap();
 225   heap->assert_gc_workers(heap->workers()->active_workers());
 226   shenandoah_assert_safepoint();
 227 
 228   ShenandoahMergeWriteTable task;
 229   heap->old_generation()->parallel_heap_region_iterate(&task);
 230 }
 231 
 232 void ShenandoahGeneration::prepare_gc() {
 233   // Invalidate the marking context
 234   set_mark_incomplete();
 235 
 236   // Capture Top At Mark Start for this generation (typically young) and reset mark bitmap.
 237   ShenandoahResetUpdateRegionStateClosure cl;
 238   parallel_region_iterate_free(&cl);
 239 }
 240 
 241 void ShenandoahGeneration::parallel_region_iterate_free(ShenandoahHeapRegionClosure* cl) {
 242   ShenandoahHeap::heap()->parallel_heap_region_iterate(cl);
 243 }
 244 
 245 void ShenandoahGeneration::compute_evacuation_budgets(ShenandoahHeap* const heap) {
 246 
 247   ShenandoahOldGeneration* const old_generation = heap->old_generation();
 248   ShenandoahYoungGeneration* const young_generation = heap->young_generation();
 249 
 250   // During initialization and phase changes, it is more likely that fewer objects die young and old-gen
 251   // memory is not yet full (or is in the process of being replaced).  During these times especially, it
 252   // is beneficial to loan memory from old-gen to young-gen during the evacuation and update-refs phases
 253   // of execution.
 254 
 255   // Calculate EvacuationReserve before PromotionReserve.  Evacuation is more critical than promotion.
 256   // If we cannot evacuate old-gen, we will not be able to reclaim old-gen memory.  Promotions are less
 257   // critical.  If we cannot promote, there may be degradation of young-gen memory because old objects
 258   // accumulate there until they can be promoted.  This increases the young-gen marking and evacuation work.
 259 
 260   // First priority is to reclaim the easy garbage out of young-gen.
 261 
 262   // maximum_young_evacuation_reserve is upper bound on memory to be evacuated out of young
 263   const size_t maximum_young_evacuation_reserve = (young_generation->max_capacity() * ShenandoahEvacReserve) / 100;
 264   const size_t young_evacuation_reserve = MIN2(maximum_young_evacuation_reserve, young_generation->available_with_reserve());
 265 
 266   // maximum_old_evacuation_reserve is an upper bound on memory evacuated from old and evacuated to old (promoted),
 267   // clamped by the old generation space available.
 268   //
 269   // Here's the algebra.
 270   // Let SOEP = ShenandoahOldEvacRatioPercent,
 271   //     OE = old evac,
 272   //     YE = young evac, and
 273   //     TE = total evac = OE + YE
 274   // By definition:
 275   //            SOEP/100 = OE/TE
 276   //                     = OE/(OE+YE)
 277   //  => SOEP/(100-SOEP) = OE/((OE+YE)-OE)         // componendo-dividendo: If a/b = c/d, then a/(b-a) = c/(d-c)
 278   //                     = OE/YE
 279   //  =>              OE = YE*SOEP/(100-SOEP)
 280 
 281   // We have to be careful in the event that SOEP is set to 100 by the user.
 282   assert(ShenandoahOldEvacRatioPercent <= 100, "Error");
 283   const size_t old_available = old_generation->available();
 284   const size_t maximum_old_evacuation_reserve = (ShenandoahOldEvacRatioPercent == 100) ?
 285     old_available : MIN2((maximum_young_evacuation_reserve * ShenandoahOldEvacRatioPercent) / (100 - ShenandoahOldEvacRatioPercent),
 286                           old_available);
 287 
 288 
 289   // Second priority is to reclaim garbage out of old-gen if there are old-gen collection candidates.  Third priority
 290   // is to promote as much as we have room to promote.  However, if old-gen memory is in short supply, this means young
 291   // GC is operating under "duress" and was unable to transfer the memory that we would normally expect.  In this case,
 292   // old-gen will refrain from compacting itself in order to allow a quicker young-gen cycle (by avoiding the update-refs
 293   // through ALL of old-gen).  If there is some memory available in old-gen, we will use this for promotions as promotions
 294   // do not add to the update-refs burden of GC.
 295 
 296   size_t old_evacuation_reserve, old_promo_reserve;
 297   if (is_global()) {
 298     // Global GC is typically triggered by user invocation of System.gc(), and typically indicates that there is lots
 299     // of garbage to be reclaimed because we are starting a new phase of execution.  Marking for global GC may take
 300     // significantly longer than typical young marking because we must mark through all old objects.  To expedite
 301     // evacuation and update-refs, we give emphasis to reclaiming garbage first, wherever that garbage is found.
 302     // Global GC will adjust generation sizes to accommodate the collection set it chooses.
 303 
 304     // Set old_promo_reserve to enforce that no regions are preselected for promotion.  Such regions typically
 305     // have relatively high memory utilization.  We still call select_aged_regions() because this will prepare for
 306     // promotions in place, if relevant.
 307     old_promo_reserve = 0;
 308 
 309     // Dedicate all available old memory to old_evacuation reserve.  This may be small, because old-gen is only
 310     // expanded based on an existing mixed evacuation workload at the end of the previous GC cycle.  We'll expand
 311     // the budget for evacuation of old during GLOBAL cset selection.
 312     old_evacuation_reserve = maximum_old_evacuation_reserve;
 313   } else if (old_generation->has_unprocessed_collection_candidates()) {
 314     // We reserved all old-gen memory at end of previous GC to hold anticipated evacuations to old-gen.  If this is
 315     // mixed evacuation, reserve all of this memory for compaction of old-gen and do not promote.  Prioritize compaction
 316     // over promotion in order to defragment OLD so that it will be better prepared to efficiently receive promoted memory.
 317     old_evacuation_reserve = maximum_old_evacuation_reserve;
 318     old_promo_reserve = 0;
 319   } else {
 320     // Make all old-evacuation memory for promotion, but if we can't use it all for promotion, we'll allow some evacuation.
 321     old_evacuation_reserve = 0;
 322     old_promo_reserve = maximum_old_evacuation_reserve;
 323   }
 324   assert(old_evacuation_reserve <= old_available, "Error");
 325 
 326   // We see too many old-evacuation failures if we force ourselves to evacuate into regions that are not initially empty.
 327   // So we limit the old-evacuation reserve to unfragmented memory.  Even so, old-evacuation is free to fill in nooks and
 328   // crannies within existing partially used regions and it generally tries to do so.
 329   const size_t old_free_unfragmented = old_generation->free_unaffiliated_regions() * ShenandoahHeapRegion::region_size_bytes();
 330   if (old_evacuation_reserve > old_free_unfragmented) {
 331     const size_t delta = old_evacuation_reserve - old_free_unfragmented;
 332     old_evacuation_reserve -= delta;
 333     // Let promo consume fragments of old-gen memory if not global
 334     if (!is_global()) {
 335       old_promo_reserve += delta;
 336     }
 337   }
 338 
 339   // Preselect regions for promotion by evacuation (obtaining the live data to seed promoted_reserve),
 340   // and identify regions that will promote in place. These use the tenuring threshold.
 341   const size_t consumed_by_advance_promotion = select_aged_regions(old_promo_reserve);
 342   assert(consumed_by_advance_promotion <= maximum_old_evacuation_reserve, "Cannot promote more than available old-gen memory");
 343 
 344   // Note that unused old_promo_reserve might not be entirely consumed_by_advance_promotion.  Do not transfer this
 345   // to old_evacuation_reserve because this memory is likely very fragmented, and we do not want to increase the likelihood
 346   // of old evacuation failure.
 347   young_generation->set_evacuation_reserve(young_evacuation_reserve);
 348   old_generation->set_evacuation_reserve(old_evacuation_reserve);
 349   old_generation->set_promoted_reserve(consumed_by_advance_promotion);
 350 
 351   // There is no need to expand OLD because all memory used here was set aside at end of previous GC, except in the
 352   // case of a GLOBAL gc.  During choose_collection_set() of GLOBAL, old will be expanded on demand.
 353 }
 354 
 355 // Having chosen the collection set, adjust the budgets for generational mode based on its composition.  Note
 356 // that young_generation->available() now knows about recently discovered immediate garbage.
 357 //
 358 void ShenandoahGeneration::adjust_evacuation_budgets(ShenandoahHeap* const heap, ShenandoahCollectionSet* const collection_set) {
 359   // We may find that old_evacuation_reserve and/or loaned_for_young_evacuation are not fully consumed, in which case we may
 360   //  be able to increase regions_available_to_loan
 361 
 362   // The role of adjust_evacuation_budgets() is to compute the correct value of regions_available_to_loan and to make
 363   // effective use of this memory, including the remnant memory within these regions that may result from rounding loan to
 364   // integral number of regions.  Excess memory that is available to be loaned is applied to an allocation supplement,
 365   // which allows mutators to allocate memory beyond the current capacity of young-gen on the promise that the loan
 366   // will be repaid as soon as we finish updating references for the recently evacuated collection set.
 367 
 368   // We cannot recalculate regions_available_to_loan by simply dividing old_generation->available() by region_size_bytes
 369   // because the available memory may be distributed between many partially occupied regions that are already holding old-gen
 370   // objects.  Memory in partially occupied regions is not "available" to be loaned.  Note that an increase in old-gen
 371   // available that results from a decrease in memory consumed by old evacuation is not necessarily available to be loaned
 372   // to young-gen.
 373 
 374   size_t region_size_bytes = ShenandoahHeapRegion::region_size_bytes();
 375   ShenandoahOldGeneration* const old_generation = heap->old_generation();
 376   ShenandoahYoungGeneration* const young_generation = heap->young_generation();
 377 
 378   size_t old_evacuated = collection_set->get_old_bytes_reserved_for_evacuation();
 379   size_t old_evacuated_committed = (size_t) (ShenandoahOldEvacWaste * old_evacuated);
 380   size_t old_evacuation_reserve = old_generation->get_evacuation_reserve();
 381 
 382   if (old_evacuated_committed > old_evacuation_reserve) {
 383     // This should only happen due to round-off errors when enforcing ShenandoahOldEvacWaste
 384     assert(old_evacuated_committed <= (33 * old_evacuation_reserve) / 32,
 385            "Round-off errors should be less than 3.125%%, committed: " SIZE_FORMAT ", reserved: " SIZE_FORMAT,
 386            old_evacuated_committed, old_evacuation_reserve);
 387     old_evacuated_committed = old_evacuation_reserve;
 388     // Leave old_evac_reserve as previously configured
 389   } else if (old_evacuated_committed < old_evacuation_reserve) {
 390     // This happens if the old-gen collection consumes less than full budget.
 391     old_evacuation_reserve = old_evacuated_committed;
 392     old_generation->set_evacuation_reserve(old_evacuation_reserve);
 393   }
 394 
 395   size_t young_advance_promoted = collection_set->get_young_bytes_to_be_promoted();
 396   size_t young_advance_promoted_reserve_used = (size_t) (ShenandoahPromoEvacWaste * young_advance_promoted);
 397 
 398   size_t young_evacuated = collection_set->get_young_bytes_reserved_for_evacuation();
 399   size_t young_evacuated_reserve_used = (size_t) (ShenandoahEvacWaste * young_evacuated);
 400 
 401   size_t total_young_available = young_generation->available_with_reserve();
 402   assert(young_evacuated_reserve_used <= total_young_available, "Cannot evacuate more than is available in young");
 403   young_generation->set_evacuation_reserve(young_evacuated_reserve_used);
 404 
 405   size_t old_available = old_generation->available();
 406   // Now that we've established the collection set, we know how much memory is really required by old-gen for evacuation
 407   // and promotion reserves.  Try shrinking OLD now in case that gives us a bit more runway for mutator allocations during
 408   // evac and update phases.
 409   size_t old_consumed = old_evacuated_committed + young_advance_promoted_reserve_used;
 410 
 411   if (old_available < old_consumed) {
 412     // This can happen due to round-off errors when adding the results of truncated integer arithmetic.
 413     // We've already truncated old_evacuated_committed.  Truncate young_advance_promoted_reserve_used here.
 414     assert(young_advance_promoted_reserve_used <= (33 * (old_available - old_evacuated_committed)) / 32,
 415            "Round-off errors should be less than 3.125%%, committed: " SIZE_FORMAT ", reserved: " SIZE_FORMAT,
 416            young_advance_promoted_reserve_used, old_available - old_evacuated_committed);
 417     young_advance_promoted_reserve_used = old_available - old_evacuated_committed;
 418     old_consumed = old_evacuated_committed + young_advance_promoted_reserve_used;
 419   }
 420 
 421   assert(old_available >= old_consumed, "Cannot consume (" SIZE_FORMAT ") more than is available (" SIZE_FORMAT ")",
 422          old_consumed, old_available);
 423   size_t excess_old = old_available - old_consumed;
 424   size_t unaffiliated_old_regions = old_generation->free_unaffiliated_regions();
 425   size_t unaffiliated_old = unaffiliated_old_regions * region_size_bytes;
 426   assert(old_available >= unaffiliated_old, "Unaffiliated old is a subset of old available");
 427 
 428   // Make sure old_evac_committed is unaffiliated
 429   if (old_evacuated_committed > 0) {
 430     if (unaffiliated_old > old_evacuated_committed) {
 431       size_t giveaway = unaffiliated_old - old_evacuated_committed;
 432       size_t giveaway_regions = giveaway / region_size_bytes;  // round down
 433       if (giveaway_regions > 0) {
 434         excess_old = MIN2(excess_old, giveaway_regions * region_size_bytes);
 435       } else {
 436         excess_old = 0;
 437       }
 438     } else {
 439       excess_old = 0;
 440     }
 441   }
 442 
 443   // If we find that OLD has excess regions, give them back to YOUNG now to reduce likelihood we run out of allocation
 444   // runway during evacuation and update-refs.
 445   size_t regions_to_xfer = 0;
 446   if (excess_old > unaffiliated_old) {
 447     // we can give back unaffiliated_old (all of unaffiliated is excess)
 448     if (unaffiliated_old_regions > 0) {
 449       regions_to_xfer = unaffiliated_old_regions;
 450     }
 451   } else if (unaffiliated_old_regions > 0) {
 452     // excess_old < unaffiliated old: we can give back MIN(excess_old/region_size_bytes, unaffiliated_old_regions)
 453     size_t excess_regions = excess_old / region_size_bytes;
 454     size_t regions_to_xfer = MIN2(excess_regions, unaffiliated_old_regions);
 455   }
 456 
 457   if (regions_to_xfer > 0) {
 458     bool result = heap->generation_sizer()->transfer_to_young(regions_to_xfer);
 459     assert(excess_old > regions_to_xfer * region_size_bytes, "Cannot xfer more than excess old");
 460     excess_old -= regions_to_xfer * region_size_bytes;
 461     log_info(gc, ergo)("%s transferred " SIZE_FORMAT " excess regions to young before start of evacuation",
 462                        result? "Successfully": "Unsuccessfully", regions_to_xfer);
 463   }
 464 
 465   // Add in the excess_old memory to hold unanticipated promotions, if any.  If there are more unanticipated
 466   // promotions than fit in reserved memory, they will be deferred until a future GC pass.
 467   size_t total_promotion_reserve = young_advance_promoted_reserve_used + excess_old;
 468   old_generation->set_promoted_reserve(total_promotion_reserve);
 469   old_generation->reset_promoted_expended();
 470 }
 471 
 472 typedef struct {
 473   ShenandoahHeapRegion* _region;
 474   size_t _live_data;
 475 } AgedRegionData;
 476 
 477 static int compare_by_aged_live(AgedRegionData a, AgedRegionData b) {
 478   if (a._live_data < b._live_data)
 479     return -1;
 480   else if (a._live_data > b._live_data)
 481     return 1;
 482   else return 0;
 483 }
 484 
 485 inline void assert_no_in_place_promotions() {
 486 #ifdef ASSERT
 487   class ShenandoahNoInPlacePromotions : public ShenandoahHeapRegionClosure {
 488   public:
 489     void heap_region_do(ShenandoahHeapRegion *r) override {
 490       assert(r->get_top_before_promote() == nullptr,
 491              "Region " SIZE_FORMAT " should not be ready for in-place promotion", r->index());
 492     }
 493   } cl;
 494   ShenandoahHeap::heap()->heap_region_iterate(&cl);
 495 #endif
 496 }
 497 
 498 // Preselect for inclusion into the collection set regions whose age is at or above tenure age which contain more than
 499 // ShenandoahOldGarbageThreshold amounts of garbage.  We identify these regions by setting the appropriate entry of
 500 // the collection set's preselected regions array to true.  All entries are initialized to false before calling this
 501 // function.
 502 //
 503 // During the subsequent selection of the collection set, we give priority to these promotion set candidates.
 504 // Without this prioritization, we found that the aged regions tend to be ignored because they typically have
 505 // much less garbage and much more live data than the recently allocated "eden" regions.  When aged regions are
 506 // repeatedly excluded from the collection set, the amount of live memory within the young generation tends to
 507 // accumulate and this has the undesirable side effect of causing young-generation collections to require much more
 508 // CPU and wall-clock time.
 509 //
 510 // A second benefit of treating aged regions differently than other regions during collection set selection is
 511 // that this allows us to more accurately budget memory to hold the results of evacuation.  Memory for evacuation
 512 // of aged regions must be reserved in the old generation.  Memory for evacuation of all other regions must be
 513 // reserved in the young generation.
 514 size_t ShenandoahGeneration::select_aged_regions(size_t old_available) {
 515 
 516   // There should be no regions configured for subsequent in-place-promotions carried over from the previous cycle.
 517   assert_no_in_place_promotions();
 518 
 519   auto const heap = ShenandoahGenerationalHeap::heap();
 520   bool* const candidate_regions_for_promotion_by_copy = heap->collection_set()->preselected_regions();
 521   ShenandoahMarkingContext* const ctx = heap->marking_context();
 522 
 523   const uint tenuring_threshold = heap->age_census()->tenuring_threshold();
 524   const size_t old_garbage_threshold = (ShenandoahHeapRegion::region_size_bytes() * ShenandoahOldGarbageThreshold) / 100;
 525 
 526   size_t old_consumed = 0;
 527   size_t promo_potential = 0;
 528   size_t candidates = 0;
 529 
 530   // Tracks the padding of space above top in regions eligible for promotion in place
 531   size_t promote_in_place_pad = 0;
 532 
 533   // Sort the promotion-eligible regions in order of increasing live-data-bytes so that we can first reclaim regions that require
 534   // less evacuation effort.  This prioritizes garbage first, expanding the allocation pool early before we reclaim regions that
 535   // have more live data.
 536   const size_t num_regions = heap->num_regions();
 537 
 538   ResourceMark rm;
 539   AgedRegionData* sorted_regions = NEW_RESOURCE_ARRAY(AgedRegionData, num_regions);
 540 
 541   for (size_t i = 0; i < num_regions; i++) {
 542     ShenandoahHeapRegion* const r = heap->get_region(i);
 543     if (r->is_empty() || !r->has_live() || !r->is_young() || !r->is_regular()) {
 544       // skip over regions that aren't regular young with some live data
 545       continue;
 546     }
 547     if (r->age() >= tenuring_threshold) {
 548       if ((r->garbage() < old_garbage_threshold)) {
 549         // This tenure-worthy region has too little garbage, so we do not want to expend the copying effort to
 550         // reclaim the garbage; instead this region may be eligible for promotion-in-place to the
 551         // old generation.
 552         HeapWord* tams = ctx->top_at_mark_start(r);
 553         HeapWord* original_top = r->top();
 554         if (!heap->is_concurrent_old_mark_in_progress() && tams == original_top) {
 555           // No allocations from this region have been made during concurrent mark. It meets all the criteria
 556           // for in-place-promotion. Though we only need the value of top when we fill the end of the region,
 557           // we use this field to indicate that this region should be promoted in place during the evacuation
 558           // phase.
 559           r->save_top_before_promote();
 560 
 561           size_t remnant_size = r->free() / HeapWordSize;
 562           if (remnant_size > ShenandoahHeap::min_fill_size()) {
 563             ShenandoahHeap::fill_with_object(original_top, remnant_size);
 564             // Fill the remnant memory within this region to assure no allocations prior to promote in place.  Otherwise,
 565             // newly allocated objects will not be parsable when promote in place tries to register them.  Furthermore, any
 566             // new allocations would not necessarily be eligible for promotion.  This addresses both issues.
 567             r->set_top(r->end());
 568             promote_in_place_pad += remnant_size * HeapWordSize;
 569           } else {
 570             // Since the remnant is so small that it cannot be filled, we don't have to worry about any accidental
 571             // allocations occurring within this region before the region is promoted in place.
 572           }
 573         }
 574         // Else, we do not promote this region (either in place or by copy) because it has received new allocations.
 575 
 576         // During evacuation, we exclude from promotion regions for which age > tenure threshold, garbage < garbage-threshold,
 577         //  and get_top_before_promote() != tams
 578       } else {
 579         // Record this promotion-eligible candidate region. After sorting and selecting the best candidates below,
 580         // we may still decide to exclude this promotion-eligible region from the current collection set.  If this
 581         // happens, we will consider this region as part of the anticipated promotion potential for the next GC
 582         // pass; see further below.
 583         sorted_regions[candidates]._region = r;
 584         sorted_regions[candidates++]._live_data = r->get_live_data_bytes();
 585       }
 586     } else {
 587       // We only evacuate & promote objects from regular regions whose garbage() is above old-garbage-threshold.
 588       // Objects in tenure-worthy regions with less garbage are promoted in place. These take a different path to
 589       // old-gen.  Regions excluded from promotion because their garbage content is too low (causing us to anticipate that
 590       // the region would be promoted in place) may be eligible for evacuation promotion by the time promotion takes
 591       // place during a subsequent GC pass because more garbage is found within the region between now and then.  This
 592       // should not happen if we are properly adapting the tenure age.  The theory behind adaptive tenuring threshold
 593       // is to choose the youngest age that demonstrates no "significant" further loss of population since the previous
 594       // age.  If not this, we expect the tenure age to demonstrate linear population decay for at least two population
 595       // samples, whereas we expect to observe exponential population decay for ages younger than the tenure age.
 596       //
 597       // In the case that certain regions which were anticipated to be promoted in place need to be promoted by
 598       // evacuation, it may be the case that there is not sufficient reserve within old-gen to hold evacuation of
 599       // these regions.  The likely outcome is that these regions will not be selected for evacuation or promotion
 600       // in the current cycle and we will anticipate that they will be promoted in the next cycle.  This will cause
 601       // us to reserve more old-gen memory so that these objects can be promoted in the subsequent cycle.
 602       //
 603       // TODO:
 604       //   If we are auto-tuning the tenure age and regions that were anticipated to be promoted in place end up
 605       //   being promoted by evacuation, this event should feed into the tenure-age-selection heuristic so that
 606       //   the tenure age can be increased.
 607       if (heap->is_aging_cycle() && (r->age() + 1 == tenuring_threshold)) {
 608         if (r->garbage() >= old_garbage_threshold) {
 609           promo_potential += r->get_live_data_bytes();
 610         }
 611       }
 612     }
 613     // Note that we keep going even if one region is excluded from selection.
 614     // Subsequent regions may be selected if they have smaller live data.
 615   }
 616   // Sort in increasing order according to live data bytes.  Note that candidates represents the number of regions
 617   // that qualify to be promoted by evacuation.
 618   if (candidates > 0) {
 619     size_t selected_regions = 0;
 620     size_t selected_live = 0;
 621     QuickSort::sort<AgedRegionData>(sorted_regions, candidates, compare_by_aged_live, false);
 622     for (size_t i = 0; i < candidates; i++) {
 623       ShenandoahHeapRegion* const region = sorted_regions[i]._region;
 624       size_t region_live_data = sorted_regions[i]._live_data;
 625       size_t promotion_need = (size_t) (region_live_data * ShenandoahPromoEvacWaste);
 626       if (old_consumed + promotion_need <= old_available) {
 627         old_consumed += promotion_need;
 628         candidate_regions_for_promotion_by_copy[region->index()] = true;
 629         selected_regions++;
 630         selected_live += region_live_data;
 631       } else {
 632         // We rejected this promotable region from the collection set because we had no room to hold its copy.
 633         // Add this region to promo potential for next GC.
 634         promo_potential += region_live_data;
 635         assert(!candidate_regions_for_promotion_by_copy[region->index()], "Shouldn't be selected");
 636       }
 637       // We keep going even if one region is excluded from selection because we need to accumulate all eligible
 638       // regions that are not preselected into promo_potential
 639     }
 640     log_info(gc)("Preselected " SIZE_FORMAT " regions containing " SIZE_FORMAT " live bytes,"
 641                  " consuming: " SIZE_FORMAT " of budgeted: " SIZE_FORMAT,
 642                  selected_regions, selected_live, old_consumed, old_available);
 643   }
 644 
 645   heap->old_generation()->set_pad_for_promote_in_place(promote_in_place_pad);
 646   heap->old_generation()->set_promotion_potential(promo_potential);
 647   return old_consumed;
 648 }
 649 
 650 void ShenandoahGeneration::prepare_regions_and_collection_set(bool concurrent) {
 651   ShenandoahHeap* heap = ShenandoahHeap::heap();
 652   ShenandoahCollectionSet* collection_set = heap->collection_set();
 653   bool is_generational = heap->mode()->is_generational();
 654 
 655   assert(!heap->is_full_gc_in_progress(), "Only for concurrent and degenerated GC");
 656   assert(!is_old(), "Only YOUNG and GLOBAL GC perform evacuations");
 657   {
 658     ShenandoahGCPhase phase(concurrent ? ShenandoahPhaseTimings::final_update_region_states :
 659                             ShenandoahPhaseTimings::degen_gc_final_update_region_states);
 660     ShenandoahFinalMarkUpdateRegionStateClosure cl(complete_marking_context());
 661     parallel_heap_region_iterate(&cl);
 662 
 663     if (is_young()) {
 664       // We always need to update the watermark for old regions. If there
 665       // are mixed collections pending, we also need to synchronize the
 666       // pinned status for old regions. Since we are already visiting every
 667       // old region here, go ahead and sync the pin status too.
 668       ShenandoahFinalMarkUpdateRegionStateClosure old_cl(nullptr);
 669       heap->old_generation()->parallel_heap_region_iterate(&old_cl);
 670     }
 671   }
 672 
 673   // Tally the census counts and compute the adaptive tenuring threshold
 674   if (is_generational && ShenandoahGenerationalAdaptiveTenuring && !ShenandoahGenerationalCensusAtEvac) {
 675     // Objects above TAMS weren't included in the age census. Since they were all
 676     // allocated in this cycle they belong in the age 0 cohort. We walk over all
 677     // young regions and sum the volume of objects between TAMS and top.
 678     ShenandoahUpdateCensusZeroCohortClosure age0_cl(complete_marking_context());
 679     heap->young_generation()->heap_region_iterate(&age0_cl);
 680     size_t age0_pop = age0_cl.get_age0_population();
 681 
 682     // Update the global census, including the missed age 0 cohort above,
 683     // along with the census done during marking, and compute the tenuring threshold.
 684     heap->age_census()->update_census(age0_pop);
 685 #ifndef PRODUCT
 686     size_t total_pop = age0_cl.get_total_population();
 687     size_t total_census = heap->age_census()->get_total();
 688     // Usually total_pop > total_census, but not by too much.
 689     // We use integer division so anything up to just less than 2 is considered
 690     // reasonable, and the "+1" is to avoid divide-by-zero.
 691     assert((total_pop+1)/(total_census+1) ==  1, "Extreme divergence: "
 692            SIZE_FORMAT "/" SIZE_FORMAT, total_pop, total_census);
 693 #endif
 694   }
 695 
 696   {
 697     ShenandoahGCPhase phase(concurrent ? ShenandoahPhaseTimings::choose_cset :
 698                             ShenandoahPhaseTimings::degen_gc_choose_cset);
 699 
 700     collection_set->clear();
 701     ShenandoahHeapLocker locker(heap->lock());
 702     if (is_generational) {
 703       // Seed the collection set with resource area-allocated
 704       // preselected regions, which are removed when we exit this scope.
 705       ResourceMark rm;
 706       ShenandoahCollectionSetPreselector preselector(collection_set, heap->num_regions());
 707 
 708       // TODO: young_available can include available (between top() and end()) within each young region that is not
 709       // part of the collection set.  Making this memory available to the young_evacuation_reserve allows a larger
 710       // young collection set to be chosen when available memory is under extreme pressure.  Implementing this "improvement"
 711       // is tricky, because the incremental construction of the collection set actually changes the amount of memory
 712       // available to hold evacuated young-gen objects.  As currently implemented, the memory that is available within
 713       // non-empty regions that are not selected as part of the collection set can be allocated by the mutator while
 714       // GC is evacuating and updating references.
 715 
 716       // Find the amount that will be promoted, regions that will be promoted in
 717       // place, and preselect older regions that will be promoted by evacuation.
 718       compute_evacuation_budgets(heap);
 719 
 720       // Choose the collection set, including the regions preselected above for
 721       // promotion into the old generation.
 722       _heuristics->choose_collection_set(collection_set);
 723       if (!collection_set->is_empty()) {
 724         // only make use of evacuation budgets when we are evacuating
 725         adjust_evacuation_budgets(heap, collection_set);
 726       }
 727 
 728       if (is_global()) {
 729         // We have just chosen a collection set for a global cycle. The mark bitmap covering old regions is complete, so
 730         // the remembered set scan can use that to avoid walking into garbage. When the next old mark begins, we will
 731         // use the mark bitmap to make the old regions parsable by coalescing and filling any unmarked objects. Thus,
 732         // we prepare for old collections by remembering which regions are old at this time. Note that any objects
 733         // promoted into old regions will be above TAMS, and so will be considered marked. However, free regions that
 734         // become old after this point will not be covered correctly by the mark bitmap, so we must be careful not to
 735         // coalesce those regions. Only the old regions which are not part of the collection set at this point are
 736         // eligible for coalescing. As implemented now, this has the side effect of possibly initiating mixed-evacuations
 737         // after a global cycle for old regions that were not included in this collection set.
 738         heap->old_generation()->prepare_for_mixed_collections_after_global_gc();
 739       }
 740     } else {
 741       _heuristics->choose_collection_set(collection_set);
 742     }
 743   }
 744 
 745 
 746   {
 747     ShenandoahGCPhase phase(concurrent ? ShenandoahPhaseTimings::final_rebuild_freeset :
 748                             ShenandoahPhaseTimings::degen_gc_final_rebuild_freeset);
 749     ShenandoahHeapLocker locker(heap->lock());
 750     size_t young_cset_regions, old_cset_regions;
 751 
 752     // We are preparing for evacuation.  At this time, we ignore cset region tallies.
 753     size_t first_old, last_old, num_old;
 754     heap->free_set()->prepare_to_rebuild(young_cset_regions, old_cset_regions, first_old, last_old, num_old);
 755     // Free set construction uses reserve quantities, because they are known to be valid here
 756     heap->free_set()->rebuild(young_cset_regions, old_cset_regions, true);
 757   }
 758 }
 759 
 760 bool ShenandoahGeneration::is_bitmap_clear() {
 761   ShenandoahHeap* heap = ShenandoahHeap::heap();
 762   ShenandoahMarkingContext* context = heap->marking_context();
 763   const size_t num_regions = heap->num_regions();
 764   for (size_t idx = 0; idx < num_regions; idx++) {
 765     ShenandoahHeapRegion* r = heap->get_region(idx);
 766     if (contains(r) && r->is_affiliated()) {
 767       if (heap->is_bitmap_slice_committed(r) && (context->top_at_mark_start(r) > r->bottom()) &&
 768           !context->is_bitmap_clear_range(r->bottom(), r->end())) {
 769         return false;
 770       }
 771     }
 772   }
 773   return true;
 774 }
 775 
 776 bool ShenandoahGeneration::is_mark_complete() {
 777   return _is_marking_complete.is_set();
 778 }
 779 
 780 void ShenandoahGeneration::set_mark_complete() {
 781   _is_marking_complete.set();
 782 }
 783 
 784 void ShenandoahGeneration::set_mark_incomplete() {
 785   _is_marking_complete.unset();
 786 }
 787 
 788 ShenandoahMarkingContext* ShenandoahGeneration::complete_marking_context() {
 789   assert(is_mark_complete(), "Marking must be completed.");
 790   return ShenandoahHeap::heap()->marking_context();
 791 }
 792 
 793 void ShenandoahGeneration::cancel_marking() {
 794   log_info(gc)("Cancel marking: %s", name());
 795   if (is_concurrent_mark_in_progress()) {
 796     set_mark_incomplete();
 797   }
 798   _task_queues->clear();
 799   ref_processor()->abandon_partial_discovery();
 800   set_concurrent_mark_in_progress(false);
 801 }
 802 
 803 ShenandoahGeneration::ShenandoahGeneration(ShenandoahGenerationType type,
 804                                            uint max_workers,
 805                                            size_t max_capacity,
 806                                            size_t soft_max_capacity) :
 807   _type(type),
 808   _task_queues(new ShenandoahObjToScanQueueSet(max_workers)),
 809   _ref_processor(new ShenandoahReferenceProcessor(MAX2(max_workers, 1U))),
 810   _affiliated_region_count(0), _humongous_waste(0), _evacuation_reserve(0),
 811   _used(0), _bytes_allocated_since_gc_start(0),
 812   _max_capacity(max_capacity), _soft_max_capacity(soft_max_capacity),
 813   _heuristics(nullptr)
 814 {
 815   _is_marking_complete.set();
 816   assert(max_workers > 0, "At least one queue");
 817   for (uint i = 0; i < max_workers; ++i) {
 818     ShenandoahObjToScanQueue* task_queue = new ShenandoahObjToScanQueue();
 819     _task_queues->register_queue(i, task_queue);
 820   }
 821 }
 822 
 823 ShenandoahGeneration::~ShenandoahGeneration() {
 824   for (uint i = 0; i < _task_queues->size(); ++i) {
 825     ShenandoahObjToScanQueue* q = _task_queues->queue(i);
 826     delete q;
 827   }
 828   delete _task_queues;
 829 }
 830 
 831 void ShenandoahGeneration::reserve_task_queues(uint workers) {
 832   _task_queues->reserve(workers);
 833 }
 834 
 835 ShenandoahObjToScanQueueSet* ShenandoahGeneration::old_gen_task_queues() const {
 836   return nullptr;
 837 }
 838 
 839 void ShenandoahGeneration::scan_remembered_set(bool is_concurrent) {
 840   assert(is_young(), "Should only scan remembered set for young generation.");
 841 
 842   ShenandoahHeap* const heap = ShenandoahHeap::heap();
 843   uint nworkers = heap->workers()->active_workers();
 844   reserve_task_queues(nworkers);
 845 
 846   ShenandoahReferenceProcessor* rp = ref_processor();
 847   ShenandoahRegionChunkIterator work_list(nworkers);
 848   ShenandoahScanRememberedTask task(task_queues(), old_gen_task_queues(), rp, &work_list, is_concurrent);
 849   heap->assert_gc_workers(nworkers);
 850   heap->workers()->run_task(&task);
 851   if (ShenandoahEnableCardStats) {
 852     assert(heap->card_scan() != nullptr, "Not generational");
 853     heap->card_scan()->log_card_stats(nworkers, CARD_STAT_SCAN_RS);
 854   }
 855 }
 856 
 857 size_t ShenandoahGeneration::increment_affiliated_region_count() {
 858   shenandoah_assert_heaplocked_or_fullgc_safepoint();
 859   // During full gc, multiple GC worker threads may change region affiliations without a lock.  No lock is enforced
 860   // on read and write of _affiliated_region_count.  At the end of full gc, a single thread overwrites the count with
 861   // a coherent value.
 862   _affiliated_region_count++;
 863   return _affiliated_region_count;
 864 }
 865 
 866 size_t ShenandoahGeneration::decrement_affiliated_region_count() {
 867   shenandoah_assert_heaplocked_or_fullgc_safepoint();
 868   // During full gc, multiple GC worker threads may change region affiliations without a lock.  No lock is enforced
 869   // on read and write of _affiliated_region_count.  At the end of full gc, a single thread overwrites the count with
 870   // a coherent value.
 871   _affiliated_region_count--;
 872   // TODO: REMOVE IS_GLOBAL() QUALIFIER AFTER WE FIX GLOBAL AFFILIATED REGION ACCOUNTING
 873   assert(is_global() || ShenandoahHeap::heap()->is_full_gc_in_progress() ||
 874          (_used + _humongous_waste <= _affiliated_region_count * ShenandoahHeapRegion::region_size_bytes()),
 875          "used + humongous cannot exceed regions");
 876   return _affiliated_region_count;
 877 }
 878 
 879 size_t ShenandoahGeneration::increase_affiliated_region_count(size_t delta) {
 880   shenandoah_assert_heaplocked_or_fullgc_safepoint();
 881   _affiliated_region_count += delta;
 882   return _affiliated_region_count;
 883 }
 884 
 885 size_t ShenandoahGeneration::decrease_affiliated_region_count(size_t delta) {
 886   shenandoah_assert_heaplocked_or_fullgc_safepoint();
 887   assert(_affiliated_region_count >= delta, "Affiliated region count cannot be negative");
 888 
 889   _affiliated_region_count -= delta;
 890   // TODO: REMOVE IS_GLOBAL() QUALIFIER AFTER WE FIX GLOBAL AFFILIATED REGION ACCOUNTING
 891   assert(is_global() || ShenandoahHeap::heap()->is_full_gc_in_progress() ||
 892          (_used + _humongous_waste <= _affiliated_region_count * ShenandoahHeapRegion::region_size_bytes()),
 893          "used + humongous cannot exceed regions");
 894   return _affiliated_region_count;
 895 }
 896 
 897 void ShenandoahGeneration::establish_usage(size_t num_regions, size_t num_bytes, size_t humongous_waste) {
 898   assert(ShenandoahSafepoint::is_at_shenandoah_safepoint(), "must be at a safepoint");
 899   _affiliated_region_count = num_regions;
 900   _used = num_bytes;
 901   _humongous_waste = humongous_waste;
 902 }
 903 
 904 void ShenandoahGeneration::increase_used(size_t bytes) {
 905   Atomic::add(&_used, bytes);
 906 }
 907 
 908 void ShenandoahGeneration::increase_humongous_waste(size_t bytes) {
 909   if (bytes > 0) {
 910     Atomic::add(&_humongous_waste, bytes);
 911   }
 912 }
 913 
 914 void ShenandoahGeneration::decrease_humongous_waste(size_t bytes) {
 915   if (bytes > 0) {
 916     assert(ShenandoahHeap::heap()->is_full_gc_in_progress() || (_humongous_waste >= bytes),
 917            "Waste (" SIZE_FORMAT ") cannot be negative (after subtracting " SIZE_FORMAT ")", _humongous_waste, bytes);
 918     Atomic::sub(&_humongous_waste, bytes);
 919   }
 920 }
 921 
 922 void ShenandoahGeneration::decrease_used(size_t bytes) {
 923   // TODO: REMOVE IS_GLOBAL() QUALIFIER AFTER WE FIX GLOBAL AFFILIATED REGION ACCOUNTING
 924   assert(is_global() || ShenandoahHeap::heap()->is_full_gc_in_progress() ||
 925          (_used >= bytes), "cannot reduce bytes used by generation below zero");
 926   Atomic::sub(&_used, bytes);
 927 }
 928 
 929 size_t ShenandoahGeneration::used_regions() const {
 930   return _affiliated_region_count;
 931 }
 932 
 933 size_t ShenandoahGeneration::free_unaffiliated_regions() const {
 934   size_t result = max_capacity() / ShenandoahHeapRegion::region_size_bytes();
 935   if (_affiliated_region_count > result) {
 936     result = 0;
 937   } else {
 938     result -= _affiliated_region_count;
 939   }
 940   return result;
 941 }
 942 
 943 size_t ShenandoahGeneration::used_regions_size() const {
 944   return _affiliated_region_count * ShenandoahHeapRegion::region_size_bytes();
 945 }
 946 
 947 size_t ShenandoahGeneration::available() const {
 948   return available(max_capacity());
 949 }
 950 
 951 // For ShenandoahYoungGeneration, Include the young available that may have been reserved for the Collector.
 952 size_t ShenandoahGeneration::available_with_reserve() const {
 953   return available(max_capacity());
 954 }
 955 
 956 size_t ShenandoahGeneration::soft_available() const {
 957   return available(soft_max_capacity());
 958 }
 959 
 960 size_t ShenandoahGeneration::available(size_t capacity) const {
 961   size_t in_use = used() + get_humongous_waste();
 962   return in_use > capacity ? 0 : capacity - in_use;
 963 }
 964 
 965 void ShenandoahGeneration::increase_capacity(size_t increment) {
 966   shenandoah_assert_heaplocked_or_safepoint();
 967 
 968   // We do not enforce that new capacity >= heap->max_size_for(this).  The maximum generation size is treated as a rule of thumb
 969   // which may be violated during certain transitions, such as when we are forcing transfers for the purpose of promoting regions
 970   // in place.
 971   // TODO: REMOVE IS_GLOBAL() QUALIFIER AFTER WE FIX GLOBAL AFFILIATED REGION ACCOUNTING
 972   assert(is_global() || ShenandoahHeap::heap()->is_full_gc_in_progress() ||
 973          (_max_capacity + increment <= ShenandoahHeap::heap()->max_capacity()), "Generation cannot be larger than heap size");
 974   assert(increment % ShenandoahHeapRegion::region_size_bytes() == 0, "Generation capacity must be multiple of region size");
 975   _max_capacity += increment;
 976 
 977   // This detects arithmetic wraparound on _used
 978   // TODO: REMOVE IS_GLOBAL() QUALIFIER AFTER WE FIX GLOBAL AFFILIATED REGION ACCOUNTING
 979   assert(is_global() || ShenandoahHeap::heap()->is_full_gc_in_progress() ||
 980          (_affiliated_region_count * ShenandoahHeapRegion::region_size_bytes() >= _used),
 981          "Affiliated regions must hold more than what is currently used");
 982 }
 983 
 984 void ShenandoahGeneration::decrease_capacity(size_t decrement) {
 985   shenandoah_assert_heaplocked_or_safepoint();
 986 
 987   // We do not enforce that new capacity >= heap->min_size_for(this).  The minimum generation size is treated as a rule of thumb
 988   // which may be violated during certain transitions, such as when we are forcing transfers for the purpose of promoting regions
 989   // in place.
 990   assert(decrement % ShenandoahHeapRegion::region_size_bytes() == 0, "Generation capacity must be multiple of region size");
 991   assert(_max_capacity >= decrement, "Generation capacity cannot be negative");
 992 
 993   _max_capacity -= decrement;
 994 
 995   // This detects arithmetic wraparound on _used
 996   // TODO: REMOVE IS_GLOBAL() QUALIFIER AFTER WE FIX GLOBAL AFFILIATED REGION ACCOUNTING
 997   assert(is_global() || ShenandoahHeap::heap()->is_full_gc_in_progress() ||
 998          (_affiliated_region_count * ShenandoahHeapRegion::region_size_bytes() >= _used),
 999          "Affiliated regions must hold more than what is currently used");
1000   // TODO: REMOVE IS_GLOBAL() QUALIFIER AFTER WE FIX GLOBAL AFFILIATED REGION ACCOUNTING
1001   assert(is_global() || ShenandoahHeap::heap()->is_full_gc_in_progress() ||
1002          (_used <= _max_capacity), "Cannot use more than capacity");
1003   // TODO: REMOVE IS_GLOBAL() QUALIFIER AFTER WE FIX GLOBAL AFFILIATED REGION ACCOUNTING
1004   assert(is_global() || ShenandoahHeap::heap()->is_full_gc_in_progress() ||
1005          (_affiliated_region_count * ShenandoahHeapRegion::region_size_bytes() <= _max_capacity),
1006          "Cannot use more than capacity");
1007 }
1008 
1009 void ShenandoahGeneration::record_success_concurrent(bool abbreviated) {
1010   heuristics()->record_success_concurrent(abbreviated);
1011   ShenandoahHeap::heap()->shenandoah_policy()->record_success_concurrent(is_young(), abbreviated);
1012 }