1 /*
   2  * Copyright (c) 2023, 2025, Oracle and/or its affiliates. All rights reserved.
   3  * Copyright (c) 2013, 2022, Red Hat, Inc. All rights reserved.
   4  * Copyright Amazon.com Inc. or its affiliates. All Rights Reserved.
   5  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
   6  *
   7  * This code is free software; you can redistribute it and/or modify it
   8  * under the terms of the GNU General Public License version 2 only, as
   9  * published by the Free Software Foundation.
  10  *
  11  * This code is distributed in the hope that it will be useful, but WITHOUT
  12  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  13  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  14  * version 2 for more details (a copy is included in the LICENSE file that
  15  * accompanied this code).
  16  *
  17  * You should have received a copy of the GNU General Public License version
  18  * 2 along with this work; if not, write to the Free Software Foundation,
  19  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
  20  *
  21  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
  22  * or visit www.oracle.com if you need additional information or have any
  23  * questions.
  24  *
  25  */
  26 
  27 
  28 #include "cds/archiveHeapWriter.hpp"
  29 #include "classfile/systemDictionary.hpp"
  30 #include "gc/shared/classUnloadingContext.hpp"
  31 #include "gc/shared/fullGCForwarding.hpp"
  32 #include "gc/shared/gcArguments.hpp"
  33 #include "gc/shared/gcTimer.hpp"
  34 #include "gc/shared/gcTraceTime.inline.hpp"
  35 #include "gc/shared/locationPrinter.inline.hpp"
  36 #include "gc/shared/memAllocator.hpp"
  37 #include "gc/shared/plab.hpp"
  38 #include "gc/shared/tlab_globals.hpp"
  39 #include "gc/shenandoah/heuristics/shenandoahOldHeuristics.hpp"
  40 #include "gc/shenandoah/heuristics/shenandoahYoungHeuristics.hpp"
  41 #include "gc/shenandoah/mode/shenandoahGenerationalMode.hpp"
  42 #include "gc/shenandoah/mode/shenandoahPassiveMode.hpp"
  43 #include "gc/shenandoah/mode/shenandoahSATBMode.hpp"
  44 #include "gc/shenandoah/shenandoahAllocRequest.hpp"
  45 #include "gc/shenandoah/shenandoahBarrierSet.hpp"
  46 #include "gc/shenandoah/shenandoahClosures.inline.hpp"
  47 #include "gc/shenandoah/shenandoahCodeRoots.hpp"
  48 #include "gc/shenandoah/shenandoahCollectionSet.hpp"
  49 #include "gc/shenandoah/shenandoahCollectorPolicy.hpp"
  50 #include "gc/shenandoah/shenandoahConcurrentMark.hpp"
  51 #include "gc/shenandoah/shenandoahControlThread.hpp"
  52 #include "gc/shenandoah/shenandoahFreeSet.hpp"
  53 #include "gc/shenandoah/shenandoahGenerationalEvacuationTask.hpp"
  54 #include "gc/shenandoah/shenandoahGenerationalHeap.hpp"
  55 #include "gc/shenandoah/shenandoahGlobalGeneration.hpp"
  56 #include "gc/shenandoah/shenandoahHeap.inline.hpp"
  57 #include "gc/shenandoah/shenandoahHeapRegion.inline.hpp"
  58 #include "gc/shenandoah/shenandoahHeapRegionClosures.hpp"
  59 #include "gc/shenandoah/shenandoahHeapRegionSet.hpp"
  60 #include "gc/shenandoah/shenandoahInitLogger.hpp"
  61 #include "gc/shenandoah/shenandoahMarkingContext.inline.hpp"
  62 #include "gc/shenandoah/shenandoahMemoryPool.hpp"
  63 #include "gc/shenandoah/shenandoahMonitoringSupport.hpp"
  64 #include "gc/shenandoah/shenandoahOldGeneration.hpp"
  65 #include "gc/shenandoah/shenandoahPacer.inline.hpp"
  66 #include "gc/shenandoah/shenandoahPadding.hpp"
  67 #include "gc/shenandoah/shenandoahParallelCleaning.inline.hpp"
  68 #include "gc/shenandoah/shenandoahPhaseTimings.hpp"
  69 #include "gc/shenandoah/shenandoahReferenceProcessor.hpp"
  70 #include "gc/shenandoah/shenandoahRootProcessor.inline.hpp"
  71 #include "gc/shenandoah/shenandoahScanRemembered.inline.hpp"
  72 #include "gc/shenandoah/shenandoahSTWMark.hpp"
  73 #include "gc/shenandoah/shenandoahUncommitThread.hpp"
  74 #include "gc/shenandoah/shenandoahUtils.hpp"
  75 #include "gc/shenandoah/shenandoahVerifier.hpp"
  76 #include "gc/shenandoah/shenandoahVMOperations.hpp"
  77 #include "gc/shenandoah/shenandoahWorkerPolicy.hpp"
  78 #include "gc/shenandoah/shenandoahWorkGroup.hpp"
  79 #include "gc/shenandoah/shenandoahYoungGeneration.hpp"
  80 #include "memory/allocation.hpp"
  81 #include "memory/classLoaderMetaspace.hpp"
  82 #include "memory/memoryReserver.hpp"
  83 #include "memory/metaspaceUtils.hpp"
  84 #include "memory/universe.hpp"
  85 #include "nmt/mallocTracker.hpp"
  86 #include "nmt/memTracker.hpp"
  87 #include "oops/compressedOops.inline.hpp"
  88 #include "prims/jvmtiTagMap.hpp"
  89 #include "runtime/atomic.hpp"
  90 #include "runtime/globals.hpp"
  91 #include "runtime/interfaceSupport.inline.hpp"
  92 #include "runtime/java.hpp"
  93 #include "runtime/orderAccess.hpp"
  94 #include "runtime/safepointMechanism.hpp"
  95 #include "runtime/stackWatermarkSet.hpp"
  96 #include "runtime/threads.hpp"
  97 #include "runtime/vmThread.hpp"
  98 #include "utilities/events.hpp"
  99 #include "utilities/globalDefinitions.hpp"
 100 #include "utilities/powerOfTwo.hpp"
 101 #if INCLUDE_JVMCI
 102 #include "jvmci/jvmci.hpp"
 103 #endif
 104 #if INCLUDE_JFR
 105 #include "gc/shenandoah/shenandoahJfrSupport.hpp"
 106 #endif
 107 
 108 class ShenandoahPretouchHeapTask : public WorkerTask {
 109 private:
 110   ShenandoahRegionIterator _regions;
 111   const size_t _page_size;
 112 public:
 113   ShenandoahPretouchHeapTask(size_t page_size) :
 114     WorkerTask("Shenandoah Pretouch Heap"),
 115     _page_size(page_size) {}
 116 
 117   virtual void work(uint worker_id) {
 118     ShenandoahHeapRegion* r = _regions.next();
 119     while (r != nullptr) {
 120       if (r->is_committed()) {
 121         os::pretouch_memory(r->bottom(), r->end(), _page_size);
 122       }
 123       r = _regions.next();
 124     }
 125   }
 126 };
 127 
 128 class ShenandoahPretouchBitmapTask : public WorkerTask {
 129 private:
 130   ShenandoahRegionIterator _regions;
 131   char* _bitmap_base;
 132   const size_t _bitmap_size;
 133   const size_t _page_size;
 134 public:
 135   ShenandoahPretouchBitmapTask(char* bitmap_base, size_t bitmap_size, size_t page_size) :
 136     WorkerTask("Shenandoah Pretouch Bitmap"),
 137     _bitmap_base(bitmap_base),
 138     _bitmap_size(bitmap_size),
 139     _page_size(page_size) {}
 140 
 141   virtual void work(uint worker_id) {
 142     ShenandoahHeapRegion* r = _regions.next();
 143     while (r != nullptr) {
 144       size_t start = r->index()       * ShenandoahHeapRegion::region_size_bytes() / MarkBitMap::heap_map_factor();
 145       size_t end   = (r->index() + 1) * ShenandoahHeapRegion::region_size_bytes() / MarkBitMap::heap_map_factor();
 146       assert (end <= _bitmap_size, "end is sane: %zu < %zu", end, _bitmap_size);
 147 
 148       if (r->is_committed()) {
 149         os::pretouch_memory(_bitmap_base + start, _bitmap_base + end, _page_size);
 150       }
 151 
 152       r = _regions.next();
 153     }
 154   }
 155 };
 156 
 157 static ReservedSpace reserve(size_t size, size_t preferred_page_size) {
 158   // When a page size is given we don't want to mix large
 159   // and normal pages. If the size is not a multiple of the
 160   // page size it will be aligned up to achieve this.
 161   size_t alignment = os::vm_allocation_granularity();
 162   if (preferred_page_size != os::vm_page_size()) {
 163     alignment = MAX2(preferred_page_size, alignment);
 164     size = align_up(size, alignment);
 165   }
 166 
 167   const ReservedSpace reserved = MemoryReserver::reserve(size, alignment, preferred_page_size, mtGC);
 168   if (!reserved.is_reserved()) {
 169     vm_exit_during_initialization("Could not reserve space");
 170   }
 171   return reserved;
 172 }
 173 
 174 jint ShenandoahHeap::initialize() {
 175   //
 176   // Figure out heap sizing
 177   //
 178 
 179   size_t init_byte_size = InitialHeapSize;
 180   size_t min_byte_size  = MinHeapSize;
 181   size_t max_byte_size  = MaxHeapSize;
 182   size_t heap_alignment = HeapAlignment;
 183 
 184   size_t reg_size_bytes = ShenandoahHeapRegion::region_size_bytes();
 185 
 186   Universe::check_alignment(max_byte_size,  reg_size_bytes, "Shenandoah heap");
 187   Universe::check_alignment(init_byte_size, reg_size_bytes, "Shenandoah heap");
 188 
 189   _num_regions = ShenandoahHeapRegion::region_count();
 190   assert(_num_regions == (max_byte_size / reg_size_bytes),
 191          "Regions should cover entire heap exactly: %zu != %zu/%zu",
 192          _num_regions, max_byte_size, reg_size_bytes);
 193 
 194   size_t num_committed_regions = init_byte_size / reg_size_bytes;
 195   num_committed_regions = MIN2(num_committed_regions, _num_regions);
 196   assert(num_committed_regions <= _num_regions, "sanity");
 197   _initial_size = num_committed_regions * reg_size_bytes;
 198 
 199   size_t num_min_regions = min_byte_size / reg_size_bytes;
 200   num_min_regions = MIN2(num_min_regions, _num_regions);
 201   assert(num_min_regions <= _num_regions, "sanity");
 202   _minimum_size = num_min_regions * reg_size_bytes;
 203 
 204   // Default to max heap size.
 205   _soft_max_size = _num_regions * reg_size_bytes;
 206 
 207   _committed = _initial_size;
 208 
 209   size_t heap_page_size   = UseLargePages ? os::large_page_size() : os::vm_page_size();
 210   size_t bitmap_page_size = UseLargePages ? os::large_page_size() : os::vm_page_size();
 211   size_t region_page_size = UseLargePages ? os::large_page_size() : os::vm_page_size();
 212 
 213   //
 214   // Reserve and commit memory for heap
 215   //
 216 
 217   ReservedHeapSpace heap_rs = Universe::reserve_heap(max_byte_size, heap_alignment);
 218   initialize_reserved_region(heap_rs);
 219   _heap_region = MemRegion((HeapWord*)heap_rs.base(), heap_rs.size() / HeapWordSize);
 220   _heap_region_special = heap_rs.special();
 221 
 222   assert((((size_t) base()) & ShenandoahHeapRegion::region_size_bytes_mask()) == 0,
 223          "Misaligned heap: " PTR_FORMAT, p2i(base()));
 224   os::trace_page_sizes_for_requested_size("Heap",
 225                                           max_byte_size, heap_alignment,
 226                                           heap_rs.base(),
 227                                           heap_rs.size(), heap_rs.page_size());
 228 
 229 #if SHENANDOAH_OPTIMIZED_MARKTASK
 230   // The optimized ShenandoahMarkTask takes some bits away from the full object bits.
 231   // Fail if we ever attempt to address more than we can.
 232   if ((uintptr_t)heap_rs.end() >= ShenandoahMarkTask::max_addressable()) {
 233     FormatBuffer<512> buf("Shenandoah reserved [" PTR_FORMAT ", " PTR_FORMAT") for the heap, \n"
 234                           "but max object address is " PTR_FORMAT ". Try to reduce heap size, or try other \n"
 235                           "VM options that allocate heap at lower addresses (HeapBaseMinAddress, AllocateHeapAt, etc).",
 236                 p2i(heap_rs.base()), p2i(heap_rs.end()), ShenandoahMarkTask::max_addressable());
 237     vm_exit_during_initialization("Fatal Error", buf);
 238   }
 239 #endif
 240 
 241   ReservedSpace sh_rs = heap_rs.first_part(max_byte_size);
 242   if (!_heap_region_special) {
 243     os::commit_memory_or_exit(sh_rs.base(), _initial_size, heap_alignment, false,
 244                               "Cannot commit heap memory");
 245   }
 246 
 247   BarrierSet::set_barrier_set(new ShenandoahBarrierSet(this, _heap_region));
 248 
 249   // Now we know the number of regions and heap sizes, initialize the heuristics.
 250   initialize_heuristics();
 251 
 252   assert(_heap_region.byte_size() == heap_rs.size(), "Need to know reserved size for card table");
 253 
 254   //
 255   // Worker threads must be initialized after the barrier is configured
 256   //
 257   _workers = new ShenandoahWorkerThreads("Shenandoah GC Threads", _max_workers);
 258   if (_workers == nullptr) {
 259     vm_exit_during_initialization("Failed necessary allocation.");
 260   } else {
 261     _workers->initialize_workers();
 262   }
 263 
 264   if (ParallelGCThreads > 1) {
 265     _safepoint_workers = new ShenandoahWorkerThreads("Safepoint Cleanup Thread", ParallelGCThreads);
 266     _safepoint_workers->initialize_workers();
 267   }
 268 
 269   //
 270   // Reserve and commit memory for bitmap(s)
 271   //
 272 
 273   size_t bitmap_size_orig = ShenandoahMarkBitMap::compute_size(heap_rs.size());
 274   _bitmap_size = align_up(bitmap_size_orig, bitmap_page_size);
 275 
 276   size_t bitmap_bytes_per_region = reg_size_bytes / ShenandoahMarkBitMap::heap_map_factor();
 277 
 278   guarantee(bitmap_bytes_per_region != 0,
 279             "Bitmap bytes per region should not be zero");
 280   guarantee(is_power_of_2(bitmap_bytes_per_region),
 281             "Bitmap bytes per region should be power of two: %zu", bitmap_bytes_per_region);
 282 
 283   if (bitmap_page_size > bitmap_bytes_per_region) {
 284     _bitmap_regions_per_slice = bitmap_page_size / bitmap_bytes_per_region;
 285     _bitmap_bytes_per_slice = bitmap_page_size;
 286   } else {
 287     _bitmap_regions_per_slice = 1;
 288     _bitmap_bytes_per_slice = bitmap_bytes_per_region;
 289   }
 290 
 291   guarantee(_bitmap_regions_per_slice >= 1,
 292             "Should have at least one region per slice: %zu",
 293             _bitmap_regions_per_slice);
 294 
 295   guarantee(((_bitmap_bytes_per_slice) % bitmap_page_size) == 0,
 296             "Bitmap slices should be page-granular: bps = %zu, page size = %zu",
 297             _bitmap_bytes_per_slice, bitmap_page_size);
 298 
 299   ReservedSpace bitmap = reserve(_bitmap_size, bitmap_page_size);
 300   os::trace_page_sizes_for_requested_size("Mark Bitmap",
 301                                           bitmap_size_orig, bitmap_page_size,
 302                                           bitmap.base(),
 303                                           bitmap.size(), bitmap.page_size());
 304   MemTracker::record_virtual_memory_tag(bitmap, mtGC);
 305   _bitmap_region = MemRegion((HeapWord*) bitmap.base(), bitmap.size() / HeapWordSize);
 306   _bitmap_region_special = bitmap.special();
 307 
 308   size_t bitmap_init_commit = _bitmap_bytes_per_slice *
 309     align_up(num_committed_regions, _bitmap_regions_per_slice) / _bitmap_regions_per_slice;
 310   bitmap_init_commit = MIN2(_bitmap_size, bitmap_init_commit);
 311   if (!_bitmap_region_special) {
 312     os::commit_memory_or_exit((char *) _bitmap_region.start(), bitmap_init_commit, bitmap_page_size, false,
 313                               "Cannot commit bitmap memory");
 314   }
 315 
 316   _marking_context = new ShenandoahMarkingContext(_heap_region, _bitmap_region, _num_regions);
 317 
 318   if (ShenandoahVerify) {
 319     ReservedSpace verify_bitmap = reserve(_bitmap_size, bitmap_page_size);
 320     os::trace_page_sizes_for_requested_size("Verify Bitmap",
 321                                             bitmap_size_orig, bitmap_page_size,
 322                                             verify_bitmap.base(),
 323                                             verify_bitmap.size(), verify_bitmap.page_size());
 324     if (!verify_bitmap.special()) {
 325       os::commit_memory_or_exit(verify_bitmap.base(), verify_bitmap.size(), bitmap_page_size, false,
 326                                 "Cannot commit verification bitmap memory");
 327     }
 328     MemTracker::record_virtual_memory_tag(verify_bitmap, mtGC);
 329     MemRegion verify_bitmap_region = MemRegion((HeapWord *) verify_bitmap.base(), verify_bitmap.size() / HeapWordSize);
 330     _verification_bit_map.initialize(_heap_region, verify_bitmap_region);
 331     _verifier = new ShenandoahVerifier(this, &_verification_bit_map);
 332   }
 333 
 334   // Reserve aux bitmap for use in object_iterate(). We don't commit it here.
 335   size_t aux_bitmap_page_size = bitmap_page_size;
 336 
 337   ReservedSpace aux_bitmap = reserve(_bitmap_size, aux_bitmap_page_size);
 338   os::trace_page_sizes_for_requested_size("Aux Bitmap",
 339                                           bitmap_size_orig, aux_bitmap_page_size,
 340                                           aux_bitmap.base(),
 341                                           aux_bitmap.size(), aux_bitmap.page_size());
 342   MemTracker::record_virtual_memory_tag(aux_bitmap, mtGC);
 343   _aux_bitmap_region = MemRegion((HeapWord*) aux_bitmap.base(), aux_bitmap.size() / HeapWordSize);
 344   _aux_bitmap_region_special = aux_bitmap.special();
 345   _aux_bit_map.initialize(_heap_region, _aux_bitmap_region);
 346 
 347   //
 348   // Create regions and region sets
 349   //
 350   size_t region_align = align_up(sizeof(ShenandoahHeapRegion), SHENANDOAH_CACHE_LINE_SIZE);
 351   size_t region_storage_size_orig = region_align * _num_regions;
 352   size_t region_storage_size = align_up(region_storage_size_orig,
 353                                         MAX2(region_page_size, os::vm_allocation_granularity()));
 354 
 355   ReservedSpace region_storage = reserve(region_storage_size, region_page_size);
 356   os::trace_page_sizes_for_requested_size("Region Storage",
 357                                           region_storage_size_orig, region_page_size,
 358                                           region_storage.base(),
 359                                           region_storage.size(), region_storage.page_size());
 360   MemTracker::record_virtual_memory_tag(region_storage, mtGC);
 361   if (!region_storage.special()) {
 362     os::commit_memory_or_exit(region_storage.base(), region_storage_size, region_page_size, false,
 363                               "Cannot commit region memory");
 364   }
 365 
 366   // Try to fit the collection set bitmap at lower addresses. This optimizes code generation for cset checks.
 367   // Go up until a sensible limit (subject to encoding constraints) and try to reserve the space there.
 368   // If not successful, bite a bullet and allocate at whatever address.
 369   {
 370     const size_t cset_align = MAX2<size_t>(os::vm_page_size(), os::vm_allocation_granularity());
 371     const size_t cset_size = align_up(((size_t) sh_rs.base() + sh_rs.size()) >> ShenandoahHeapRegion::region_size_bytes_shift(), cset_align);
 372     const size_t cset_page_size = os::vm_page_size();
 373 
 374     uintptr_t min = round_up_power_of_2(cset_align);
 375     uintptr_t max = (1u << 30u);
 376     ReservedSpace cset_rs;
 377 
 378     for (uintptr_t addr = min; addr <= max; addr <<= 1u) {
 379       char* req_addr = (char*)addr;
 380       assert(is_aligned(req_addr, cset_align), "Should be aligned");
 381       cset_rs = MemoryReserver::reserve(req_addr, cset_size, cset_align, cset_page_size, mtGC);
 382       if (cset_rs.is_reserved()) {
 383         assert(cset_rs.base() == req_addr, "Allocated where requested: " PTR_FORMAT ", " PTR_FORMAT, p2i(cset_rs.base()), addr);
 384         _collection_set = new ShenandoahCollectionSet(this, cset_rs, sh_rs.base());
 385         break;
 386       }
 387     }
 388 
 389     if (_collection_set == nullptr) {
 390       cset_rs = MemoryReserver::reserve(cset_size, cset_align, os::vm_page_size(), mtGC);
 391       if (!cset_rs.is_reserved()) {
 392         vm_exit_during_initialization("Cannot reserve memory for collection set");
 393       }
 394 
 395       _collection_set = new ShenandoahCollectionSet(this, cset_rs, sh_rs.base());
 396     }
 397     os::trace_page_sizes_for_requested_size("Collection Set",
 398                                             cset_size, cset_page_size,
 399                                             cset_rs.base(),
 400                                             cset_rs.size(), cset_rs.page_size());
 401   }
 402 
 403   _regions = NEW_C_HEAP_ARRAY(ShenandoahHeapRegion*, _num_regions, mtGC);
 404   _affiliations = NEW_C_HEAP_ARRAY(uint8_t, _num_regions, mtGC);
 405 
 406   {
 407     ShenandoahHeapLocker locker(lock());
 408     _free_set = new ShenandoahFreeSet(this, _num_regions);
 409     for (size_t i = 0; i < _num_regions; i++) {
 410       HeapWord* start = (HeapWord*)sh_rs.base() + ShenandoahHeapRegion::region_size_words() * i;
 411       bool is_committed = i < num_committed_regions;
 412       void* loc = region_storage.base() + i * region_align;
 413 
 414       ShenandoahHeapRegion* r = new (loc) ShenandoahHeapRegion(start, i, is_committed);
 415       assert(is_aligned(r, SHENANDOAH_CACHE_LINE_SIZE), "Sanity");
 416 
 417       _marking_context->initialize_top_at_mark_start(r);
 418       _regions[i] = r;
 419       assert(!collection_set()->is_in(i), "New region should not be in collection set");
 420 
 421       _affiliations[i] = ShenandoahAffiliation::FREE;
 422     }
 423 
 424     size_t young_cset_regions, old_cset_regions;
 425 
 426     // We are initializing free set.  We ignore cset region tallies.
 427     size_t first_old, last_old, num_old;
 428     _free_set->prepare_to_rebuild(young_cset_regions, old_cset_regions, first_old, last_old, num_old);
 429     _free_set->finish_rebuild(young_cset_regions, old_cset_regions, num_old);
 430   }
 431 
 432   if (AlwaysPreTouch) {
 433     // For NUMA, it is important to pre-touch the storage under bitmaps with worker threads,
 434     // before initialize() below zeroes it with initializing thread. For any given region,
 435     // we touch the region and the corresponding bitmaps from the same thread.
 436     ShenandoahPushWorkerScope scope(workers(), _max_workers, false);
 437 
 438     _pretouch_heap_page_size = heap_page_size;
 439     _pretouch_bitmap_page_size = bitmap_page_size;
 440 
 441     // OS memory managers may want to coalesce back-to-back pages. Make their jobs
 442     // simpler by pre-touching continuous spaces (heap and bitmap) separately.
 443 
 444     ShenandoahPretouchBitmapTask bcl(bitmap.base(), _bitmap_size, _pretouch_bitmap_page_size);
 445     _workers->run_task(&bcl);
 446 
 447     ShenandoahPretouchHeapTask hcl(_pretouch_heap_page_size);
 448     _workers->run_task(&hcl);
 449   }
 450 
 451   //
 452   // Initialize the rest of GC subsystems
 453   //
 454 
 455   _liveness_cache = NEW_C_HEAP_ARRAY(ShenandoahLiveData*, _max_workers, mtGC);
 456   for (uint worker = 0; worker < _max_workers; worker++) {
 457     _liveness_cache[worker] = NEW_C_HEAP_ARRAY(ShenandoahLiveData, _num_regions, mtGC);
 458     Copy::fill_to_bytes(_liveness_cache[worker], _num_regions * sizeof(ShenandoahLiveData));
 459   }
 460 
 461   // There should probably be Shenandoah-specific options for these,
 462   // just as there are G1-specific options.
 463   {
 464     ShenandoahSATBMarkQueueSet& satbqs = ShenandoahBarrierSet::satb_mark_queue_set();
 465     satbqs.set_process_completed_buffers_threshold(20); // G1SATBProcessCompletedThreshold
 466     satbqs.set_buffer_enqueue_threshold_percentage(60); // G1SATBBufferEnqueueingThresholdPercent
 467   }
 468 
 469   _monitoring_support = new ShenandoahMonitoringSupport(this);
 470   _phase_timings = new ShenandoahPhaseTimings(max_workers());
 471   ShenandoahCodeRoots::initialize();
 472 
 473   if (ShenandoahPacing) {
 474     _pacer = new ShenandoahPacer(this);
 475     _pacer->setup_for_idle();
 476   }
 477 
 478   initialize_controller();
 479 
 480   if (ShenandoahUncommit) {
 481     _uncommit_thread = new ShenandoahUncommitThread(this);
 482   }
 483 
 484   print_init_logger();
 485 
 486   FullGCForwarding::initialize(_heap_region);
 487 
 488   return JNI_OK;
 489 }
 490 
 491 void ShenandoahHeap::initialize_controller() {
 492   _control_thread = new ShenandoahControlThread();
 493 }
 494 
 495 void ShenandoahHeap::print_init_logger() const {
 496   ShenandoahInitLogger::print();
 497 }
 498 
 499 void ShenandoahHeap::initialize_mode() {
 500   if (ShenandoahGCMode != nullptr) {
 501     if (strcmp(ShenandoahGCMode, "satb") == 0) {
 502       _gc_mode = new ShenandoahSATBMode();
 503     } else if (strcmp(ShenandoahGCMode, "passive") == 0) {
 504       _gc_mode = new ShenandoahPassiveMode();
 505     } else if (strcmp(ShenandoahGCMode, "generational") == 0) {
 506       _gc_mode = new ShenandoahGenerationalMode();
 507     } else {
 508       vm_exit_during_initialization("Unknown -XX:ShenandoahGCMode option");
 509     }
 510   } else {
 511     vm_exit_during_initialization("Unknown -XX:ShenandoahGCMode option (null)");
 512   }
 513   _gc_mode->initialize_flags();
 514   if (_gc_mode->is_diagnostic() && !UnlockDiagnosticVMOptions) {
 515     vm_exit_during_initialization(
 516             err_msg("GC mode \"%s\" is diagnostic, and must be enabled via -XX:+UnlockDiagnosticVMOptions.",
 517                     _gc_mode->name()));
 518   }
 519   if (_gc_mode->is_experimental() && !UnlockExperimentalVMOptions) {
 520     vm_exit_during_initialization(
 521             err_msg("GC mode \"%s\" is experimental, and must be enabled via -XX:+UnlockExperimentalVMOptions.",
 522                     _gc_mode->name()));
 523   }
 524 }
 525 
 526 void ShenandoahHeap::initialize_heuristics() {
 527   _global_generation = new ShenandoahGlobalGeneration(mode()->is_generational(), max_workers(), max_capacity(), max_capacity());
 528   _global_generation->initialize_heuristics(mode());
 529 }
 530 
 531 #ifdef _MSC_VER
 532 #pragma warning( push )
 533 #pragma warning( disable:4355 ) // 'this' : used in base member initializer list
 534 #endif
 535 
 536 ShenandoahHeap::ShenandoahHeap(ShenandoahCollectorPolicy* policy) :
 537   CollectedHeap(),
 538   _gc_generation(nullptr),
 539   _active_generation(nullptr),
 540   _initial_size(0),
 541   _committed(0),
 542   _max_workers(MAX3(ConcGCThreads, ParallelGCThreads, 1U)),
 543   _workers(nullptr),
 544   _safepoint_workers(nullptr),
 545   _heap_region_special(false),
 546   _num_regions(0),
 547   _regions(nullptr),
 548   _affiliations(nullptr),
 549   _gc_state_changed(false),
 550   _gc_no_progress_count(0),
 551   _cancel_requested_time(0),
 552   _update_refs_iterator(this),
 553   _global_generation(nullptr),
 554   _control_thread(nullptr),
 555   _uncommit_thread(nullptr),
 556   _young_generation(nullptr),
 557   _old_generation(nullptr),
 558   _shenandoah_policy(policy),
 559   _gc_mode(nullptr),
 560   _free_set(nullptr),
 561   _pacer(nullptr),
 562   _verifier(nullptr),
 563   _phase_timings(nullptr),
 564   _monitoring_support(nullptr),
 565   _memory_pool(nullptr),
 566   _stw_memory_manager("Shenandoah Pauses"),
 567   _cycle_memory_manager("Shenandoah Cycles"),
 568   _gc_timer(new ConcurrentGCTimer()),
 569   _log_min_obj_alignment_in_bytes(LogMinObjAlignmentInBytes),
 570   _marking_context(nullptr),
 571   _bitmap_size(0),
 572   _bitmap_regions_per_slice(0),
 573   _bitmap_bytes_per_slice(0),
 574   _bitmap_region_special(false),
 575   _aux_bitmap_region_special(false),
 576   _liveness_cache(nullptr),
 577   _collection_set(nullptr)
 578 {
 579   // Initialize GC mode early, many subsequent initialization procedures depend on it
 580   initialize_mode();
 581   _cancelled_gc.set(GCCause::_no_gc);
 582 }
 583 
 584 #ifdef _MSC_VER
 585 #pragma warning( pop )
 586 #endif
 587 
 588 void ShenandoahHeap::print_heap_on(outputStream* st) const {
 589   st->print_cr("Shenandoah Heap");
 590   st->print_cr(" %zu%s max, %zu%s soft max, %zu%s committed, %zu%s used",
 591                byte_size_in_proper_unit(max_capacity()), proper_unit_for_byte_size(max_capacity()),
 592                byte_size_in_proper_unit(soft_max_capacity()), proper_unit_for_byte_size(soft_max_capacity()),
 593                byte_size_in_proper_unit(committed()),    proper_unit_for_byte_size(committed()),
 594                byte_size_in_proper_unit(used()),         proper_unit_for_byte_size(used()));
 595   st->print_cr(" %zu x %zu %s regions",
 596                num_regions(),
 597                byte_size_in_proper_unit(ShenandoahHeapRegion::region_size_bytes()),
 598                proper_unit_for_byte_size(ShenandoahHeapRegion::region_size_bytes()));
 599 
 600   st->print("Status: ");
 601   if (has_forwarded_objects())                 st->print("has forwarded objects, ");
 602   if (!mode()->is_generational()) {
 603     if (is_concurrent_mark_in_progress())      st->print("marking,");
 604   } else {
 605     if (is_concurrent_old_mark_in_progress())    st->print("old marking, ");
 606     if (is_concurrent_young_mark_in_progress())  st->print("young marking, ");
 607   }
 608   if (is_evacuation_in_progress())             st->print("evacuating, ");
 609   if (is_update_refs_in_progress())            st->print("updating refs, ");
 610   if (is_degenerated_gc_in_progress())         st->print("degenerated gc, ");
 611   if (is_full_gc_in_progress())                st->print("full gc, ");
 612   if (is_full_gc_move_in_progress())           st->print("full gc move, ");
 613   if (is_concurrent_weak_root_in_progress())   st->print("concurrent weak roots, ");
 614   if (is_concurrent_strong_root_in_progress() &&
 615       !is_concurrent_weak_root_in_progress())  st->print("concurrent strong roots, ");
 616 
 617   if (cancelled_gc()) {
 618     st->print("cancelled");
 619   } else {
 620     st->print("not cancelled");
 621   }
 622   st->cr();
 623 
 624   st->print_cr("Reserved region:");
 625   st->print_cr(" - [" PTR_FORMAT ", " PTR_FORMAT ") ",
 626                p2i(reserved_region().start()),
 627                p2i(reserved_region().end()));
 628 
 629   ShenandoahCollectionSet* cset = collection_set();
 630   st->print_cr("Collection set:");
 631   if (cset != nullptr) {
 632     st->print_cr(" - map (vanilla): " PTR_FORMAT, p2i(cset->map_address()));
 633     st->print_cr(" - map (biased):  " PTR_FORMAT, p2i(cset->biased_map_address()));
 634   } else {
 635     st->print_cr(" (null)");
 636   }
 637 
 638   st->cr();
 639 
 640   if (Verbose) {
 641     st->cr();
 642     print_heap_regions_on(st);
 643   }
 644 }
 645 
 646 void ShenandoahHeap::print_gc_on(outputStream* st) const {
 647   print_heap_regions_on(st);
 648 }
 649 
 650 class ShenandoahInitWorkerGCLABClosure : public ThreadClosure {
 651 public:
 652   void do_thread(Thread* thread) {
 653     assert(thread != nullptr, "Sanity");
 654     ShenandoahThreadLocalData::initialize_gclab(thread);
 655   }
 656 };
 657 
 658 void ShenandoahHeap::post_initialize() {
 659   CollectedHeap::post_initialize();
 660 
 661   // Schedule periodic task to report on gc thread CPU utilization
 662   _mmu_tracker.initialize();
 663 
 664   MutexLocker ml(Threads_lock);
 665 
 666   ShenandoahInitWorkerGCLABClosure init_gclabs;
 667   _workers->threads_do(&init_gclabs);
 668 
 669   // gclab can not be initialized early during VM startup, as it can not determinate its max_size.
 670   // Now, we will let WorkerThreads to initialize gclab when new worker is created.
 671   _workers->set_initialize_gclab();
 672 
 673   // Note that the safepoint workers may require gclabs if the threads are used to create a heap dump
 674   // during a concurrent evacuation phase.
 675   if (_safepoint_workers != nullptr) {
 676     _safepoint_workers->threads_do(&init_gclabs);
 677     _safepoint_workers->set_initialize_gclab();
 678   }
 679 
 680   JFR_ONLY(ShenandoahJFRSupport::register_jfr_type_serializers();)
 681 }
 682 
 683 ShenandoahHeuristics* ShenandoahHeap::heuristics() {
 684   return _global_generation->heuristics();
 685 }
 686 
 687 size_t ShenandoahHeap::used() const {
 688   return global_generation()->used();
 689 }
 690 
 691 size_t ShenandoahHeap::committed() const {
 692   return Atomic::load(&_committed);
 693 }
 694 
 695 void ShenandoahHeap::increase_committed(size_t bytes) {
 696   shenandoah_assert_heaplocked_or_safepoint();
 697   _committed += bytes;
 698 }
 699 
 700 void ShenandoahHeap::decrease_committed(size_t bytes) {
 701   shenandoah_assert_heaplocked_or_safepoint();
 702   _committed -= bytes;
 703 }
 704 
 705 // For tracking usage based on allocations, it should be the case that:
 706 // * The sum of regions::used == heap::used
 707 // * The sum of a generation's regions::used == generation::used
 708 // * The sum of a generation's humongous regions::free == generation::humongous_waste
 709 // These invariants are checked by the verifier on GC safepoints.
 710 //
 711 // Additional notes:
 712 // * When a mutator's allocation request causes a region to be retired, the
 713 //   free memory left in that region is considered waste. It does not contribute
 714 //   to the usage, but it _does_ contribute to allocation rate.
 715 // * The bottom of a PLAB must be aligned on card size. In some cases this will
 716 //   require padding in front of the PLAB (a filler object). Because this padding
 717 //   is included in the region's used memory we include the padding in the usage
 718 //   accounting as waste.
 719 // * Mutator allocations are used to compute an allocation rate. They are also
 720 //   sent to the Pacer for those purposes.
 721 // * There are three sources of waste:
 722 //  1. The padding used to align a PLAB on card size
 723 //  2. Region's free is less than minimum TLAB size and is retired
 724 //  3. The unused portion of memory in the last region of a humongous object
 725 void ShenandoahHeap::increase_used(const ShenandoahAllocRequest& req) {
 726   size_t actual_bytes = req.actual_size() * HeapWordSize;
 727   size_t wasted_bytes = req.waste() * HeapWordSize;
 728   ShenandoahGeneration* generation = generation_for(req.affiliation());
 729 
 730   if (req.is_gc_alloc()) {
 731     assert(wasted_bytes == 0 || req.type() == ShenandoahAllocRequest::_alloc_plab, "Only PLABs have waste");
 732     increase_used(generation, actual_bytes + wasted_bytes);
 733   } else {
 734     assert(req.is_mutator_alloc(), "Expected mutator alloc here");
 735     // padding and actual size both count towards allocation counter
 736     generation->increase_allocated(actual_bytes + wasted_bytes);
 737 
 738     // only actual size counts toward usage for mutator allocations
 739     increase_used(generation, actual_bytes);
 740 
 741     // notify pacer of both actual size and waste
 742     notify_mutator_alloc_words(req.actual_size(), req.waste());
 743 
 744     if (wasted_bytes > 0 && ShenandoahHeapRegion::requires_humongous(req.actual_size())) {
 745       increase_humongous_waste(generation,wasted_bytes);
 746     }
 747   }
 748 }
 749 
 750 void ShenandoahHeap::increase_humongous_waste(ShenandoahGeneration* generation, size_t bytes) {
 751   generation->increase_humongous_waste(bytes);
 752   if (!generation->is_global()) {
 753     global_generation()->increase_humongous_waste(bytes);
 754   }
 755 }
 756 
 757 void ShenandoahHeap::decrease_humongous_waste(ShenandoahGeneration* generation, size_t bytes) {
 758   generation->decrease_humongous_waste(bytes);
 759   if (!generation->is_global()) {
 760     global_generation()->decrease_humongous_waste(bytes);
 761   }
 762 }
 763 
 764 void ShenandoahHeap::increase_used(ShenandoahGeneration* generation, size_t bytes) {
 765   generation->increase_used(bytes);
 766   if (!generation->is_global()) {
 767     global_generation()->increase_used(bytes);
 768   }
 769 }
 770 
 771 void ShenandoahHeap::decrease_used(ShenandoahGeneration* generation, size_t bytes) {
 772   generation->decrease_used(bytes);
 773   if (!generation->is_global()) {
 774     global_generation()->decrease_used(bytes);
 775   }
 776 }
 777 
 778 void ShenandoahHeap::notify_mutator_alloc_words(size_t words, size_t waste) {
 779   if (ShenandoahPacing) {
 780     control_thread()->pacing_notify_alloc(words);
 781     if (waste > 0) {
 782       pacer()->claim_for_alloc<true>(waste);
 783     }
 784   }
 785 }
 786 
 787 size_t ShenandoahHeap::capacity() const {
 788   return committed();
 789 }
 790 
 791 size_t ShenandoahHeap::max_capacity() const {
 792   return _num_regions * ShenandoahHeapRegion::region_size_bytes();
 793 }
 794 
 795 size_t ShenandoahHeap::soft_max_capacity() const {
 796   size_t v = Atomic::load(&_soft_max_size);
 797   assert(min_capacity() <= v && v <= max_capacity(),
 798          "Should be in bounds: %zu <= %zu <= %zu",
 799          min_capacity(), v, max_capacity());
 800   return v;
 801 }
 802 
 803 void ShenandoahHeap::set_soft_max_capacity(size_t v) {
 804   assert(min_capacity() <= v && v <= max_capacity(),
 805          "Should be in bounds: %zu <= %zu <= %zu",
 806          min_capacity(), v, max_capacity());
 807   Atomic::store(&_soft_max_size, v);
 808 }
 809 
 810 size_t ShenandoahHeap::min_capacity() const {
 811   return _minimum_size;
 812 }
 813 
 814 size_t ShenandoahHeap::initial_capacity() const {
 815   return _initial_size;
 816 }
 817 
 818 bool ShenandoahHeap::is_in(const void* p) const {
 819   if (!is_in_reserved(p)) {
 820     return false;
 821   }
 822 
 823   if (is_full_gc_move_in_progress()) {
 824     // Full GC move is running, we do not have a consistent region
 825     // information yet. But we know the pointer is in heap.
 826     return true;
 827   }
 828 
 829   // Now check if we point to a live section in active region.
 830   const ShenandoahHeapRegion* r = heap_region_containing(p);
 831   if (p >= r->top()) {
 832     return false;
 833   }
 834 
 835   if (r->is_active()) {
 836     return true;
 837   }
 838 
 839   // The region is trash, but won't be recycled until after concurrent weak
 840   // roots. We also don't allow mutators to allocate from trash regions
 841   // during weak roots. Concurrent class unloading may access unmarked oops
 842   // in trash regions.
 843   return r->is_trash() && is_concurrent_weak_root_in_progress();
 844 }
 845 
 846 void ShenandoahHeap::notify_soft_max_changed() {
 847   if (_uncommit_thread != nullptr) {
 848     _uncommit_thread->notify_soft_max_changed();
 849   }
 850 }
 851 
 852 void ShenandoahHeap::notify_explicit_gc_requested() {
 853   if (_uncommit_thread != nullptr) {
 854     _uncommit_thread->notify_explicit_gc_requested();
 855   }
 856 }
 857 
 858 bool ShenandoahHeap::check_soft_max_changed() {
 859   size_t new_soft_max = Atomic::load(&SoftMaxHeapSize);
 860   size_t old_soft_max = soft_max_capacity();
 861   if (new_soft_max != old_soft_max) {
 862     new_soft_max = MAX2(min_capacity(), new_soft_max);
 863     new_soft_max = MIN2(max_capacity(), new_soft_max);
 864     if (new_soft_max != old_soft_max) {
 865       log_info(gc)("Soft Max Heap Size: %zu%s -> %zu%s",
 866                    byte_size_in_proper_unit(old_soft_max), proper_unit_for_byte_size(old_soft_max),
 867                    byte_size_in_proper_unit(new_soft_max), proper_unit_for_byte_size(new_soft_max)
 868       );
 869       set_soft_max_capacity(new_soft_max);
 870       return true;
 871     }
 872   }
 873   return false;
 874 }
 875 
 876 void ShenandoahHeap::notify_heap_changed() {
 877   // Update monitoring counters when we took a new region. This amortizes the
 878   // update costs on slow path.
 879   monitoring_support()->notify_heap_changed();
 880   _heap_changed.try_set();
 881 }
 882 
 883 void ShenandoahHeap::set_forced_counters_update(bool value) {
 884   monitoring_support()->set_forced_counters_update(value);
 885 }
 886 
 887 void ShenandoahHeap::handle_force_counters_update() {
 888   monitoring_support()->handle_force_counters_update();
 889 }
 890 
 891 HeapWord* ShenandoahHeap::allocate_from_gclab_slow(Thread* thread, size_t size) {
 892   // New object should fit the GCLAB size
 893   size_t min_size = MAX2(size, PLAB::min_size());
 894 
 895   // Figure out size of new GCLAB, looking back at heuristics. Expand aggressively.
 896   size_t new_size = ShenandoahThreadLocalData::gclab_size(thread) * 2;
 897 
 898   new_size = MIN2(new_size, PLAB::max_size());
 899   new_size = MAX2(new_size, PLAB::min_size());
 900 
 901   // Record new heuristic value even if we take any shortcut. This captures
 902   // the case when moderately-sized objects always take a shortcut. At some point,
 903   // heuristics should catch up with them.
 904   log_debug(gc, free)("Set new GCLAB size: %zu", new_size);
 905   ShenandoahThreadLocalData::set_gclab_size(thread, new_size);
 906 
 907   if (new_size < size) {
 908     // New size still does not fit the object. Fall back to shared allocation.
 909     // This avoids retiring perfectly good GCLABs, when we encounter a large object.
 910     log_debug(gc, free)("New gclab size (%zu) is too small for %zu", new_size, size);
 911     return nullptr;
 912   }
 913 
 914   // Retire current GCLAB, and allocate a new one.
 915   PLAB* gclab = ShenandoahThreadLocalData::gclab(thread);
 916   gclab->retire();
 917 
 918   size_t actual_size = 0;
 919   HeapWord* gclab_buf = allocate_new_gclab(min_size, new_size, &actual_size);
 920   if (gclab_buf == nullptr) {
 921     return nullptr;
 922   }
 923 
 924   assert (size <= actual_size, "allocation should fit");
 925 
 926   // ...and clear or zap just allocated TLAB, if needed.
 927   if (ZeroTLAB) {
 928     Copy::zero_to_words(gclab_buf, actual_size);
 929   } else if (ZapTLAB) {
 930     // Skip mangling the space corresponding to the object header to
 931     // ensure that the returned space is not considered parsable by
 932     // any concurrent GC thread.
 933     size_t hdr_size = oopDesc::header_size();
 934     Copy::fill_to_words(gclab_buf + hdr_size, actual_size - hdr_size, badHeapWordVal);
 935   }
 936   gclab->set_buf(gclab_buf, actual_size);
 937   return gclab->allocate(size);
 938 }
 939 
 940 // Called from stubs in JIT code or interpreter
 941 HeapWord* ShenandoahHeap::allocate_new_tlab(size_t min_size,
 942                                             size_t requested_size,
 943                                             size_t* actual_size) {
 944   ShenandoahAllocRequest req = ShenandoahAllocRequest::for_tlab(min_size, requested_size);
 945   HeapWord* res = allocate_memory(req);
 946   if (res != nullptr) {
 947     *actual_size = req.actual_size();
 948   } else {
 949     *actual_size = 0;
 950   }
 951   return res;
 952 }
 953 
 954 HeapWord* ShenandoahHeap::allocate_new_gclab(size_t min_size,
 955                                              size_t word_size,
 956                                              size_t* actual_size) {
 957   ShenandoahAllocRequest req = ShenandoahAllocRequest::for_gclab(min_size, word_size);
 958   HeapWord* res = allocate_memory(req);
 959   if (res != nullptr) {
 960     *actual_size = req.actual_size();
 961   } else {
 962     *actual_size = 0;
 963   }
 964   return res;
 965 }
 966 
 967 HeapWord* ShenandoahHeap::allocate_memory(ShenandoahAllocRequest& req) {
 968   intptr_t pacer_epoch = 0;
 969   bool in_new_region = false;
 970   HeapWord* result = nullptr;
 971 
 972   if (req.is_mutator_alloc()) {
 973     if (ShenandoahPacing) {
 974       pacer()->pace_for_alloc(req.size());
 975       pacer_epoch = pacer()->epoch();
 976     }
 977 
 978     if (!ShenandoahAllocFailureALot || !should_inject_alloc_failure()) {
 979       result = allocate_memory_under_lock(req, in_new_region);
 980     }
 981 
 982     // Check that gc overhead is not exceeded.
 983     //
 984     // Shenandoah will grind along for quite a while allocating one
 985     // object at a time using shared (non-tlab) allocations. This check
 986     // is testing that the GC overhead limit has not been exceeded.
 987     // This will notify the collector to start a cycle, but will raise
 988     // an OOME to the mutator if the last Full GCs have not made progress.
 989     // gc_no_progress_count is incremented following each degen or full GC that fails to achieve is_good_progress().
 990     if (result == nullptr && !req.is_lab_alloc() && get_gc_no_progress_count() > ShenandoahNoProgressThreshold) {
 991       control_thread()->handle_alloc_failure(req, false);
 992       req.set_actual_size(0);
 993       return nullptr;
 994     }
 995 
 996     if (result == nullptr) {
 997       // Block until control thread reacted, then retry allocation.
 998       //
 999       // It might happen that one of the threads requesting allocation would unblock
1000       // way later after GC happened, only to fail the second allocation, because
1001       // other threads have already depleted the free storage. In this case, a better
1002       // strategy is to try again, until at least one full GC has completed.
1003       //
1004       // Stop retrying and return nullptr to cause OOMError exception if our allocation failed even after:
1005       //   a) We experienced a GC that had good progress, or
1006       //   b) We experienced at least one Full GC (whether or not it had good progress)
1007 
1008       const size_t original_count = shenandoah_policy()->full_gc_count();
1009       while (result == nullptr && should_retry_allocation(original_count)) {
1010         control_thread()->handle_alloc_failure(req, true);
1011         result = allocate_memory_under_lock(req, in_new_region);
1012       }
1013       if (result != nullptr) {
1014         // If our allocation request has been satisfied after it initially failed, we count this as good gc progress
1015         notify_gc_progress();
1016       }
1017       if (log_develop_is_enabled(Debug, gc, alloc)) {
1018         ResourceMark rm;
1019         log_debug(gc, alloc)("Thread: %s, Result: " PTR_FORMAT ", Request: %s, Size: %zu"
1020                              ", Original: %zu, Latest: %zu",
1021                              Thread::current()->name(), p2i(result), req.type_string(), req.size(),
1022                              original_count, get_gc_no_progress_count());
1023       }
1024     }
1025   } else {
1026     assert(req.is_gc_alloc(), "Can only accept GC allocs here");
1027     result = allocate_memory_under_lock(req, in_new_region);
1028     // Do not call handle_alloc_failure() here, because we cannot block.
1029     // The allocation failure would be handled by the LRB slowpath with handle_alloc_failure_evac().
1030   }
1031 
1032   if (in_new_region) {
1033     notify_heap_changed();
1034   }
1035 
1036   if (result == nullptr) {
1037     req.set_actual_size(0);
1038   }
1039 
1040   // This is called regardless of the outcome of the allocation to account
1041   // for any waste created by retiring regions with this request.
1042   increase_used(req);
1043 
1044   if (result != nullptr) {
1045     size_t requested = req.size();
1046     size_t actual = req.actual_size();
1047 
1048     assert (req.is_lab_alloc() || (requested == actual),
1049             "Only LAB allocations are elastic: %s, requested = %zu, actual = %zu",
1050             ShenandoahAllocRequest::alloc_type_to_string(req.type()), requested, actual);
1051 
1052     if (req.is_mutator_alloc()) {
1053       // If we requested more than we were granted, give the rest back to pacer.
1054       // This only matters if we are in the same pacing epoch: do not try to unpace
1055       // over the budget for the other phase.
1056       if (ShenandoahPacing && (pacer_epoch > 0) && (requested > actual)) {
1057         pacer()->unpace_for_alloc(pacer_epoch, requested - actual);
1058       }
1059     }
1060   }
1061 
1062   return result;
1063 }
1064 
1065 inline bool ShenandoahHeap::should_retry_allocation(size_t original_full_gc_count) const {
1066   return shenandoah_policy()->full_gc_count() == original_full_gc_count
1067       && !shenandoah_policy()->is_at_shutdown();
1068 }
1069 
1070 HeapWord* ShenandoahHeap::allocate_memory_under_lock(ShenandoahAllocRequest& req, bool& in_new_region) {
1071   // If we are dealing with mutator allocation, then we may need to block for safepoint.
1072   // We cannot block for safepoint for GC allocations, because there is a high chance
1073   // we are already running at safepoint or from stack watermark machinery, and we cannot
1074   // block again.
1075   ShenandoahHeapLocker locker(lock(), req.is_mutator_alloc());
1076 
1077   // Make sure the old generation has room for either evacuations or promotions before trying to allocate.
1078   if (req.is_old() && !old_generation()->can_allocate(req)) {
1079     return nullptr;
1080   }
1081 
1082   // If TLAB request size is greater than available, allocate() will attempt to downsize request to fit within available
1083   // memory.
1084   HeapWord* result = _free_set->allocate(req, in_new_region);
1085 
1086   // Record the plab configuration for this result and register the object.
1087   if (result != nullptr && req.is_old()) {
1088     old_generation()->configure_plab_for_current_thread(req);
1089     if (req.type() == ShenandoahAllocRequest::_alloc_shared_gc) {
1090       // Register the newly allocated object while we're holding the global lock since there's no synchronization
1091       // built in to the implementation of register_object().  There are potential races when multiple independent
1092       // threads are allocating objects, some of which might span the same card region.  For example, consider
1093       // a card table's memory region within which three objects are being allocated by three different threads:
1094       //
1095       // objects being "concurrently" allocated:
1096       //    [-----a------][-----b-----][--------------c------------------]
1097       //            [---- card table memory range --------------]
1098       //
1099       // Before any objects are allocated, this card's memory range holds no objects.  Note that allocation of object a
1100       // wants to set the starts-object, first-start, and last-start attributes of the preceding card region.
1101       // Allocation of object b wants to set the starts-object, first-start, and last-start attributes of this card region.
1102       // Allocation of object c also wants to set the starts-object, first-start, and last-start attributes of this
1103       // card region.
1104       //
1105       // The thread allocating b and the thread allocating c can "race" in various ways, resulting in confusion, such as
1106       // last-start representing object b while first-start represents object c.  This is why we need to require all
1107       // register_object() invocations to be "mutually exclusive" with respect to each card's memory range.
1108       old_generation()->card_scan()->register_object(result);
1109     }
1110   }
1111 
1112   return result;
1113 }
1114 
1115 HeapWord* ShenandoahHeap::mem_allocate(size_t size,
1116                                         bool*  gc_overhead_limit_was_exceeded) {
1117   ShenandoahAllocRequest req = ShenandoahAllocRequest::for_shared(size);
1118   return allocate_memory(req);
1119 }
1120 
1121 MetaWord* ShenandoahHeap::satisfy_failed_metadata_allocation(ClassLoaderData* loader_data,
1122                                                              size_t size,
1123                                                              Metaspace::MetadataType mdtype) {
1124   MetaWord* result;
1125 
1126   // Inform metaspace OOM to GC heuristics if class unloading is possible.
1127   ShenandoahHeuristics* h = global_generation()->heuristics();
1128   if (h->can_unload_classes()) {
1129     h->record_metaspace_oom();
1130   }
1131 
1132   // Expand and retry allocation
1133   result = loader_data->metaspace_non_null()->expand_and_allocate(size, mdtype);
1134   if (result != nullptr) {
1135     return result;
1136   }
1137 
1138   // Start full GC
1139   collect(GCCause::_metadata_GC_clear_soft_refs);
1140 
1141   // Retry allocation
1142   result = loader_data->metaspace_non_null()->allocate(size, mdtype);
1143   if (result != nullptr) {
1144     return result;
1145   }
1146 
1147   // Expand and retry allocation
1148   result = loader_data->metaspace_non_null()->expand_and_allocate(size, mdtype);
1149   if (result != nullptr) {
1150     return result;
1151   }
1152 
1153   // Out of memory
1154   return nullptr;
1155 }
1156 
1157 class ShenandoahConcurrentEvacuateRegionObjectClosure : public ObjectClosure {
1158 private:
1159   ShenandoahHeap* const _heap;
1160   Thread* const _thread;
1161 public:
1162   ShenandoahConcurrentEvacuateRegionObjectClosure(ShenandoahHeap* heap) :
1163     _heap(heap), _thread(Thread::current()) {}
1164 
1165   void do_object(oop p) {
1166     shenandoah_assert_marked(nullptr, p);
1167     if (!p->is_forwarded()) {
1168       _heap->evacuate_object(p, _thread);
1169     }
1170   }
1171 };
1172 
1173 class ShenandoahEvacuationTask : public WorkerTask {
1174 private:
1175   ShenandoahHeap* const _sh;
1176   ShenandoahCollectionSet* const _cs;
1177   bool _concurrent;
1178 public:
1179   ShenandoahEvacuationTask(ShenandoahHeap* sh,
1180                            ShenandoahCollectionSet* cs,
1181                            bool concurrent) :
1182     WorkerTask("Shenandoah Evacuation"),
1183     _sh(sh),
1184     _cs(cs),
1185     _concurrent(concurrent)
1186   {}
1187 
1188   void work(uint worker_id) {
1189     if (_concurrent) {
1190       ShenandoahConcurrentWorkerSession worker_session(worker_id);
1191       ShenandoahSuspendibleThreadSetJoiner stsj;
1192       ShenandoahEvacOOMScope oom_evac_scope;
1193       do_work();
1194     } else {
1195       ShenandoahParallelWorkerSession worker_session(worker_id);
1196       ShenandoahEvacOOMScope oom_evac_scope;
1197       do_work();
1198     }
1199   }
1200 
1201 private:
1202   void do_work() {
1203     ShenandoahConcurrentEvacuateRegionObjectClosure cl(_sh);
1204     ShenandoahHeapRegion* r;
1205     while ((r =_cs->claim_next()) != nullptr) {
1206       assert(r->has_live(), "Region %zu should have been reclaimed early", r->index());
1207       _sh->marked_object_iterate(r, &cl);
1208 
1209       if (ShenandoahPacing) {
1210         _sh->pacer()->report_evac(r->used() >> LogHeapWordSize);
1211       }
1212 
1213       if (_sh->check_cancelled_gc_and_yield(_concurrent)) {
1214         break;
1215       }
1216     }
1217   }
1218 };
1219 
1220 class ShenandoahRetireGCLABClosure : public ThreadClosure {
1221 private:
1222   bool const _resize;
1223 public:
1224   explicit ShenandoahRetireGCLABClosure(bool resize) : _resize(resize) {}
1225   void do_thread(Thread* thread) override {
1226     PLAB* gclab = ShenandoahThreadLocalData::gclab(thread);
1227     assert(gclab != nullptr, "GCLAB should be initialized for %s", thread->name());
1228     gclab->retire();
1229     if (_resize && ShenandoahThreadLocalData::gclab_size(thread) > 0) {
1230       ShenandoahThreadLocalData::set_gclab_size(thread, 0);
1231     }
1232 
1233     if (ShenandoahHeap::heap()->mode()->is_generational()) {
1234       PLAB* plab = ShenandoahThreadLocalData::plab(thread);
1235       assert(plab != nullptr, "PLAB should be initialized for %s", thread->name());
1236 
1237       // There are two reasons to retire all plabs between old-gen evacuation passes.
1238       //  1. We need to make the plab memory parsable by remembered-set scanning.
1239       //  2. We need to establish a trustworthy UpdateWaterMark value within each old-gen heap region
1240       ShenandoahGenerationalHeap::heap()->retire_plab(plab, thread);
1241       if (_resize && ShenandoahThreadLocalData::plab_size(thread) > 0) {
1242         ShenandoahThreadLocalData::set_plab_size(thread, 0);
1243       }
1244     }
1245   }
1246 };
1247 
1248 class ShenandoahGCStatePropagator : public HandshakeClosure {
1249 public:
1250   explicit ShenandoahGCStatePropagator(char gc_state) :
1251     HandshakeClosure("Shenandoah GC State Change"),
1252     _gc_state(gc_state) {}
1253 
1254   void do_thread(Thread* thread) override {
1255     ShenandoahThreadLocalData::set_gc_state(thread, _gc_state);
1256   }
1257 private:
1258   char _gc_state;
1259 };
1260 
1261 class ShenandoahPrepareForUpdateRefs : public HandshakeClosure {
1262 public:
1263   explicit ShenandoahPrepareForUpdateRefs(char gc_state) :
1264     HandshakeClosure("Shenandoah Prepare for Update Refs"),
1265     _retire(ResizeTLAB), _propagator(gc_state) {}
1266 
1267   void do_thread(Thread* thread) override {
1268     _propagator.do_thread(thread);
1269     if (ShenandoahThreadLocalData::gclab(thread) != nullptr) {
1270       _retire.do_thread(thread);
1271     }
1272   }
1273 private:
1274   ShenandoahRetireGCLABClosure _retire;
1275   ShenandoahGCStatePropagator _propagator;
1276 };
1277 
1278 void ShenandoahHeap::evacuate_collection_set(bool concurrent) {
1279   ShenandoahEvacuationTask task(this, _collection_set, concurrent);
1280   workers()->run_task(&task);
1281 }
1282 
1283 void ShenandoahHeap::concurrent_prepare_for_update_refs() {
1284   {
1285     // Java threads take this lock while they are being attached and added to the list of thread.
1286     // If another thread holds this lock before we update the gc state, it will receive a stale
1287     // gc state, but they will have been added to the list of java threads and so will be corrected
1288     // by the following handshake.
1289     MutexLocker lock(Threads_lock);
1290 
1291     // A cancellation at this point means the degenerated cycle must resume from update-refs.
1292     set_gc_state_concurrent(EVACUATION, false);
1293     set_gc_state_concurrent(WEAK_ROOTS, false);
1294     set_gc_state_concurrent(UPDATE_REFS, true);
1295   }
1296 
1297   // This will propagate the gc state and retire gclabs and plabs for threads that require it.
1298   ShenandoahPrepareForUpdateRefs prepare_for_update_refs(_gc_state.raw_value());
1299 
1300   // The handshake won't touch worker threads (or control thread, or VM thread), so do those separately.
1301   Threads::non_java_threads_do(&prepare_for_update_refs);
1302 
1303   // Now retire gclabs and plabs and propagate gc_state for mutator threads
1304   Handshake::execute(&prepare_for_update_refs);
1305 
1306   _update_refs_iterator.reset();
1307 }
1308 
1309 class ShenandoahCompositeHandshakeClosure : public HandshakeClosure {
1310   HandshakeClosure* _handshake_1;
1311   HandshakeClosure* _handshake_2;
1312   public:
1313     ShenandoahCompositeHandshakeClosure(HandshakeClosure* handshake_1, HandshakeClosure* handshake_2) :
1314       HandshakeClosure(handshake_2->name()),
1315       _handshake_1(handshake_1), _handshake_2(handshake_2) {}
1316 
1317   void do_thread(Thread* thread) override {
1318       _handshake_1->do_thread(thread);
1319       _handshake_2->do_thread(thread);
1320     }
1321 };
1322 
1323 void ShenandoahHeap::concurrent_final_roots(HandshakeClosure* handshake_closure) {
1324   {
1325     assert(!is_evacuation_in_progress(), "Should not evacuate for abbreviated or old cycles");
1326     MutexLocker lock(Threads_lock);
1327     set_gc_state_concurrent(WEAK_ROOTS, false);
1328   }
1329 
1330   ShenandoahGCStatePropagator propagator(_gc_state.raw_value());
1331   Threads::non_java_threads_do(&propagator);
1332   if (handshake_closure == nullptr) {
1333     Handshake::execute(&propagator);
1334   } else {
1335     ShenandoahCompositeHandshakeClosure composite(&propagator, handshake_closure);
1336     Handshake::execute(&composite);
1337   }
1338 }
1339 
1340 oop ShenandoahHeap::evacuate_object(oop p, Thread* thread) {
1341   assert(thread == Thread::current(), "Expected thread parameter to be current thread.");
1342   if (ShenandoahThreadLocalData::is_oom_during_evac(thread)) {
1343     // This thread went through the OOM during evac protocol. It is safe to return
1344     // the forward pointer. It must not attempt to evacuate any other objects.
1345     return ShenandoahBarrierSet::resolve_forwarded(p);
1346   }
1347 
1348   assert(ShenandoahThreadLocalData::is_evac_allowed(thread), "must be enclosed in oom-evac scope");
1349 
1350   ShenandoahHeapRegion* r = heap_region_containing(p);
1351   assert(!r->is_humongous(), "never evacuate humongous objects");
1352 
1353   ShenandoahAffiliation target_gen = r->affiliation();
1354   return try_evacuate_object(p, thread, r, target_gen);
1355 }
1356 
1357 oop ShenandoahHeap::try_evacuate_object(oop p, Thread* thread, ShenandoahHeapRegion* from_region,
1358                                                ShenandoahAffiliation target_gen) {
1359   assert(target_gen == YOUNG_GENERATION, "Only expect evacuations to young in this mode");
1360   assert(from_region->is_young(), "Only expect evacuations from young in this mode");
1361   bool alloc_from_lab = true;
1362   HeapWord* copy = nullptr;
1363 
1364   markWord mark = p->mark();
1365   if (ShenandoahForwarding::is_forwarded(mark)) {
1366     return ShenandoahForwarding::get_forwardee(p);
1367   }
1368   size_t old_size = ShenandoahForwarding::size(p);
1369   size_t size = p->copy_size(old_size, mark);
1370 
1371 #ifdef ASSERT
1372   if (ShenandoahOOMDuringEvacALot &&
1373       (os::random() & 1) == 0) { // Simulate OOM every ~2nd slow-path call
1374     copy = nullptr;
1375   } else {
1376 #endif
1377     if (UseTLAB) {
1378       copy = allocate_from_gclab(thread, size);
1379     }
1380     if (copy == nullptr) {
1381       // If we failed to allocate in LAB, we'll try a shared allocation.
1382       ShenandoahAllocRequest req = ShenandoahAllocRequest::for_shared_gc(size, target_gen);
1383       copy = allocate_memory(req);
1384       alloc_from_lab = false;
1385     }
1386 #ifdef ASSERT
1387   }
1388 #endif
1389 
1390   if (copy == nullptr) {
1391     control_thread()->handle_alloc_failure_evac(size);
1392 
1393     _oom_evac_handler.handle_out_of_memory_during_evacuation();
1394 
1395     return ShenandoahBarrierSet::resolve_forwarded(p);
1396   }
1397 
1398   // Copy the object:
1399   Copy::aligned_disjoint_words(cast_from_oop<HeapWord*>(p), copy, old_size);
1400 
1401   // Try to install the new forwarding pointer.
1402   oop copy_val = cast_to_oop(copy);
1403   oop result = ShenandoahForwarding::try_update_forwardee(p, copy_val);
1404   if (result == copy_val) {
1405     // Successfully evacuated. Our copy is now the public one!
1406     copy_val->initialize_hash_if_necessary(p);
1407     ContinuationGCSupport::relativize_stack_chunk(copy_val);
1408     shenandoah_assert_correct(nullptr, copy_val);
1409     return copy_val;
1410   }  else {
1411     // Failed to evacuate. We need to deal with the object that is left behind. Since this
1412     // new allocation is certainly after TAMS, it will be considered live in the next cycle.
1413     // But if it happens to contain references to evacuated regions, those references would
1414     // not get updated for this stale copy during this cycle, and we will crash while scanning
1415     // it the next cycle.
1416     if (alloc_from_lab) {
1417       // For LAB allocations, it is enough to rollback the allocation ptr. Either the next
1418       // object will overwrite this stale copy, or the filler object on LAB retirement will
1419       // do this.
1420       ShenandoahThreadLocalData::gclab(thread)->undo_allocation(copy, size);
1421     } else {
1422       // For non-LAB allocations, we have no way to retract the allocation, and
1423       // have to explicitly overwrite the copy with the filler object. With that overwrite,
1424       // we have to keep the fwdptr initialized and pointing to our (stale) copy.
1425       assert(size >= ShenandoahHeap::min_fill_size(), "previously allocated object known to be larger than min_size");
1426       fill_with_object(copy, size);
1427       shenandoah_assert_correct(nullptr, copy_val);
1428       // For non-LAB allocations, the object has already been registered
1429     }
1430     shenandoah_assert_correct(nullptr, result);
1431     return result;
1432   }
1433 }
1434 
1435 void ShenandoahHeap::trash_cset_regions() {
1436   ShenandoahHeapLocker locker(lock());
1437 
1438   ShenandoahCollectionSet* set = collection_set();
1439   ShenandoahHeapRegion* r;
1440   set->clear_current_index();
1441   while ((r = set->next()) != nullptr) {
1442     r->make_trash();
1443   }
1444   collection_set()->clear();
1445 }
1446 
1447 void ShenandoahHeap::print_heap_regions_on(outputStream* st) const {
1448   st->print_cr("Heap Regions:");
1449   st->print_cr("Region state: EU=empty-uncommitted, EC=empty-committed, R=regular, H=humongous start, HP=pinned humongous start");
1450   st->print_cr("              HC=humongous continuation, CS=collection set, TR=trash, P=pinned, CSP=pinned collection set");
1451   st->print_cr("BTE=bottom/top/end, TAMS=top-at-mark-start");
1452   st->print_cr("UWM=update watermark, U=used");
1453   st->print_cr("T=TLAB allocs, G=GCLAB allocs");
1454   st->print_cr("S=shared allocs, L=live data");
1455   st->print_cr("CP=critical pins");
1456 
1457   for (size_t i = 0; i < num_regions(); i++) {
1458     get_region(i)->print_on(st);
1459   }
1460 }
1461 
1462 size_t ShenandoahHeap::trash_humongous_region_at(ShenandoahHeapRegion* start) {
1463   assert(start->is_humongous_start(), "reclaim regions starting with the first one");
1464 
1465   oop humongous_obj = cast_to_oop(start->bottom());
1466   size_t size = humongous_obj->size();
1467   size_t required_regions = ShenandoahHeapRegion::required_regions(size * HeapWordSize);
1468   size_t index = start->index() + required_regions - 1;
1469 
1470   assert(!start->has_live(), "liveness must be zero");
1471 
1472   for(size_t i = 0; i < required_regions; i++) {
1473     // Reclaim from tail. Otherwise, assertion fails when printing region to trace log,
1474     // as it expects that every region belongs to a humongous region starting with a humongous start region.
1475     ShenandoahHeapRegion* region = get_region(index --);
1476 
1477     assert(region->is_humongous(), "expect correct humongous start or continuation");
1478     assert(!region->is_cset(), "Humongous region should not be in collection set");
1479 
1480     region->make_trash_immediate();
1481   }
1482   return required_regions;
1483 }
1484 
1485 class ShenandoahCheckCleanGCLABClosure : public ThreadClosure {
1486 public:
1487   ShenandoahCheckCleanGCLABClosure() {}
1488   void do_thread(Thread* thread) {
1489     PLAB* gclab = ShenandoahThreadLocalData::gclab(thread);
1490     assert(gclab != nullptr, "GCLAB should be initialized for %s", thread->name());
1491     assert(gclab->words_remaining() == 0, "GCLAB should not need retirement");
1492 
1493     if (ShenandoahHeap::heap()->mode()->is_generational()) {
1494       PLAB* plab = ShenandoahThreadLocalData::plab(thread);
1495       assert(plab != nullptr, "PLAB should be initialized for %s", thread->name());
1496       assert(plab->words_remaining() == 0, "PLAB should not need retirement");
1497     }
1498   }
1499 };
1500 
1501 void ShenandoahHeap::labs_make_parsable() {
1502   assert(UseTLAB, "Only call with UseTLAB");
1503 
1504   ShenandoahRetireGCLABClosure cl(false);
1505 
1506   for (JavaThreadIteratorWithHandle jtiwh; JavaThread *t = jtiwh.next(); ) {
1507     ThreadLocalAllocBuffer& tlab = t->tlab();
1508     tlab.make_parsable();
1509     cl.do_thread(t);
1510   }
1511 
1512   workers()->threads_do(&cl);
1513 
1514   if (safepoint_workers() != nullptr) {
1515     safepoint_workers()->threads_do(&cl);
1516   }
1517 }
1518 
1519 void ShenandoahHeap::tlabs_retire(bool resize) {
1520   assert(UseTLAB, "Only call with UseTLAB");
1521   assert(!resize || ResizeTLAB, "Only call for resize when ResizeTLAB is enabled");
1522 
1523   ThreadLocalAllocStats stats;
1524 
1525   for (JavaThreadIteratorWithHandle jtiwh; JavaThread *t = jtiwh.next(); ) {
1526     ThreadLocalAllocBuffer& tlab = t->tlab();
1527     tlab.retire(&stats);
1528     if (resize) {
1529       tlab.resize();
1530     }
1531   }
1532 
1533   stats.publish();
1534 
1535 #ifdef ASSERT
1536   ShenandoahCheckCleanGCLABClosure cl;
1537   for (JavaThreadIteratorWithHandle jtiwh; JavaThread *t = jtiwh.next(); ) {
1538     cl.do_thread(t);
1539   }
1540   workers()->threads_do(&cl);
1541 #endif
1542 }
1543 
1544 void ShenandoahHeap::gclabs_retire(bool resize) {
1545   assert(UseTLAB, "Only call with UseTLAB");
1546   assert(!resize || ResizeTLAB, "Only call for resize when ResizeTLAB is enabled");
1547 
1548   ShenandoahRetireGCLABClosure cl(resize);
1549   for (JavaThreadIteratorWithHandle jtiwh; JavaThread *t = jtiwh.next(); ) {
1550     cl.do_thread(t);
1551   }
1552 
1553   workers()->threads_do(&cl);
1554 
1555   if (safepoint_workers() != nullptr) {
1556     safepoint_workers()->threads_do(&cl);
1557   }
1558 }
1559 
1560 // Returns size in bytes
1561 size_t ShenandoahHeap::unsafe_max_tlab_alloc(Thread *thread) const {
1562   // Return the max allowed size, and let the allocation path
1563   // figure out the safe size for current allocation.
1564   return ShenandoahHeapRegion::max_tlab_size_bytes();
1565 }
1566 
1567 size_t ShenandoahHeap::max_tlab_size() const {
1568   // Returns size in words
1569   return ShenandoahHeapRegion::max_tlab_size_words();
1570 }
1571 
1572 void ShenandoahHeap::collect_as_vm_thread(GCCause::Cause cause) {
1573   // These requests are ignored because we can't easily have Shenandoah jump into
1574   // a synchronous (degenerated or full) cycle while it is in the middle of a concurrent
1575   // cycle. We _could_ cancel the concurrent cycle and then try to run a cycle directly
1576   // on the VM thread, but this would confuse the control thread mightily and doesn't
1577   // seem worth the trouble. Instead, we will have the caller thread run (and wait for) a
1578   // concurrent cycle in the prologue of the heap inspect/dump operation. This is how
1579   // other concurrent collectors in the JVM handle this scenario as well.
1580   assert(Thread::current()->is_VM_thread(), "Should be the VM thread");
1581   guarantee(cause == GCCause::_heap_dump || cause == GCCause::_heap_inspection, "Invalid cause");
1582 }
1583 
1584 void ShenandoahHeap::collect(GCCause::Cause cause) {
1585   control_thread()->request_gc(cause);
1586 }
1587 
1588 void ShenandoahHeap::do_full_collection(bool clear_all_soft_refs) {
1589   //assert(false, "Shouldn't need to do full collections");
1590 }
1591 
1592 HeapWord* ShenandoahHeap::block_start(const void* addr) const {
1593   ShenandoahHeapRegion* r = heap_region_containing(addr);
1594   if (r != nullptr) {
1595     return r->block_start(addr);
1596   }
1597   return nullptr;
1598 }
1599 
1600 bool ShenandoahHeap::block_is_obj(const HeapWord* addr) const {
1601   ShenandoahHeapRegion* r = heap_region_containing(addr);
1602   return r->block_is_obj(addr);
1603 }
1604 
1605 bool ShenandoahHeap::print_location(outputStream* st, void* addr) const {
1606   return BlockLocationPrinter<ShenandoahHeap>::print_location(st, addr);
1607 }
1608 
1609 void ShenandoahHeap::prepare_for_verify() {
1610   if (SafepointSynchronize::is_at_safepoint() && UseTLAB) {
1611     labs_make_parsable();
1612   }
1613 }
1614 
1615 void ShenandoahHeap::gc_threads_do(ThreadClosure* tcl) const {
1616   if (_shenandoah_policy->is_at_shutdown()) {
1617     return;
1618   }
1619 
1620   if (_control_thread != nullptr) {
1621     tcl->do_thread(_control_thread);
1622   }
1623 
1624   if (_uncommit_thread != nullptr) {
1625     tcl->do_thread(_uncommit_thread);
1626   }
1627 
1628   workers()->threads_do(tcl);
1629   if (_safepoint_workers != nullptr) {
1630     _safepoint_workers->threads_do(tcl);
1631   }
1632 }
1633 
1634 void ShenandoahHeap::print_tracing_info() const {
1635   LogTarget(Info, gc, stats) lt;
1636   if (lt.is_enabled()) {
1637     ResourceMark rm;
1638     LogStream ls(lt);
1639 
1640     phase_timings()->print_global_on(&ls);
1641 
1642     ls.cr();
1643     ls.cr();
1644 
1645     shenandoah_policy()->print_gc_stats(&ls);
1646 
1647     ls.cr();
1648     ls.cr();
1649   }
1650 }
1651 
1652 void ShenandoahHeap::set_gc_generation(ShenandoahGeneration* generation) {
1653   shenandoah_assert_control_or_vm_thread_at_safepoint();
1654   _gc_generation = generation;
1655 }
1656 
1657 // Active generation may only be set by the VM thread at a safepoint.
1658 void ShenandoahHeap::set_active_generation() {
1659   assert(Thread::current()->is_VM_thread(), "Only the VM Thread");
1660   assert(SafepointSynchronize::is_at_safepoint(), "Only at a safepoint!");
1661   assert(_gc_generation != nullptr, "Will set _active_generation to nullptr");
1662   _active_generation = _gc_generation;
1663 }
1664 
1665 void ShenandoahHeap::on_cycle_start(GCCause::Cause cause, ShenandoahGeneration* generation) {
1666   shenandoah_policy()->record_collection_cause(cause);
1667 
1668   const GCCause::Cause current = gc_cause();
1669   assert(current == GCCause::_no_gc, "Over-writing cause: %s, with: %s",
1670          GCCause::to_string(current), GCCause::to_string(cause));
1671   assert(_gc_generation == nullptr, "Over-writing _gc_generation");
1672 
1673   set_gc_cause(cause);
1674   set_gc_generation(generation);
1675 
1676   generation->heuristics()->record_cycle_start();
1677 }
1678 
1679 void ShenandoahHeap::on_cycle_end(ShenandoahGeneration* generation) {
1680   assert(gc_cause() != GCCause::_no_gc, "cause wasn't set");
1681   assert(_gc_generation != nullptr, "_gc_generation wasn't set");
1682 
1683   generation->heuristics()->record_cycle_end();
1684   if (mode()->is_generational() && generation->is_global()) {
1685     // If we just completed a GLOBAL GC, claim credit for completion of young-gen and old-gen GC as well
1686     young_generation()->heuristics()->record_cycle_end();
1687     old_generation()->heuristics()->record_cycle_end();
1688   }
1689 
1690   set_gc_generation(nullptr);
1691   set_gc_cause(GCCause::_no_gc);
1692 }
1693 
1694 void ShenandoahHeap::verify(VerifyOption vo) {
1695   if (ShenandoahSafepoint::is_at_shenandoah_safepoint()) {
1696     if (ShenandoahVerify) {
1697       verifier()->verify_generic(vo);
1698     } else {
1699       // TODO: Consider allocating verification bitmaps on demand,
1700       // and turn this on unconditionally.
1701     }
1702   }
1703 }
1704 size_t ShenandoahHeap::tlab_capacity(Thread *thr) const {
1705   return _free_set->capacity();
1706 }
1707 
1708 class ObjectIterateScanRootClosure : public BasicOopIterateClosure {
1709 private:
1710   MarkBitMap* _bitmap;
1711   ShenandoahScanObjectStack* _oop_stack;
1712   ShenandoahHeap* const _heap;
1713   ShenandoahMarkingContext* const _marking_context;
1714 
1715   template <class T>
1716   void do_oop_work(T* p) {
1717     T o = RawAccess<>::oop_load(p);
1718     if (!CompressedOops::is_null(o)) {
1719       oop obj = CompressedOops::decode_not_null(o);
1720       if (_heap->is_concurrent_weak_root_in_progress() && !_marking_context->is_marked(obj)) {
1721         // There may be dead oops in weak roots in concurrent root phase, do not touch them.
1722         return;
1723       }
1724       obj = ShenandoahBarrierSet::barrier_set()->load_reference_barrier(obj);
1725 
1726       assert(oopDesc::is_oop(obj), "must be a valid oop");
1727       if (!_bitmap->is_marked(obj)) {
1728         _bitmap->mark(obj);
1729         _oop_stack->push(obj);
1730       }
1731     }
1732   }
1733 public:
1734   ObjectIterateScanRootClosure(MarkBitMap* bitmap, ShenandoahScanObjectStack* oop_stack) :
1735     _bitmap(bitmap), _oop_stack(oop_stack), _heap(ShenandoahHeap::heap()),
1736     _marking_context(_heap->marking_context()) {}
1737   void do_oop(oop* p)       { do_oop_work(p); }
1738   void do_oop(narrowOop* p) { do_oop_work(p); }
1739 };
1740 
1741 /*
1742  * This is public API, used in preparation of object_iterate().
1743  * Since we don't do linear scan of heap in object_iterate() (see comment below), we don't
1744  * need to make the heap parsable. For Shenandoah-internal linear heap scans that we can
1745  * control, we call SH::tlabs_retire, SH::gclabs_retire.
1746  */
1747 void ShenandoahHeap::ensure_parsability(bool retire_tlabs) {
1748   // No-op.
1749 }
1750 
1751 /*
1752  * Iterates objects in the heap. This is public API, used for, e.g., heap dumping.
1753  *
1754  * We cannot safely iterate objects by doing a linear scan at random points in time. Linear
1755  * scanning needs to deal with dead objects, which may have dead Klass* pointers (e.g.
1756  * calling oopDesc::size() would crash) or dangling reference fields (crashes) etc. Linear
1757  * scanning therefore depends on having a valid marking bitmap to support it. However, we only
1758  * have a valid marking bitmap after successful marking. In particular, we *don't* have a valid
1759  * marking bitmap during marking, after aborted marking or during/after cleanup (when we just
1760  * wiped the bitmap in preparation for next marking).
1761  *
1762  * For all those reasons, we implement object iteration as a single marking traversal, reporting
1763  * objects as we mark+traverse through the heap, starting from GC roots. JVMTI IterateThroughHeap
1764  * is allowed to report dead objects, but is not required to do so.
1765  */
1766 void ShenandoahHeap::object_iterate(ObjectClosure* cl) {
1767   // Reset bitmap
1768   if (!prepare_aux_bitmap_for_iteration())
1769     return;
1770 
1771   ShenandoahScanObjectStack oop_stack;
1772   ObjectIterateScanRootClosure oops(&_aux_bit_map, &oop_stack);
1773   // Seed the stack with root scan
1774   scan_roots_for_iteration(&oop_stack, &oops);
1775 
1776   // Work through the oop stack to traverse heap
1777   while (! oop_stack.is_empty()) {
1778     oop obj = oop_stack.pop();
1779     assert(oopDesc::is_oop(obj), "must be a valid oop");
1780     cl->do_object(obj);
1781     obj->oop_iterate(&oops);
1782   }
1783 
1784   assert(oop_stack.is_empty(), "should be empty");
1785   // Reclaim bitmap
1786   reclaim_aux_bitmap_for_iteration();
1787 }
1788 
1789 bool ShenandoahHeap::prepare_aux_bitmap_for_iteration() {
1790   assert(SafepointSynchronize::is_at_safepoint(), "safe iteration is only available during safepoints");
1791 
1792   if (!_aux_bitmap_region_special && !os::commit_memory((char*)_aux_bitmap_region.start(), _aux_bitmap_region.byte_size(), false)) {
1793     log_warning(gc)("Could not commit native memory for auxiliary marking bitmap for heap iteration");
1794     return false;
1795   }
1796   // Reset bitmap
1797   _aux_bit_map.clear();
1798   return true;
1799 }
1800 
1801 void ShenandoahHeap::scan_roots_for_iteration(ShenandoahScanObjectStack* oop_stack, ObjectIterateScanRootClosure* oops) {
1802   // Process GC roots according to current GC cycle
1803   // This populates the work stack with initial objects
1804   // It is important to relinquish the associated locks before diving
1805   // into heap dumper
1806   uint n_workers = safepoint_workers() != nullptr ? safepoint_workers()->active_workers() : 1;
1807   ShenandoahHeapIterationRootScanner rp(n_workers);
1808   rp.roots_do(oops);
1809 }
1810 
1811 void ShenandoahHeap::reclaim_aux_bitmap_for_iteration() {
1812   if (!_aux_bitmap_region_special && !os::uncommit_memory((char*)_aux_bitmap_region.start(), _aux_bitmap_region.byte_size())) {
1813     log_warning(gc)("Could not uncommit native memory for auxiliary marking bitmap for heap iteration");
1814   }
1815 }
1816 
1817 // Closure for parallelly iterate objects
1818 class ShenandoahObjectIterateParScanClosure : public BasicOopIterateClosure {
1819 private:
1820   MarkBitMap* _bitmap;
1821   ShenandoahObjToScanQueue* _queue;
1822   ShenandoahHeap* const _heap;
1823   ShenandoahMarkingContext* const _marking_context;
1824 
1825   template <class T>
1826   void do_oop_work(T* p) {
1827     T o = RawAccess<>::oop_load(p);
1828     if (!CompressedOops::is_null(o)) {
1829       oop obj = CompressedOops::decode_not_null(o);
1830       if (_heap->is_concurrent_weak_root_in_progress() && !_marking_context->is_marked(obj)) {
1831         // There may be dead oops in weak roots in concurrent root phase, do not touch them.
1832         return;
1833       }
1834       obj = ShenandoahBarrierSet::barrier_set()->load_reference_barrier(obj);
1835 
1836       assert(oopDesc::is_oop(obj), "Must be a valid oop");
1837       if (_bitmap->par_mark(obj)) {
1838         _queue->push(ShenandoahMarkTask(obj));
1839       }
1840     }
1841   }
1842 public:
1843   ShenandoahObjectIterateParScanClosure(MarkBitMap* bitmap, ShenandoahObjToScanQueue* q) :
1844     _bitmap(bitmap), _queue(q), _heap(ShenandoahHeap::heap()),
1845     _marking_context(_heap->marking_context()) {}
1846   void do_oop(oop* p)       { do_oop_work(p); }
1847   void do_oop(narrowOop* p) { do_oop_work(p); }
1848 };
1849 
1850 // Object iterator for parallel heap iteraion.
1851 // The root scanning phase happenes in construction as a preparation of
1852 // parallel marking queues.
1853 // Every worker processes it's own marking queue. work-stealing is used
1854 // to balance workload.
1855 class ShenandoahParallelObjectIterator : public ParallelObjectIteratorImpl {
1856 private:
1857   uint                         _num_workers;
1858   bool                         _init_ready;
1859   MarkBitMap*                  _aux_bit_map;
1860   ShenandoahHeap*              _heap;
1861   ShenandoahScanObjectStack    _roots_stack; // global roots stack
1862   ShenandoahObjToScanQueueSet* _task_queues;
1863 public:
1864   ShenandoahParallelObjectIterator(uint num_workers, MarkBitMap* bitmap) :
1865         _num_workers(num_workers),
1866         _init_ready(false),
1867         _aux_bit_map(bitmap),
1868         _heap(ShenandoahHeap::heap()) {
1869     // Initialize bitmap
1870     _init_ready = _heap->prepare_aux_bitmap_for_iteration();
1871     if (!_init_ready) {
1872       return;
1873     }
1874 
1875     ObjectIterateScanRootClosure oops(_aux_bit_map, &_roots_stack);
1876     _heap->scan_roots_for_iteration(&_roots_stack, &oops);
1877 
1878     _init_ready = prepare_worker_queues();
1879   }
1880 
1881   ~ShenandoahParallelObjectIterator() {
1882     // Reclaim bitmap
1883     _heap->reclaim_aux_bitmap_for_iteration();
1884     // Reclaim queue for workers
1885     if (_task_queues!= nullptr) {
1886       for (uint i = 0; i < _num_workers; ++i) {
1887         ShenandoahObjToScanQueue* q = _task_queues->queue(i);
1888         if (q != nullptr) {
1889           delete q;
1890           _task_queues->register_queue(i, nullptr);
1891         }
1892       }
1893       delete _task_queues;
1894       _task_queues = nullptr;
1895     }
1896   }
1897 
1898   virtual void object_iterate(ObjectClosure* cl, uint worker_id) {
1899     if (_init_ready) {
1900       object_iterate_parallel(cl, worker_id, _task_queues);
1901     }
1902   }
1903 
1904 private:
1905   // Divide global root_stack into worker queues
1906   bool prepare_worker_queues() {
1907     _task_queues = new ShenandoahObjToScanQueueSet((int) _num_workers);
1908     // Initialize queues for every workers
1909     for (uint i = 0; i < _num_workers; ++i) {
1910       ShenandoahObjToScanQueue* task_queue = new ShenandoahObjToScanQueue();
1911       _task_queues->register_queue(i, task_queue);
1912     }
1913     // Divide roots among the workers. Assume that object referencing distribution
1914     // is related with root kind, use round-robin to make every worker have same chance
1915     // to process every kind of roots
1916     size_t roots_num = _roots_stack.size();
1917     if (roots_num == 0) {
1918       // No work to do
1919       return false;
1920     }
1921 
1922     for (uint j = 0; j < roots_num; j++) {
1923       uint stack_id = j % _num_workers;
1924       oop obj = _roots_stack.pop();
1925       _task_queues->queue(stack_id)->push(ShenandoahMarkTask(obj));
1926     }
1927     return true;
1928   }
1929 
1930   void object_iterate_parallel(ObjectClosure* cl,
1931                                uint worker_id,
1932                                ShenandoahObjToScanQueueSet* queue_set) {
1933     assert(SafepointSynchronize::is_at_safepoint(), "safe iteration is only available during safepoints");
1934     assert(queue_set != nullptr, "task queue must not be null");
1935 
1936     ShenandoahObjToScanQueue* q = queue_set->queue(worker_id);
1937     assert(q != nullptr, "object iterate queue must not be null");
1938 
1939     ShenandoahMarkTask t;
1940     ShenandoahObjectIterateParScanClosure oops(_aux_bit_map, q);
1941 
1942     // Work through the queue to traverse heap.
1943     // Steal when there is no task in queue.
1944     while (q->pop(t) || queue_set->steal(worker_id, t)) {
1945       oop obj = t.obj();
1946       assert(oopDesc::is_oop(obj), "must be a valid oop");
1947       cl->do_object(obj);
1948       obj->oop_iterate(&oops);
1949     }
1950     assert(q->is_empty(), "should be empty");
1951   }
1952 };
1953 
1954 ParallelObjectIteratorImpl* ShenandoahHeap::parallel_object_iterator(uint workers) {
1955   return new ShenandoahParallelObjectIterator(workers, &_aux_bit_map);
1956 }
1957 
1958 // Keep alive an object that was loaded with AS_NO_KEEPALIVE.
1959 void ShenandoahHeap::keep_alive(oop obj) {
1960   if (is_concurrent_mark_in_progress() && (obj != nullptr)) {
1961     ShenandoahBarrierSet::barrier_set()->enqueue(obj);
1962   }
1963 }
1964 
1965 void ShenandoahHeap::heap_region_iterate(ShenandoahHeapRegionClosure* blk) const {
1966   for (size_t i = 0; i < num_regions(); i++) {
1967     ShenandoahHeapRegion* current = get_region(i);
1968     blk->heap_region_do(current);
1969   }
1970 }
1971 
1972 class ShenandoahParallelHeapRegionTask : public WorkerTask {
1973 private:
1974   ShenandoahHeap* const _heap;
1975   ShenandoahHeapRegionClosure* const _blk;
1976   size_t const _stride;
1977 
1978   shenandoah_padding(0);
1979   volatile size_t _index;
1980   shenandoah_padding(1);
1981 
1982 public:
1983   ShenandoahParallelHeapRegionTask(ShenandoahHeapRegionClosure* blk, size_t stride) :
1984           WorkerTask("Shenandoah Parallel Region Operation"),
1985           _heap(ShenandoahHeap::heap()), _blk(blk), _stride(stride), _index(0) {}
1986 
1987   void work(uint worker_id) {
1988     ShenandoahParallelWorkerSession worker_session(worker_id);
1989     size_t stride = _stride;
1990 
1991     size_t max = _heap->num_regions();
1992     while (Atomic::load(&_index) < max) {
1993       size_t cur = Atomic::fetch_then_add(&_index, stride, memory_order_relaxed);
1994       size_t start = cur;
1995       size_t end = MIN2(cur + stride, max);
1996       if (start >= max) break;
1997 
1998       for (size_t i = cur; i < end; i++) {
1999         ShenandoahHeapRegion* current = _heap->get_region(i);
2000         _blk->heap_region_do(current);
2001       }
2002     }
2003   }
2004 };
2005 
2006 void ShenandoahHeap::parallel_heap_region_iterate(ShenandoahHeapRegionClosure* blk) const {
2007   assert(blk->is_thread_safe(), "Only thread-safe closures here");
2008   const uint active_workers = workers()->active_workers();
2009   const size_t n_regions = num_regions();
2010   size_t stride = ShenandoahParallelRegionStride;
2011   if (stride == 0 && active_workers > 1) {
2012     // Automatically derive the stride to balance the work between threads
2013     // evenly. Do not try to split work if below the reasonable threshold.
2014     constexpr size_t threshold = 4096;
2015     stride = n_regions <= threshold ?
2016             threshold :
2017             (n_regions + active_workers - 1) / active_workers;
2018   }
2019 
2020   if (n_regions > stride && active_workers > 1) {
2021     ShenandoahParallelHeapRegionTask task(blk, stride);
2022     workers()->run_task(&task);
2023   } else {
2024     heap_region_iterate(blk);
2025   }
2026 }
2027 
2028 class ShenandoahRendezvousClosure : public HandshakeClosure {
2029 public:
2030   inline ShenandoahRendezvousClosure(const char* name) : HandshakeClosure(name) {}
2031   inline void do_thread(Thread* thread) {}
2032 };
2033 
2034 void ShenandoahHeap::rendezvous_threads(const char* name) {
2035   ShenandoahRendezvousClosure cl(name);
2036   Handshake::execute(&cl);
2037 }
2038 
2039 void ShenandoahHeap::recycle_trash() {
2040   free_set()->recycle_trash();
2041 }
2042 
2043 void ShenandoahHeap::do_class_unloading() {
2044   _unloader.unload();
2045   if (mode()->is_generational()) {
2046     old_generation()->set_parsable(false);
2047   }
2048 }
2049 
2050 void ShenandoahHeap::stw_weak_refs(bool full_gc) {
2051   // Weak refs processing
2052   ShenandoahPhaseTimings::Phase phase = full_gc ? ShenandoahPhaseTimings::full_gc_weakrefs
2053                                                 : ShenandoahPhaseTimings::degen_gc_weakrefs;
2054   ShenandoahTimingsTracker t(phase);
2055   ShenandoahGCWorkerPhase worker_phase(phase);
2056   shenandoah_assert_generations_reconciled();
2057   gc_generation()->ref_processor()->process_references(phase, workers(), false /* concurrent */);
2058 }
2059 
2060 void ShenandoahHeap::prepare_update_heap_references() {
2061   assert(ShenandoahSafepoint::is_at_shenandoah_safepoint(), "must be at safepoint");
2062 
2063   // Evacuation is over, no GCLABs are needed anymore. GCLABs are under URWM, so we need to
2064   // make them parsable for update code to work correctly. Plus, we can compute new sizes
2065   // for future GCLABs here.
2066   if (UseTLAB) {
2067     ShenandoahGCPhase phase(ShenandoahPhaseTimings::degen_gc_init_update_refs_manage_gclabs);
2068     gclabs_retire(ResizeTLAB);
2069   }
2070 
2071   _update_refs_iterator.reset();
2072 }
2073 
2074 void ShenandoahHeap::propagate_gc_state_to_all_threads() {
2075   assert(ShenandoahSafepoint::is_at_shenandoah_safepoint(), "Must be at Shenandoah safepoint");
2076   if (_gc_state_changed) {
2077     ShenandoahGCStatePropagator propagator(_gc_state.raw_value());
2078     Threads::threads_do(&propagator);
2079     _gc_state_changed = false;
2080   }
2081 }
2082 
2083 void ShenandoahHeap::set_gc_state_at_safepoint(uint mask, bool value) {
2084   assert(ShenandoahSafepoint::is_at_shenandoah_safepoint(), "Must be at Shenandoah safepoint");
2085   _gc_state.set_cond(mask, value);
2086   _gc_state_changed = true;
2087 }
2088 
2089 void ShenandoahHeap::set_gc_state_concurrent(uint mask, bool value) {
2090   // Holding the thread lock here assures that any thread created after we change the gc
2091   // state will have the correct state. It also prevents attaching threads from seeing
2092   // an inconsistent state. See ShenandoahBarrierSet::on_thread_attach for reference. Established
2093   // threads will use their thread local copy of the gc state (changed by a handshake, or on a
2094   // safepoint).
2095   assert(Threads_lock->is_locked(), "Must hold thread lock for concurrent gc state change");
2096   _gc_state.set_cond(mask, value);
2097 }
2098 
2099 void ShenandoahHeap::set_concurrent_young_mark_in_progress(bool in_progress) {
2100   uint mask;
2101   assert(!has_forwarded_objects(), "Young marking is not concurrent with evacuation");
2102   if (!in_progress && is_concurrent_old_mark_in_progress()) {
2103     assert(mode()->is_generational(), "Only generational GC has old marking");
2104     assert(_gc_state.is_set(MARKING), "concurrent_old_marking_in_progress implies MARKING");
2105     // If old-marking is in progress when we turn off YOUNG_MARKING, leave MARKING (and OLD_MARKING) on
2106     mask = YOUNG_MARKING;
2107   } else {
2108     mask = MARKING | YOUNG_MARKING;
2109   }
2110   set_gc_state_at_safepoint(mask, in_progress);
2111   manage_satb_barrier(in_progress);
2112 }
2113 
2114 void ShenandoahHeap::set_concurrent_old_mark_in_progress(bool in_progress) {
2115 #ifdef ASSERT
2116   // has_forwarded_objects() iff UPDATE_REFS or EVACUATION
2117   bool has_forwarded = has_forwarded_objects();
2118   bool updating_or_evacuating = _gc_state.is_set(UPDATE_REFS | EVACUATION);
2119   bool evacuating = _gc_state.is_set(EVACUATION);
2120   assert ((has_forwarded == updating_or_evacuating) || (evacuating && !has_forwarded && collection_set()->is_empty()),
2121           "Updating or evacuating iff has forwarded objects, or if evacuation phase is promoting in place without forwarding");
2122 #endif
2123   if (!in_progress && is_concurrent_young_mark_in_progress()) {
2124     // If young-marking is in progress when we turn off OLD_MARKING, leave MARKING (and YOUNG_MARKING) on
2125     assert(_gc_state.is_set(MARKING), "concurrent_young_marking_in_progress implies MARKING");
2126     set_gc_state_at_safepoint(OLD_MARKING, in_progress);
2127   } else {
2128     set_gc_state_at_safepoint(MARKING | OLD_MARKING, in_progress);
2129   }
2130   manage_satb_barrier(in_progress);
2131 }
2132 
2133 bool ShenandoahHeap::is_prepare_for_old_mark_in_progress() const {
2134   return old_generation()->is_preparing_for_mark();
2135 }
2136 
2137 void ShenandoahHeap::manage_satb_barrier(bool active) {
2138   if (is_concurrent_mark_in_progress()) {
2139     // Ignore request to deactivate barrier while concurrent mark is in progress.
2140     // Do not attempt to re-activate the barrier if it is already active.
2141     if (active && !ShenandoahBarrierSet::satb_mark_queue_set().is_active()) {
2142       ShenandoahBarrierSet::satb_mark_queue_set().set_active_all_threads(active, !active);
2143     }
2144   } else {
2145     // No concurrent marking is in progress so honor request to deactivate,
2146     // but only if the barrier is already active.
2147     if (!active && ShenandoahBarrierSet::satb_mark_queue_set().is_active()) {
2148       ShenandoahBarrierSet::satb_mark_queue_set().set_active_all_threads(active, !active);
2149     }
2150   }
2151 }
2152 
2153 void ShenandoahHeap::set_evacuation_in_progress(bool in_progress) {
2154   assert(ShenandoahSafepoint::is_at_shenandoah_safepoint(), "Only call this at safepoint");
2155   set_gc_state_at_safepoint(EVACUATION, in_progress);
2156 }
2157 
2158 void ShenandoahHeap::set_concurrent_strong_root_in_progress(bool in_progress) {
2159   if (in_progress) {
2160     _concurrent_strong_root_in_progress.set();
2161   } else {
2162     _concurrent_strong_root_in_progress.unset();
2163   }
2164 }
2165 
2166 void ShenandoahHeap::set_concurrent_weak_root_in_progress(bool cond) {
2167   set_gc_state_at_safepoint(WEAK_ROOTS, cond);
2168 }
2169 
2170 GCTracer* ShenandoahHeap::tracer() {
2171   return shenandoah_policy()->tracer();
2172 }
2173 
2174 size_t ShenandoahHeap::tlab_used(Thread* thread) const {
2175   return _free_set->used();
2176 }
2177 
2178 bool ShenandoahHeap::try_cancel_gc(GCCause::Cause cause) {
2179   const GCCause::Cause prev = _cancelled_gc.xchg(cause);
2180   return prev == GCCause::_no_gc || prev == GCCause::_shenandoah_concurrent_gc;
2181 }
2182 
2183 void ShenandoahHeap::cancel_concurrent_mark() {
2184   if (mode()->is_generational()) {
2185     young_generation()->cancel_marking();
2186     old_generation()->cancel_marking();
2187   }
2188 
2189   global_generation()->cancel_marking();
2190 
2191   ShenandoahBarrierSet::satb_mark_queue_set().abandon_partial_marking();
2192 }
2193 
2194 bool ShenandoahHeap::cancel_gc(GCCause::Cause cause) {
2195   if (try_cancel_gc(cause)) {
2196     FormatBuffer<> msg("Cancelling GC: %s", GCCause::to_string(cause));
2197     log_info(gc,thread)("%s", msg.buffer());
2198     Events::log(Thread::current(), "%s", msg.buffer());
2199     _cancel_requested_time = os::elapsedTime();
2200     return true;
2201   }
2202   return false;
2203 }
2204 
2205 uint ShenandoahHeap::max_workers() {
2206   return _max_workers;
2207 }
2208 
2209 void ShenandoahHeap::stop() {
2210   // The shutdown sequence should be able to terminate when GC is running.
2211 
2212   // Step 0. Notify policy to disable event recording and prevent visiting gc threads during shutdown
2213   _shenandoah_policy->record_shutdown();
2214 
2215   // Step 1. Stop reporting on gc thread cpu utilization
2216   mmu_tracker()->stop();
2217 
2218   // Step 2. Wait until GC worker exits normally (this will cancel any ongoing GC).
2219   control_thread()->stop();
2220 
2221   // Stop 4. Shutdown uncommit thread.
2222   if (_uncommit_thread != nullptr) {
2223     _uncommit_thread->stop();
2224   }
2225 }
2226 
2227 void ShenandoahHeap::stw_unload_classes(bool full_gc) {
2228   if (!unload_classes()) return;
2229   ClassUnloadingContext ctx(_workers->active_workers(),
2230                             true /* unregister_nmethods_during_purge */,
2231                             false /* lock_nmethod_free_separately */);
2232 
2233   // Unload classes and purge SystemDictionary.
2234   {
2235     ShenandoahPhaseTimings::Phase phase = full_gc ?
2236                                           ShenandoahPhaseTimings::full_gc_purge_class_unload :
2237                                           ShenandoahPhaseTimings::degen_gc_purge_class_unload;
2238     ShenandoahIsAliveSelector is_alive;
2239     {
2240       CodeCache::UnlinkingScope scope(is_alive.is_alive_closure());
2241       ShenandoahGCPhase gc_phase(phase);
2242       ShenandoahGCWorkerPhase worker_phase(phase);
2243       bool unloading_occurred = SystemDictionary::do_unloading(gc_timer());
2244 
2245       // Clean JVMCI metadata handles.
2246       JVMCI_ONLY(JVMCI::do_unloading(unloading_occurred));
2247 
2248       uint num_workers = _workers->active_workers();
2249       ShenandoahClassUnloadingTask unlink_task(phase, num_workers, unloading_occurred);
2250       _workers->run_task(&unlink_task);
2251     }
2252     // Release unloaded nmethods's memory.
2253     ClassUnloadingContext::context()->purge_and_free_nmethods();
2254   }
2255 
2256   {
2257     ShenandoahGCPhase phase(full_gc ?
2258                             ShenandoahPhaseTimings::full_gc_purge_cldg :
2259                             ShenandoahPhaseTimings::degen_gc_purge_cldg);
2260     ClassLoaderDataGraph::purge(true /* at_safepoint */);
2261   }
2262   // Resize and verify metaspace
2263   MetaspaceGC::compute_new_size();
2264   DEBUG_ONLY(MetaspaceUtils::verify();)
2265 }
2266 
2267 // Weak roots are either pre-evacuated (final mark) or updated (final update refs),
2268 // so they should not have forwarded oops.
2269 // However, we do need to "null" dead oops in the roots, if can not be done
2270 // in concurrent cycles.
2271 void ShenandoahHeap::stw_process_weak_roots(bool full_gc) {
2272   uint num_workers = _workers->active_workers();
2273   ShenandoahPhaseTimings::Phase timing_phase = full_gc ?
2274                                                ShenandoahPhaseTimings::full_gc_purge_weak_par :
2275                                                ShenandoahPhaseTimings::degen_gc_purge_weak_par;
2276   ShenandoahGCPhase phase(timing_phase);
2277   ShenandoahGCWorkerPhase worker_phase(timing_phase);
2278   // Cleanup weak roots
2279   if (has_forwarded_objects()) {
2280     ShenandoahForwardedIsAliveClosure is_alive;
2281     ShenandoahNonConcUpdateRefsClosure keep_alive;
2282     ShenandoahParallelWeakRootsCleaningTask<ShenandoahForwardedIsAliveClosure, ShenandoahNonConcUpdateRefsClosure>
2283       cleaning_task(timing_phase, &is_alive, &keep_alive, num_workers);
2284     _workers->run_task(&cleaning_task);
2285   } else {
2286     ShenandoahIsAliveClosure is_alive;
2287 #ifdef ASSERT
2288     ShenandoahAssertNotForwardedClosure verify_cl;
2289     ShenandoahParallelWeakRootsCleaningTask<ShenandoahIsAliveClosure, ShenandoahAssertNotForwardedClosure>
2290       cleaning_task(timing_phase, &is_alive, &verify_cl, num_workers);
2291 #else
2292     ShenandoahParallelWeakRootsCleaningTask<ShenandoahIsAliveClosure, DoNothingClosure>
2293       cleaning_task(timing_phase, &is_alive, &do_nothing_cl, num_workers);
2294 #endif
2295     _workers->run_task(&cleaning_task);
2296   }
2297 }
2298 
2299 void ShenandoahHeap::parallel_cleaning(bool full_gc) {
2300   assert(SafepointSynchronize::is_at_safepoint(), "Must be at a safepoint");
2301   assert(is_stw_gc_in_progress(), "Only for Degenerated and Full GC");
2302   ShenandoahGCPhase phase(full_gc ?
2303                           ShenandoahPhaseTimings::full_gc_purge :
2304                           ShenandoahPhaseTimings::degen_gc_purge);
2305   stw_weak_refs(full_gc);
2306   stw_process_weak_roots(full_gc);
2307   stw_unload_classes(full_gc);
2308 }
2309 
2310 void ShenandoahHeap::set_has_forwarded_objects(bool cond) {
2311   set_gc_state_at_safepoint(HAS_FORWARDED, cond);
2312 }
2313 
2314 void ShenandoahHeap::set_unload_classes(bool uc) {
2315   _unload_classes.set_cond(uc);
2316 }
2317 
2318 bool ShenandoahHeap::unload_classes() const {
2319   return _unload_classes.is_set();
2320 }
2321 
2322 address ShenandoahHeap::in_cset_fast_test_addr() {
2323   ShenandoahHeap* heap = ShenandoahHeap::heap();
2324   assert(heap->collection_set() != nullptr, "Sanity");
2325   return (address) heap->collection_set()->biased_map_address();
2326 }
2327 
2328 void ShenandoahHeap::reset_bytes_allocated_since_gc_start() {
2329   if (mode()->is_generational()) {
2330     young_generation()->reset_bytes_allocated_since_gc_start();
2331     old_generation()->reset_bytes_allocated_since_gc_start();
2332   }
2333 
2334   global_generation()->reset_bytes_allocated_since_gc_start();
2335 }
2336 
2337 void ShenandoahHeap::set_degenerated_gc_in_progress(bool in_progress) {
2338   _degenerated_gc_in_progress.set_cond(in_progress);
2339 }
2340 
2341 void ShenandoahHeap::set_full_gc_in_progress(bool in_progress) {
2342   _full_gc_in_progress.set_cond(in_progress);
2343 }
2344 
2345 void ShenandoahHeap::set_full_gc_move_in_progress(bool in_progress) {
2346   assert (is_full_gc_in_progress(), "should be");
2347   _full_gc_move_in_progress.set_cond(in_progress);
2348 }
2349 
2350 void ShenandoahHeap::set_update_refs_in_progress(bool in_progress) {
2351   set_gc_state_at_safepoint(UPDATE_REFS, in_progress);
2352 }
2353 
2354 void ShenandoahHeap::register_nmethod(nmethod* nm) {
2355   ShenandoahCodeRoots::register_nmethod(nm);
2356 }
2357 
2358 void ShenandoahHeap::unregister_nmethod(nmethod* nm) {
2359   ShenandoahCodeRoots::unregister_nmethod(nm);
2360 }
2361 
2362 void ShenandoahHeap::pin_object(JavaThread* thr, oop o) {
2363   heap_region_containing(o)->record_pin();
2364 }
2365 
2366 void ShenandoahHeap::unpin_object(JavaThread* thr, oop o) {
2367   ShenandoahHeapRegion* r = heap_region_containing(o);
2368   assert(r != nullptr, "Sanity");
2369   assert(r->pin_count() > 0, "Region %zu should have non-zero pins", r->index());
2370   r->record_unpin();
2371 }
2372 
2373 void ShenandoahHeap::sync_pinned_region_status() {
2374   ShenandoahHeapLocker locker(lock());
2375 
2376   for (size_t i = 0; i < num_regions(); i++) {
2377     ShenandoahHeapRegion *r = get_region(i);
2378     if (r->is_active()) {
2379       if (r->is_pinned()) {
2380         if (r->pin_count() == 0) {
2381           r->make_unpinned();
2382         }
2383       } else {
2384         if (r->pin_count() > 0) {
2385           r->make_pinned();
2386         }
2387       }
2388     }
2389   }
2390 
2391   assert_pinned_region_status();
2392 }
2393 
2394 #ifdef ASSERT
2395 void ShenandoahHeap::assert_pinned_region_status() {
2396   for (size_t i = 0; i < num_regions(); i++) {
2397     ShenandoahHeapRegion* r = get_region(i);
2398     shenandoah_assert_generations_reconciled();
2399     if (gc_generation()->contains(r)) {
2400       assert((r->is_pinned() && r->pin_count() > 0) || (!r->is_pinned() && r->pin_count() == 0),
2401              "Region %zu pinning status is inconsistent", i);
2402     }
2403   }
2404 }
2405 #endif
2406 
2407 ConcurrentGCTimer* ShenandoahHeap::gc_timer() const {
2408   return _gc_timer;
2409 }
2410 
2411 void ShenandoahHeap::prepare_concurrent_roots() {
2412   assert(SafepointSynchronize::is_at_safepoint(), "Must be at a safepoint");
2413   assert(!is_stw_gc_in_progress(), "Only concurrent GC");
2414   set_concurrent_strong_root_in_progress(!collection_set()->is_empty());
2415   set_concurrent_weak_root_in_progress(true);
2416   if (unload_classes()) {
2417     _unloader.prepare();
2418   }
2419 }
2420 
2421 void ShenandoahHeap::finish_concurrent_roots() {
2422   assert(SafepointSynchronize::is_at_safepoint(), "Must be at a safepoint");
2423   assert(!is_stw_gc_in_progress(), "Only concurrent GC");
2424   if (unload_classes()) {
2425     _unloader.finish();
2426   }
2427 }
2428 
2429 #ifdef ASSERT
2430 void ShenandoahHeap::assert_gc_workers(uint nworkers) {
2431   assert(nworkers > 0 && nworkers <= max_workers(), "Sanity");
2432 
2433   if (ShenandoahSafepoint::is_at_shenandoah_safepoint()) {
2434     // Use ParallelGCThreads inside safepoints
2435     assert(nworkers == ParallelGCThreads, "Use ParallelGCThreads (%u) within safepoint, not %u",
2436            ParallelGCThreads, nworkers);
2437   } else {
2438     // Use ConcGCThreads outside safepoints
2439     assert(nworkers == ConcGCThreads, "Use ConcGCThreads (%u) outside safepoints, %u",
2440            ConcGCThreads, nworkers);
2441   }
2442 }
2443 #endif
2444 
2445 ShenandoahVerifier* ShenandoahHeap::verifier() {
2446   guarantee(ShenandoahVerify, "Should be enabled");
2447   assert (_verifier != nullptr, "sanity");
2448   return _verifier;
2449 }
2450 
2451 template<bool CONCURRENT>
2452 class ShenandoahUpdateHeapRefsTask : public WorkerTask {
2453 private:
2454   ShenandoahHeap* _heap;
2455   ShenandoahRegionIterator* _regions;
2456 public:
2457   explicit ShenandoahUpdateHeapRefsTask(ShenandoahRegionIterator* regions) :
2458     WorkerTask("Shenandoah Update References"),
2459     _heap(ShenandoahHeap::heap()),
2460     _regions(regions) {
2461   }
2462 
2463   void work(uint worker_id) {
2464     if (CONCURRENT) {
2465       ShenandoahConcurrentWorkerSession worker_session(worker_id);
2466       ShenandoahSuspendibleThreadSetJoiner stsj;
2467       do_work<ShenandoahConcUpdateRefsClosure>(worker_id);
2468     } else {
2469       ShenandoahParallelWorkerSession worker_session(worker_id);
2470       do_work<ShenandoahNonConcUpdateRefsClosure>(worker_id);
2471     }
2472   }
2473 
2474 private:
2475   template<class T>
2476   void do_work(uint worker_id) {
2477     if (CONCURRENT && (worker_id == 0)) {
2478       // We ask the first worker to replenish the Mutator free set by moving regions previously reserved to hold the
2479       // results of evacuation.  These reserves are no longer necessary because evacuation has completed.
2480       size_t cset_regions = _heap->collection_set()->count();
2481 
2482       // Now that evacuation is done, we can reassign any regions that had been reserved to hold the results of evacuation
2483       // to the mutator free set.  At the end of GC, we will have cset_regions newly evacuated fully empty regions from
2484       // which we will be able to replenish the Collector free set and the OldCollector free set in preparation for the
2485       // next GC cycle.
2486       _heap->free_set()->move_regions_from_collector_to_mutator(cset_regions);
2487     }
2488     // If !CONCURRENT, there's no value in expanding Mutator free set
2489     T cl;
2490     ShenandoahHeapRegion* r = _regions->next();
2491     while (r != nullptr) {
2492       HeapWord* update_watermark = r->get_update_watermark();
2493       assert (update_watermark >= r->bottom(), "sanity");
2494       if (r->is_active() && !r->is_cset()) {
2495         _heap->marked_object_oop_iterate(r, &cl, update_watermark);
2496         if (ShenandoahPacing) {
2497           _heap->pacer()->report_update_refs(pointer_delta(update_watermark, r->bottom()));
2498         }
2499       }
2500       if (_heap->check_cancelled_gc_and_yield(CONCURRENT)) {
2501         return;
2502       }
2503       r = _regions->next();
2504     }
2505   }
2506 };
2507 
2508 void ShenandoahHeap::update_heap_references(bool concurrent) {
2509   assert(!is_full_gc_in_progress(), "Only for concurrent and degenerated GC");
2510 
2511   if (concurrent) {
2512     ShenandoahUpdateHeapRefsTask<true> task(&_update_refs_iterator);
2513     workers()->run_task(&task);
2514   } else {
2515     ShenandoahUpdateHeapRefsTask<false> task(&_update_refs_iterator);
2516     workers()->run_task(&task);
2517   }
2518 }
2519 
2520 void ShenandoahHeap::update_heap_region_states(bool concurrent) {
2521   assert(SafepointSynchronize::is_at_safepoint(), "Must be at a safepoint");
2522   assert(!is_full_gc_in_progress(), "Only for concurrent and degenerated GC");
2523 
2524   {
2525     ShenandoahGCPhase phase(concurrent ?
2526                             ShenandoahPhaseTimings::final_update_refs_update_region_states :
2527                             ShenandoahPhaseTimings::degen_gc_final_update_refs_update_region_states);
2528 
2529     final_update_refs_update_region_states();
2530 
2531     assert_pinned_region_status();
2532   }
2533 
2534   {
2535     ShenandoahGCPhase phase(concurrent ?
2536                             ShenandoahPhaseTimings::final_update_refs_trash_cset :
2537                             ShenandoahPhaseTimings::degen_gc_final_update_refs_trash_cset);
2538     trash_cset_regions();
2539   }
2540 }
2541 
2542 void ShenandoahHeap::final_update_refs_update_region_states() {
2543   ShenandoahSynchronizePinnedRegionStates cl;
2544   parallel_heap_region_iterate(&cl);
2545 }
2546 
2547 void ShenandoahHeap::rebuild_free_set(bool concurrent) {
2548   ShenandoahGCPhase phase(concurrent ?
2549                           ShenandoahPhaseTimings::final_update_refs_rebuild_freeset :
2550                           ShenandoahPhaseTimings::degen_gc_final_update_refs_rebuild_freeset);
2551   ShenandoahHeapLocker locker(lock());
2552   size_t young_cset_regions, old_cset_regions;
2553   size_t first_old_region, last_old_region, old_region_count;
2554   _free_set->prepare_to_rebuild(young_cset_regions, old_cset_regions, first_old_region, last_old_region, old_region_count);
2555   // If there are no old regions, first_old_region will be greater than last_old_region
2556   assert((first_old_region > last_old_region) ||
2557          ((last_old_region + 1 - first_old_region >= old_region_count) &&
2558           get_region(first_old_region)->is_old() && get_region(last_old_region)->is_old()),
2559          "sanity: old_region_count: %zu, first_old_region: %zu, last_old_region: %zu",
2560          old_region_count, first_old_region, last_old_region);
2561 
2562   if (mode()->is_generational()) {
2563 #ifdef ASSERT
2564     if (ShenandoahVerify) {
2565       verifier()->verify_before_rebuilding_free_set();
2566     }
2567 #endif
2568 
2569     // The computation of bytes_of_allocation_runway_before_gc_trigger is quite conservative so consider all of this
2570     // available for transfer to old. Note that transfer of humongous regions does not impact available.
2571     ShenandoahGenerationalHeap* gen_heap = ShenandoahGenerationalHeap::heap();
2572     size_t allocation_runway = gen_heap->young_generation()->heuristics()->bytes_of_allocation_runway_before_gc_trigger(young_cset_regions);
2573     gen_heap->compute_old_generation_balance(allocation_runway, old_cset_regions);
2574 
2575     // Total old_available may have been expanded to hold anticipated promotions.  We trigger if the fragmented available
2576     // memory represents more than 16 regions worth of data.  Note that fragmentation may increase when we promote regular
2577     // regions in place when many of these regular regions have an abundant amount of available memory within them.  Fragmentation
2578     // will decrease as promote-by-copy consumes the available memory within these partially consumed regions.
2579     //
2580     // We consider old-gen to have excessive fragmentation if more than 12.5% of old-gen is free memory that resides
2581     // within partially consumed regions of memory.
2582   }
2583   // Rebuild free set based on adjusted generation sizes.
2584   _free_set->finish_rebuild(young_cset_regions, old_cset_regions, old_region_count);
2585 
2586   if (mode()->is_generational()) {
2587     ShenandoahGenerationalHeap* gen_heap = ShenandoahGenerationalHeap::heap();
2588     ShenandoahOldGeneration* old_gen = gen_heap->old_generation();
2589     old_gen->heuristics()->evaluate_triggers(first_old_region, last_old_region, old_region_count, num_regions());
2590   }
2591 }
2592 
2593 bool ShenandoahHeap::is_bitmap_slice_committed(ShenandoahHeapRegion* r, bool skip_self) {
2594   size_t slice = r->index() / _bitmap_regions_per_slice;
2595 
2596   size_t regions_from = _bitmap_regions_per_slice * slice;
2597   size_t regions_to   = MIN2(num_regions(), _bitmap_regions_per_slice * (slice + 1));
2598   for (size_t g = regions_from; g < regions_to; g++) {
2599     assert (g / _bitmap_regions_per_slice == slice, "same slice");
2600     if (skip_self && g == r->index()) continue;
2601     if (get_region(g)->is_committed()) {
2602       return true;
2603     }
2604   }
2605   return false;
2606 }
2607 
2608 bool ShenandoahHeap::commit_bitmap_slice(ShenandoahHeapRegion* r) {
2609   shenandoah_assert_heaplocked();
2610 
2611   // Bitmaps in special regions do not need commits
2612   if (_bitmap_region_special) {
2613     return true;
2614   }
2615 
2616   if (is_bitmap_slice_committed(r, true)) {
2617     // Some other region from the group is already committed, meaning the bitmap
2618     // slice is already committed, we exit right away.
2619     return true;
2620   }
2621 
2622   // Commit the bitmap slice:
2623   size_t slice = r->index() / _bitmap_regions_per_slice;
2624   size_t off = _bitmap_bytes_per_slice * slice;
2625   size_t len = _bitmap_bytes_per_slice;
2626   char* start = (char*) _bitmap_region.start() + off;
2627 
2628   if (!os::commit_memory(start, len, false)) {
2629     return false;
2630   }
2631 
2632   if (AlwaysPreTouch) {
2633     os::pretouch_memory(start, start + len, _pretouch_bitmap_page_size);
2634   }
2635 
2636   return true;
2637 }
2638 
2639 bool ShenandoahHeap::uncommit_bitmap_slice(ShenandoahHeapRegion *r) {
2640   shenandoah_assert_heaplocked();
2641 
2642   // Bitmaps in special regions do not need uncommits
2643   if (_bitmap_region_special) {
2644     return true;
2645   }
2646 
2647   if (is_bitmap_slice_committed(r, true)) {
2648     // Some other region from the group is still committed, meaning the bitmap
2649     // slice should stay committed, exit right away.
2650     return true;
2651   }
2652 
2653   // Uncommit the bitmap slice:
2654   size_t slice = r->index() / _bitmap_regions_per_slice;
2655   size_t off = _bitmap_bytes_per_slice * slice;
2656   size_t len = _bitmap_bytes_per_slice;
2657   if (!os::uncommit_memory((char*)_bitmap_region.start() + off, len)) {
2658     return false;
2659   }
2660   return true;
2661 }
2662 
2663 void ShenandoahHeap::forbid_uncommit() {
2664   if (_uncommit_thread != nullptr) {
2665     _uncommit_thread->forbid_uncommit();
2666   }
2667 }
2668 
2669 void ShenandoahHeap::allow_uncommit() {
2670   if (_uncommit_thread != nullptr) {
2671     _uncommit_thread->allow_uncommit();
2672   }
2673 }
2674 
2675 #ifdef ASSERT
2676 bool ShenandoahHeap::is_uncommit_in_progress() {
2677   if (_uncommit_thread != nullptr) {
2678     return _uncommit_thread->is_uncommit_in_progress();
2679   }
2680   return false;
2681 }
2682 #endif
2683 
2684 void ShenandoahHeap::safepoint_synchronize_begin() {
2685   StackWatermarkSet::safepoint_synchronize_begin();
2686   SuspendibleThreadSet::synchronize();
2687 }
2688 
2689 void ShenandoahHeap::safepoint_synchronize_end() {
2690   SuspendibleThreadSet::desynchronize();
2691 }
2692 
2693 void ShenandoahHeap::try_inject_alloc_failure() {
2694   if (ShenandoahAllocFailureALot && !cancelled_gc() && ((os::random() % 1000) > 950)) {
2695     _inject_alloc_failure.set();
2696     os::naked_short_sleep(1);
2697     if (cancelled_gc()) {
2698       log_info(gc)("Allocation failure was successfully injected");
2699     }
2700   }
2701 }
2702 
2703 bool ShenandoahHeap::should_inject_alloc_failure() {
2704   return _inject_alloc_failure.is_set() && _inject_alloc_failure.try_unset();
2705 }
2706 
2707 void ShenandoahHeap::initialize_serviceability() {
2708   _memory_pool = new ShenandoahMemoryPool(this);
2709   _cycle_memory_manager.add_pool(_memory_pool);
2710   _stw_memory_manager.add_pool(_memory_pool);
2711 }
2712 
2713 GrowableArray<GCMemoryManager*> ShenandoahHeap::memory_managers() {
2714   GrowableArray<GCMemoryManager*> memory_managers(2);
2715   memory_managers.append(&_cycle_memory_manager);
2716   memory_managers.append(&_stw_memory_manager);
2717   return memory_managers;
2718 }
2719 
2720 GrowableArray<MemoryPool*> ShenandoahHeap::memory_pools() {
2721   GrowableArray<MemoryPool*> memory_pools(1);
2722   memory_pools.append(_memory_pool);
2723   return memory_pools;
2724 }
2725 
2726 MemoryUsage ShenandoahHeap::memory_usage() {
2727   return MemoryUsage(_initial_size, used(), committed(), max_capacity());
2728 }
2729 
2730 ShenandoahRegionIterator::ShenandoahRegionIterator() :
2731   _heap(ShenandoahHeap::heap()),
2732   _index(0) {}
2733 
2734 ShenandoahRegionIterator::ShenandoahRegionIterator(ShenandoahHeap* heap) :
2735   _heap(heap),
2736   _index(0) {}
2737 
2738 void ShenandoahRegionIterator::reset() {
2739   _index = 0;
2740 }
2741 
2742 bool ShenandoahRegionIterator::has_next() const {
2743   return _index < _heap->num_regions();
2744 }
2745 
2746 char ShenandoahHeap::gc_state() const {
2747   return _gc_state.raw_value();
2748 }
2749 
2750 bool ShenandoahHeap::is_gc_state(GCState state) const {
2751   // If the global gc state has been changed, but hasn't yet been propagated to all threads, then
2752   // the global gc state is the correct value. Once the gc state has been synchronized with all threads,
2753   // _gc_state_changed will be toggled to false and we need to use the thread local state.
2754   return _gc_state_changed ? _gc_state.is_set(state) : ShenandoahThreadLocalData::is_gc_state(state);
2755 }
2756 
2757 
2758 ShenandoahLiveData* ShenandoahHeap::get_liveness_cache(uint worker_id) {
2759 #ifdef ASSERT
2760   assert(_liveness_cache != nullptr, "sanity");
2761   assert(worker_id < _max_workers, "sanity");
2762   for (uint i = 0; i < num_regions(); i++) {
2763     assert(_liveness_cache[worker_id][i] == 0, "liveness cache should be empty");
2764   }
2765 #endif
2766   return _liveness_cache[worker_id];
2767 }
2768 
2769 void ShenandoahHeap::flush_liveness_cache(uint worker_id) {
2770   assert(worker_id < _max_workers, "sanity");
2771   assert(_liveness_cache != nullptr, "sanity");
2772   ShenandoahLiveData* ld = _liveness_cache[worker_id];
2773   for (uint i = 0; i < num_regions(); i++) {
2774     ShenandoahLiveData live = ld[i];
2775     if (live > 0) {
2776       ShenandoahHeapRegion* r = get_region(i);
2777       r->increase_live_data_gc_words(live);
2778       ld[i] = 0;
2779     }
2780   }
2781 }
2782 
2783 bool ShenandoahHeap::requires_barriers(stackChunkOop obj) const {
2784   if (is_idle()) return false;
2785 
2786   // Objects allocated after marking start are implicitly alive, don't need any barriers during
2787   // marking phase.
2788   if (is_concurrent_mark_in_progress() &&
2789      !marking_context()->allocated_after_mark_start(obj)) {
2790     return true;
2791   }
2792 
2793   // Can not guarantee obj is deeply good.
2794   if (has_forwarded_objects()) {
2795     return true;
2796   }
2797 
2798   return false;
2799 }
2800 
2801 HeapWord* ShenandoahHeap::allocate_loaded_archive_space(size_t size) {
2802 #if INCLUDE_CDS_JAVA_HEAP
2803   // CDS wants a continuous memory range to load a bunch of objects.
2804   // This effectively bypasses normal allocation paths, and requires
2805   // a bit of massaging to unbreak GC invariants.
2806 
2807   ShenandoahAllocRequest req = ShenandoahAllocRequest::for_shared(size);
2808 
2809   // Easy case: a single regular region, no further adjustments needed.
2810   if (!ShenandoahHeapRegion::requires_humongous(size)) {
2811     return allocate_memory(req);
2812   }
2813 
2814   // Hard case: the requested size would cause a humongous allocation.
2815   // We need to make sure it looks like regular allocation to the rest of GC.
2816 
2817   // CDS code would guarantee no objects straddle multiple regions, as long as
2818   // regions are as large as MIN_GC_REGION_ALIGNMENT. It is impractical at this
2819   // point to deal with case when Shenandoah runs with smaller regions.
2820   // TODO: This check can be dropped once MIN_GC_REGION_ALIGNMENT agrees more with Shenandoah.
2821   if (ShenandoahHeapRegion::region_size_bytes() < ArchiveHeapWriter::MIN_GC_REGION_ALIGNMENT) {
2822     return nullptr;
2823   }
2824 
2825   HeapWord* mem = allocate_memory(req);
2826   size_t start_idx = heap_region_index_containing(mem);
2827   size_t num_regions = ShenandoahHeapRegion::required_regions(size * HeapWordSize);
2828 
2829   // Flip humongous -> regular.
2830   {
2831     ShenandoahHeapLocker locker(lock(), false);
2832     for (size_t c = start_idx; c < start_idx + num_regions; c++) {
2833       get_region(c)->make_regular_bypass();
2834     }
2835   }
2836 
2837   return mem;
2838 #else
2839   assert(false, "Archive heap loader should not be available, should not be here");
2840   return nullptr;
2841 #endif // INCLUDE_CDS_JAVA_HEAP
2842 }
2843 
2844 void ShenandoahHeap::complete_loaded_archive_space(MemRegion archive_space) {
2845   // Nothing to do here, except checking that heap looks fine.
2846 #ifdef ASSERT
2847   HeapWord* start = archive_space.start();
2848   HeapWord* end = archive_space.end();
2849 
2850   // No unclaimed space between the objects.
2851   // Objects are properly allocated in correct regions.
2852   HeapWord* cur = start;
2853   while (cur < end) {
2854     oop oop = cast_to_oop(cur);
2855     shenandoah_assert_in_correct_region(nullptr, oop);
2856     cur += oop->size();
2857   }
2858 
2859   // No unclaimed tail at the end of archive space.
2860   assert(cur == end,
2861          "Archive space should be fully used: " PTR_FORMAT " " PTR_FORMAT,
2862          p2i(cur), p2i(end));
2863 
2864   // Region bounds are good.
2865   ShenandoahHeapRegion* begin_reg = heap_region_containing(start);
2866   ShenandoahHeapRegion* end_reg = heap_region_containing(end);
2867   assert(begin_reg->is_regular(), "Must be");
2868   assert(end_reg->is_regular(), "Must be");
2869   assert(begin_reg->bottom() == start,
2870          "Must agree: archive-space-start: " PTR_FORMAT ", begin-region-bottom: " PTR_FORMAT,
2871          p2i(start), p2i(begin_reg->bottom()));
2872   assert(end_reg->top() == end,
2873          "Must agree: archive-space-end: " PTR_FORMAT ", end-region-top: " PTR_FORMAT,
2874          p2i(end), p2i(end_reg->top()));
2875 #endif
2876 }
2877 
2878 ShenandoahGeneration* ShenandoahHeap::generation_for(ShenandoahAffiliation affiliation) const {
2879   if (!mode()->is_generational()) {
2880     return global_generation();
2881   } else if (affiliation == YOUNG_GENERATION) {
2882     return young_generation();
2883   } else if (affiliation == OLD_GENERATION) {
2884     return old_generation();
2885   }
2886 
2887   ShouldNotReachHere();
2888   return nullptr;
2889 }
2890 
2891 void ShenandoahHeap::log_heap_status(const char* msg) const {
2892   if (mode()->is_generational()) {
2893     young_generation()->log_status(msg);
2894     old_generation()->log_status(msg);
2895   } else {
2896     global_generation()->log_status(msg);
2897   }
2898 }